diff --git a/.drone.yml b/.drone.yml index 9bdf616aa..8cb331bfb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,22 +2,22 @@ kind: pipeline type: docker name: unit - workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: unit pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - go version - ./test.sh - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username @@ -48,20 +48,22 @@ name: integration-internal workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: integration-internal pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - data/docker/integration/docker-entrypoint.sh - ./integration_test.sh -dbtype=*internal + failure: ignore - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username @@ -83,7 +85,7 @@ steps: recipients_file: recipients recipients_only: true when: - status: failure + status: failure --- kind: pipeline @@ -92,20 +94,22 @@ name: integration-mysql workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: integration-mysql pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - data/docker/integration/docker-entrypoint.sh - ./integration_test.sh -dbtype=*mysql + failure: ignore - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username @@ -136,20 +140,22 @@ name: integration-mongo workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: integration-mongo pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - data/docker/integration/docker-entrypoint.sh - ./integration_test.sh -dbtype=*mongo + failure: ignore - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username @@ -180,20 +186,22 @@ name: integration-postgres workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: integration-postgres pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - data/docker/integration/docker-entrypoint.sh - ./integration_test.sh -dbtype=*postgres + failure: ignore - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username diff --git a/data/docker/integration/Dockerfile b/data/docker/integration/Dockerfile index 8536af44a..f915c73c1 100644 --- a/data/docker/integration/Dockerfile +++ b/data/docker/integration/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster-slim +FROM ubuntu:jammy RUN groupadd -r mongodb && useradd -r -g mongodb mongodb RUN groupadd -r redis && useradd -r -g redis redis @@ -6,37 +6,35 @@ RUN groupadd -r kafka && useradd -r -g kafka kafka RUN mkdir -p /usr/share/cgrates/conf /usr/share/man/man1 /var/spool/cgrates/ers/in /var/spool/cgrates/ers/out /var/spool/cgrates/cdre/csv /var/spool/cgrates/cdre/fwv /var/spool/cgrates/tpe /var/spool/cgrates/failed_posts /var/spool/cgrates/analyzers /run /data/configdb /data/db /kafka /logs -RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections -# Install necessary libs -RUN apt-get update +ENV DEBIAN_FRONTEND=noninteractive + + +RUN apt-get update && apt-get install -y ansible + +# Install Erlang packages +RUN apt-get install -y erlang-base erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key erlang-runtime-tools erlang-snmp erlang-ssl erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl + RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk-headless neovim net-tools -RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - -RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list +RUN curl -fsSL https://pgp.mongodb.com/server-7.0.asc | gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor +RUN echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list -RUN wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list - -RUN wget https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb -RUN dpkg -i erlang-solutions_1.0_all.deb -RUN apt-get update -RUN apt-get install -y erlang erlang-nox -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash +RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN apt-get update -RUN apt-get install -y mongodb-org postgresql rabbitmq-server - +RUN apt-get install -y mongodb-org postgresql +# ADD THIS AFTER postgresql rabbitmq-server WORKDIR /kafka -RUN wget "https://archive.apache.org/dist/kafka/2.1.1/kafka_2.11-2.1.1.tgz" -RUN tar -xvzf ./kafka_2.11-2.1.1.tgz --strip 1 -RUN rm kafka_2.11-2.1.1.tgz +RUN wget "https://archive.apache.org/dist/kafka/2.1.0/kafka_2.11-2.1.0.tgz" +RUN tar -xvzf ./kafka_2.11-2.1.0.tgz --strip 1 +RUN rm kafka_2.11-2.1.0.tgz COPY ./conf/server.properties /kafka/config/server.properties WORKDIR / - RUN set -ex; \ rm -rf /var/lib/apt/lists/*; \ # purge and re-create /var/lib/mysql with appropriate ownership @@ -52,8 +50,18 @@ RUN set -ex; \ # don't reverse lookup hostnames, they are usually another container echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf +# Clean up +RUN apt-get clean && \ + rm -rf /var/lib/apt/lists/* -RUN wget -O go.tgz "https://storage.googleapis.com/golang/go1.18.linux-amd64.tar.gz" --progress=dot:giga +RUN touch /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log +RUN chmod 777 /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log + +COPY main.yaml /integration_tests.yaml + +COPY roles /roles + +RUN wget -O go.tgz "https://storage.googleapis.com/golang/go1.21.0.linux-amd64.tar.gz" --progress=dot:giga RUN tar -C /usr/local -xzf go.tgz RUN rm go.tgz @@ -62,11 +70,6 @@ ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" RUN go version - -RUN touch /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log -RUN chmod 777 /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log - - COPY ./scripts /scripts COPY ./scripts/service /usr/local/bin/service diff --git a/data/docker/integration/docker-entrypoint.sh b/data/docker/integration/docker-entrypoint.sh index 55a8426c6..8efaa8b0b 100755 --- a/data/docker/integration/docker-entrypoint.sh +++ b/data/docker/integration/docker-entrypoint.sh @@ -1,60 +1,73 @@ #!/bin/bash set -ev +ansible-playbook /integration_tests.yaml -i localhost, -c local -e "ansible_user=root clone_repository=true" -vvv + +# # Create symbolic links +ln -s "/go/src/github.com/cgrates/cgrates/data" "/usr/share/cgrates" +ln -s "/go/bin/cgr-engine" "/usr/bin/cgr-engine" +ln -s "/go/bin/cgr-loader" "/usr/bin/cgr-loader" +ln -s "/go/bin/cgr-migrator" "/usr/bin/cgr-migrator" +ln -s "/go/bin/cgr-console" "/usr/bin/cgr-console" +ln -s "/go/bin/cgr-tester" "/usr/bin/cgr-tester" + # start basic subsystems -export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M" -/kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties -/kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties +# export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M" +# /kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties +# /kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties rsyslogd -f /etc/rsyslogd.conf version=$(ls /var/lib/postgresql) pg_ctlcluster $version main start & mongod --bind_ip 127.0.0.1 --logpath /logs/mongodb.log & redis-server /etc/redis/redis.conf & -MYSQL_ROOT_PASSWORD="CGRateS.org" /scripts/mariadb-ep.sh mysqld > /logs/mariadb_script.log 2>&1 -rabbitmq-server > /logs/rabbitmq.log 2>&1 & +MYSQL_ROOT_PASSWORD="CGRateS.org" /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mariadb-ep.sh mysqld +# rabbitmq-server > /logs/rabbitmq.log 2>&1 & -START_TIMEOUT=600 +# START_TIMEOUT=600 -start_timeout_exceeded=false -count=0 -step=10 -while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do - echo "waiting for kafka to be ready" - sleep $step; - count=$((count + step)) - if [ $count -gt $START_TIMEOUT ]; then - start_timeout_exceeded=true - break - fi -done +# start_timeout_exceeded=false +# count=0 +# step=10 +# while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do +# echo "waiting for kafka to be ready" +# sleep $step; +# count=$((count + step)) +# if [ $count -gt $START_TIMEOUT ]; then +# start_timeout_exceeded=true +# break +# fi +# done -if $start_timeout_exceeded; then - echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" - exit 1 -fi +# if $start_timeout_exceeded; then +# echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" +# exit 1 +# fi -/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates -/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs +# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates +# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs -gosu postgres psql -c "CREATE USER cgrates password 'CGRateS.org';" > /dev/null 2>&1 -gosu postgres createdb -e -O cgrates cgrates > /dev/null 2>&1 +# gosu postgres psql -c "CREATE USER cgrates password 'CGRateS.org';" > /dev/null 2>&1 +# gosu postgres createdb -e -O cgrates cgrates > /dev/null 2>&1 -PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /scripts/postgres/create_cdrs_tables.sql >/dev/null 2>&1 -PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /scripts/postgres/create_tariffplan_tables.sql >/dev/null 2>&1 +# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_cdrs_tables.sql >/dev/null 2>&1 +# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql >/dev/null 2>&1 -mongo --quiet /scripts/create_user.js >/dev/null 2>&1 +# mongo --quiet /cgrates/data/docker/integration/scripts/create_user.js >/dev/null 2>&1 +echo "Starting..." +sleep 60 # Pause for 1 min +echo "Resuming..." +mysql -u root -pCGRateS.org -h localhost < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_db_with_users.sql +mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_cdrs_tables.sql +mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql +cp -r /go/src/github.com/cgrates/cgrates/data/. /usr/share/cgrates -mysql -u root -pCGRateS.org -h localhost < /scripts/mysql/create_db_with_users.sql > /dev/null 2>&1 -mysql -u root -pCGRateS.org -h localhost < /scripts/mysql/create_db_with_users_extra.sql > /dev/null 2>&1 -mysql -u root -pCGRateS.org -h localhost -D cgrates < /scripts/mysql/create_cdrs_tables.sql > /dev/null 2>&1 -mysql -u root -pCGRateS.org -h localhost -D cgrates < /scripts/mysql/create_tariffplan_tables.sql > /dev/null 2>&1 - -cp -r data/. /usr/share/cgrates \ No newline at end of file +# Set versions +# cgr-migrator -exec=*set_versions -config_path=/usr/share/cgrates/conf/samples/tutmysql \ No newline at end of file diff --git a/data/docker/integration/main.yaml b/data/docker/integration/main.yaml new file mode 100644 index 000000000..e4e52f0ed --- /dev/null +++ b/data/docker/integration/main.yaml @@ -0,0 +1,46 @@ +--- +- hosts: all + vars: + golang_gopath: "/go" + clone_repository: true + cgrates_dir: "{{ golang_gopath }}/src/github.com/cgrates/cgrates" + golang_install_dir: /usr/local/go + git_version: "1.0" + cgrates_dependencies: + - git + - redis-server + - mariadb-server + - make + - gcc + + tasks: + - name: Install CGRateS dependencies + become: yes + ansible.builtin.package: + name: "{{ cgrates_dependencies }}" + state: present + update_cache: yes + cache_valid_time: 86400 + + - name: Create cgrates directory + ansible.builtin.file: + state: directory + mode: "u=rwx,go=rx" + owner: "{{ ansible_user }}" + dest: "{{ cgrates_dir }}" + when: clone_repository | bool + + - name: Git clone cgrates + ansible.builtin.git: + repo: https://github.com/cgrates/cgrates.git + dest: "{{ cgrates_dir }}" + update: yes + force: yes + version: "{{ git_version }}" + when: clone_repository | bool + + - name: Build cgrates + ansible.builtin.shell: + cmd: bash -lc "sh {{ cgrates_dir }}/build.sh" + args: + chdir: "{{ cgrates_dir }}" diff --git a/data/docker/integration/scripts/mongo/create_user.js b/data/docker/integration/scripts/mongo/create_user.js new file mode 100644 index 000000000..169461df6 --- /dev/null +++ b/data/docker/integration/scripts/mongo/create_user.js @@ -0,0 +1,9 @@ +db = db.getSiblingDB('cgrates') +db.createUser( + { + user: "cgrates", + pwd: "CGRateS.org", + roles: [ { role: "dbAdmin", db: "cgrates" } ] + } +) + diff --git a/data/docker/integration/scripts/mongo/setup_cgr_db.sh b/data/docker/integration/scripts/mongo/setup_cgr_db.sh new file mode 100755 index 000000000..4cbc93516 --- /dev/null +++ b/data/docker/integration/scripts/mongo/setup_cgr_db.sh @@ -0,0 +1,14 @@ +#! /usr/bin/env sh + + +mongo --quiet create_user.js +cu=$? + +if [ $cu = 0 ]; then + echo "" + echo "\t+++ CGR-DB successfully set-up! +++" + echo "" + exit 0 +fi + + diff --git a/data/docker/integration/scripts/mysql/alter_cdr_tables_rc5_rc6.sql b/data/docker/integration/scripts/mysql/alter_cdr_tables_rc5_rc6.sql new file mode 100644 index 000000000..e3ddad60b --- /dev/null +++ b/data/docker/integration/scripts/mysql/alter_cdr_tables_rc5_rc6.sql @@ -0,0 +1,50 @@ +USE `cgrates`; + +ALTER TABLE `cdrs_primary` + CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first , + CHANGE `cgrid` `cgrid` char(40) NOT NULL after `id` , + ADD COLUMN `pdd` decimal(12,9) NOT NULL after `setup_time` , + CHANGE `answer_time` `answer_time` datetime NULL after `pdd` , + ADD COLUMN `supplier` varchar(128) NOT NULL after `usage` , + ADD COLUMN `disconnect_cause` varchar(64) NOT NULL after `supplier` , + ADD COLUMN `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP after `disconnect_cause` , + ADD COLUMN `deleted_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at` , + ADD KEY `answer_time_idx`(`answer_time`) , + ADD KEY `deleted_at_idx`(`deleted_at`) , + DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ; + +ALTER TABLE `cdrs_extra` + CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first , + CHANGE `cgrid` `cgrid` char(40) NOT NULL after `id` , + ADD COLUMN `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP after `extra_fields` , + ADD COLUMN `deleted_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at`, + ADD UNIQUE KEY `cgrid`(`cgrid`) , + ADD KEY `deleted_at_idx`(`deleted_at`) , + DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ; + +ALTER TABLE `cost_details` + CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first , + CHANGE `cost_source` `cost_source` varchar(64) NOT NULL after `timespans` , + ADD COLUMN `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP after `cost_source` , + ADD COLUMN `updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at` , + ADD COLUMN `deleted_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `updated_at` , + DROP COLUMN `cost_time` , + ADD KEY `deleted_at_idx`(`deleted_at`) , + DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ; + +ALTER TABLE `rated_cdrs` + CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first , + CHANGE `cgrid` `cgrid` char(40) NOT NULL after `id` , + CHANGE `category` `category` varchar(32) NOT NULL after `tenant` , + ADD COLUMN `pdd` decimal(12,9) NOT NULL after `setup_time` , + CHANGE `answer_time` `answer_time` datetime NULL after `pdd` , + ADD COLUMN `supplier` varchar(128) NOT NULL after `usage` , + ADD COLUMN `disconnect_cause` varchar(64) NOT NULL after `supplier` , + CHANGE `cost` `cost` decimal(20,4) NULL after `disconnect_cause` , + ADD COLUMN `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP after `extra_info` , + ADD COLUMN `updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at` , + ADD COLUMN `deleted_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `updated_at` , + DROP COLUMN `mediation_time` , + ADD KEY `deleted_at_idx`(`deleted_at`) , + DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ; + diff --git a/data/docker/integration/scripts/mysql/create_cdrs_tables.sql b/data/docker/integration/scripts/mysql/create_cdrs_tables.sql new file mode 100644 index 000000000..a563b1bac --- /dev/null +++ b/data/docker/integration/scripts/mysql/create_cdrs_tables.sql @@ -0,0 +1,52 @@ +-- +-- Table structure for table `cdrs` +-- + +DROP TABLE IF EXISTS cdrs; +CREATE TABLE cdrs ( + id int(11) NOT NULL AUTO_INCREMENT, + cgrid varchar(40) NOT NULL, + run_id varchar(64) NOT NULL, + origin_host varchar(64) NOT NULL, + source varchar(64) NOT NULL, + origin_id varchar(128) NOT NULL, + tor varchar(16) NOT NULL, + request_type varchar(24) NOT NULL, + tenant varchar(64) NOT NULL, + category varchar(64) NOT NULL, + account varchar(128) NOT NULL, + subject varchar(128) NOT NULL, + destination varchar(128) NOT NULL, + setup_time datetime NOT NULL, + answer_time datetime NULL, + `usage` BIGINT NOT NULL, + extra_fields text NOT NULL, + cost_source varchar(64) NOT NULL, + cost DECIMAL(20,4) NOT NULL, + cost_details MEDIUMTEXT, + extra_info text, + created_at TIMESTAMP NULL, + updated_at TIMESTAMP NULL, + deleted_at TIMESTAMP NULL, + PRIMARY KEY (id), + UNIQUE KEY cdrrun (cgrid, run_id) +); + +DROP TABLE IF EXISTS session_costs; +CREATE TABLE session_costs ( + id int(11) NOT NULL AUTO_INCREMENT, + cgrid varchar(40) NOT NULL, + run_id varchar(64) NOT NULL, + origin_host varchar(64) NOT NULL, + origin_id varchar(128) NOT NULL, + cost_source varchar(64) NOT NULL, + `usage` BIGINT NOT NULL, + cost_details MEDIUMTEXT, + created_at TIMESTAMP NULL, + deleted_at TIMESTAMP NULL, + PRIMARY KEY (`id`), + UNIQUE KEY costid (cgrid, run_id), + KEY origin_idx (origin_host, origin_id), + KEY run_origin_idx (run_id, origin_id), + KEY deleted_at_idx (deleted_at) +); diff --git a/data/docker/integration/scripts/mysql/create_db_with_users.sql b/data/docker/integration/scripts/mysql/create_db_with_users.sql new file mode 100644 index 000000000..3a9e88304 --- /dev/null +++ b/data/docker/integration/scripts/mysql/create_db_with_users.sql @@ -0,0 +1,10 @@ + +-- +-- Sample db and users creation. Replace here with your own details +-- + +DROP DATABASE IF EXISTS cgrates; +CREATE DATABASE cgrates; +CREATE USER IF NOT EXISTS 'cgrates'@'localhost' IDENTIFIED BY 'CGRateS.org'; +GRANT ALL PRIVILEGES ON cgrates.* TO 'cgrates'@'localhost' WITH GRANT OPTION; +FLUSH PRIVILEGES; diff --git a/data/docker/integration/scripts/mysql/create_ers_db.sql b/data/docker/integration/scripts/mysql/create_ers_db.sql new file mode 100644 index 000000000..99a6bc15e --- /dev/null +++ b/data/docker/integration/scripts/mysql/create_ers_db.sql @@ -0,0 +1,7 @@ + +-- +-- extra DB for ees and ers +DROP DATABASE IF EXISTS cgrates2; +CREATE DATABASE cgrates2; + +GRANT ALL on cgrates2.* TO 'cgrates'@'localhost' IDENTIFIED BY 'CGRateS.org'; diff --git a/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql b/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql new file mode 100644 index 000000000..c184be757 --- /dev/null +++ b/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql @@ -0,0 +1,491 @@ +-- +-- Table structure for table `tp_timings` +-- +DROP TABLE IF EXISTS `tp_timings`; +CREATE TABLE `tp_timings` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `years` varchar(255) NOT NULL, + `months` varchar(255) NOT NULL, + `month_days` varchar(255) NOT NULL, + `week_days` varchar(255) NOT NULL, + `time` varchar(32) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_tmid` (`tpid`,`tag`), + UNIQUE KEY `tpid_tag` (`tpid`,`tag`) +); + +-- +-- Table structure for table `tp_destinations` +-- + +DROP TABLE IF EXISTS `tp_destinations`; +CREATE TABLE `tp_destinations` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `prefix` varchar(24) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_dstid` (`tpid`,`tag`), + UNIQUE KEY `tpid_dest_prefix` (`tpid`,`tag`,`prefix`) +); + +-- +-- Table structure for table `tp_rates` +-- + +DROP TABLE IF EXISTS `tp_rates`; +CREATE TABLE `tp_rates` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `connect_fee` decimal(7,4) NOT NULL, + `rate` decimal(10,4) NOT NULL, + `rate_unit` varchar(16) NOT NULL, + `rate_increment` varchar(16) NOT NULL, + `group_interval_start` varchar(16) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `unique_tprate` (`tpid`,`tag`,`group_interval_start`), + KEY `tpid` (`tpid`), + KEY `tpid_rtid` (`tpid`,`tag`) +); + +-- +-- Table structure for table `destination_rates` +-- + +DROP TABLE IF EXISTS `tp_destination_rates`; +CREATE TABLE `tp_destination_rates` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `destinations_tag` varchar(64) NOT NULL, + `rates_tag` varchar(64) NOT NULL, + `rounding_method` varchar(255) NOT NULL, + `rounding_decimals` tinyint(4) NOT NULL, + `max_cost` decimal(7,4) NOT NULL, + `max_cost_strategy` varchar(16) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_drid` (`tpid`,`tag`), + UNIQUE KEY `tpid_drid_dstid` (`tpid`,`tag`,`destinations_tag`) +); + +-- +-- Table structure for table `tp_rating_plans` +-- + +DROP TABLE IF EXISTS `tp_rating_plans`; +CREATE TABLE `tp_rating_plans` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `destrates_tag` varchar(64) NOT NULL, + `timing_tag` varchar(64) NOT NULL, + `weight` DECIMAL(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_rpl` (`tpid`,`tag`), + UNIQUE KEY `tpid_rplid_destrates_timings_weight` (`tpid`,`tag`,`destrates_tag`,`timing_tag`) +); + +-- +-- Table structure for table `tp_rate_profiles` +-- + +DROP TABLE IF EXISTS `tp_rating_profiles`; +CREATE TABLE `tp_rating_profiles` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `loadid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `category` varchar(32) NOT NULL, + `subject` varchar(64) NOT NULL, + `activation_time` varchar(26) NOT NULL, + `rating_plan_tag` varchar(64) NOT NULL, + `fallback_subjects` varchar(64), + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_loadid` (`tpid`, `loadid`), + UNIQUE KEY `tpid_loadid_tenant_category_subj_atime` (`tpid`,`loadid`, `tenant`,`category`,`subject`,`activation_time`) +); + +-- +-- Table structure for table `tp_shared_groups` +-- + +DROP TABLE IF EXISTS `tp_shared_groups`; +CREATE TABLE `tp_shared_groups` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `account` varchar(64) NOT NULL, + `strategy` varchar(24) NOT NULL, + `rating_subject` varchar(24) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_shared_group` (`tpid`,`tag`,`account`,`strategy`,`rating_subject`) +); + +-- +-- Table structure for table `tp_actions` +-- + +DROP TABLE IF EXISTS `tp_actions`; +CREATE TABLE `tp_actions` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `action` varchar(24) NOT NULL, + `extra_parameters` varchar(256) NOT NULL, + `filters` varchar(256) NOT NULL, + `balance_tag` varchar(64) NOT NULL, + `balance_type` varchar(24) NOT NULL, + `categories` varchar(32) NOT NULL, + `destination_tags` varchar(64) NOT NULL, + `rating_subject` varchar(64) NOT NULL, + `shared_groups` varchar(64) NOT NULL, + `expiry_time` varchar(26) NOT NULL, + `timing_tags` varchar(128) NOT NULL, + `units` varchar(256) NOT NULL, + `balance_weight` varchar(10) NOT NULL, + `balance_blocker` varchar(5) NOT NULL, + `balance_disabled` varchar(24) NOT NULL, + `weight` DECIMAL(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_action` (`tpid`,`tag`,`action`,`balance_tag`,`balance_type`,`expiry_time`,`timing_tags`,`destination_tags`,`shared_groups`,`balance_weight`,`weight`) +); + +-- +-- Table structure for table `tp_action_timings` +-- + +DROP TABLE IF EXISTS `tp_action_plans`; +CREATE TABLE `tp_action_plans` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `actions_tag` varchar(64) NOT NULL, + `timing_tag` varchar(64) NOT NULL, + `weight` DECIMAL(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_action_schedule` (`tpid`,`tag`,`actions_tag`,`timing_tag`) +); + +-- +-- Table structure for table `tp_action_triggers` +-- + +DROP TABLE IF EXISTS `tp_action_triggers`; +CREATE TABLE `tp_action_triggers` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `unique_id` varchar(64) NOT NULL, + `threshold_type` char(64) NOT NULL, + `threshold_value` DECIMAL(20,4) NOT NULL, + `recurrent` BOOLEAN NOT NULL, + `min_sleep` varchar(16) NOT NULL, + `expiry_time` varchar(26) NOT NULL, + `activation_time` varchar(26) NOT NULL, + `balance_tag` varchar(64) NOT NULL, + `balance_type` varchar(24) NOT NULL, + `balance_categories` varchar(32) NOT NULL, + `balance_destination_tags` varchar(64) NOT NULL, + `balance_rating_subject` varchar(64) NOT NULL, + `balance_shared_groups` varchar(64) NOT NULL, + `balance_expiry_time` varchar(26) NOT NULL, + `balance_timing_tags` varchar(128) NOT NULL, + `balance_weight` varchar(10) NOT NULL, + `balance_blocker` varchar(5) NOT NULL, + `balance_disabled` varchar(5) NOT NULL, + `actions_tag` varchar(64) NOT NULL, + `weight` DECIMAL(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_trigger_definition` (`tpid`,`tag`,`balance_tag`,`balance_type`,`threshold_type`,`threshold_value`,`balance_destination_tags`,`actions_tag`) +); + +-- +-- Table structure for table `tp_account_actions` +-- + +DROP TABLE IF EXISTS `tp_account_actions`; +CREATE TABLE `tp_account_actions` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `loadid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `account` varchar(64) NOT NULL, + `action_plan_tag` varchar(64), + `action_triggers_tag` varchar(64), + `allow_negative` BOOLEAN NOT NULL, + `disabled` BOOLEAN NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_account` (`tpid`,`loadid`,`tenant`,`account`) +); + +-- +-- Table structure for table `tp_resources` +-- + +DROP TABLE IF EXISTS tp_resources; +CREATE TABLE tp_resources ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `usage_ttl` varchar(32) NOT NULL, + `limit` varchar(64) NOT NULL, + `allocation_message` varchar(64) NOT NULL, + `blocker` BOOLEAN NOT NULL, + `stored` BOOLEAN NOT NULL, + `weight` decimal(8,2) NOT NULL, + `threshold_ids` varchar(64) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_resource` (`tpid`,`tenant`, `id`,`filter_ids` ) +); + +-- +-- Table structure for table `tp_stats` +-- + +DROP TABLE IF EXISTS tp_stats; +CREATE TABLE tp_stats ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `queue_length` int(11) NOT NULL, + `ttl` varchar(32) NOT NULL, + `min_items` int(11) NOT NULL, + `metric_ids` varchar(128) NOT NULL, + `metric_filter_ids` varchar(64) NOT NULL, + `stored` BOOLEAN NOT NULL, + `blocker` BOOLEAN NOT NULL, + `weight` decimal(8,2) NOT NULL, + `threshold_ids` varchar(64) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_stats` (`tpid`, `tenant`, `id`, `filter_ids`,`metric_ids`) +); + +-- +-- Table structure for table `tp_threshold_cfgs` +-- + +DROP TABLE IF EXISTS tp_thresholds; +CREATE TABLE tp_thresholds ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `max_hits` int(11) NOT NULL, + `min_hits` int(11) NOT NULL, + `min_sleep` varchar(16) NOT NULL, + `blocker` BOOLEAN NOT NULL, + `weight` decimal(8,2) NOT NULL, + `action_ids` varchar(64) NOT NULL, + `async` BOOLEAN NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_thresholds` (`tpid`,`tenant`, `id`,`filter_ids`,`action_ids`) +); + +-- +-- Table structure for table `tp_filter` +-- + +DROP TABLE IF EXISTS tp_filters; +CREATE TABLE tp_filters ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `type` varchar(16) NOT NULL, + `element` varchar(64) NOT NULL, + `values` varchar(256) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_filters` (`tpid`,`tenant`, `id`, `type`, `element`) +); + +-- +-- Table structure for table `tp_routes` +-- + + +DROP TABLE IF EXISTS tp_routes; +CREATE TABLE tp_routes ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `sorting` varchar(32) NOT NULL, + `sorting_parameters` varchar(64) NOT NULL, + `route_id` varchar(32) NOT NULL, + `route_filter_ids` varchar(64) NOT NULL, + `route_account_ids` varchar(64) NOT NULL, + `route_ratingplan_ids` varchar(64) NOT NULL, + `route_rate_profile_ids` varchar(64) NOT NULL, + `route_resource_ids` varchar(64) NOT NULL, + `route_stat_ids` varchar(64) NOT NULL, + `route_weight` decimal(8,2) NOT NULL, + `route_blocker` BOOLEAN NOT NULL, + `route_parameters` varchar(64) NOT NULL, + `weight` decimal(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_routes` (`tpid`,`tenant`, + `id`,`filter_ids`,`route_id`,`route_filter_ids`,`route_account_ids`, + `route_ratingplan_ids`,`route_resource_ids`,`route_stat_ids` ) +); + +-- +-- Table structure for table `tp_attributes` +-- + +DROP TABLE IF EXISTS tp_attributes; +CREATE TABLE tp_attributes ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `contexts` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `attribute_filter_ids` varchar(64) NOT NULL, + `path` varchar(64) NOT NULL, + `type` varchar(64) NOT NULL, + `value` varchar(64) NOT NULL, + `blocker` BOOLEAN NOT NULL, + `weight` decimal(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_attributes` (`tpid`,`tenant`, + `id`,`filter_ids`,`path`,`value` ) +); + +-- +-- Table structure for table `tp_chargers` +-- + +DROP TABLE IF EXISTS tp_chargers; +CREATE TABLE tp_chargers ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `run_id` varchar(64) NOT NULL, + `attribute_ids` varchar(64) NOT NULL, + `weight` decimal(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_chargers` (`tpid`,`tenant`, + `id`,`filter_ids`,`run_id`,`attribute_ids`) +); + +-- +-- Table structure for table `tp_dispatchers` +-- + +DROP TABLE IF EXISTS tp_dispatcher_profiles; +CREATE TABLE tp_dispatcher_profiles ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `subsystems` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `strategy` varchar(64) NOT NULL, + `strategy_parameters` varchar(64) NOT NULL, + `conn_id` varchar(64) NOT NULL, + `conn_filter_ids` varchar(64) NOT NULL, + `conn_weight` decimal(8,2) NOT NULL, + `conn_blocker` BOOLEAN NOT NULL, + `conn_parameters` varchar(64) NOT NULL, + `weight` decimal(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_dispatcher_profiles` (`tpid`,`tenant`, + `id`,`filter_ids`,`strategy`,`conn_id`,`conn_filter_ids`) +); + +-- +-- Table structure for table `tp_dispatchers` +-- + +DROP TABLE IF EXISTS tp_dispatcher_hosts; +CREATE TABLE tp_dispatcher_hosts ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `address` varchar(64) NOT NULL, + `transport` varchar(64) NOT NULL, + `connect_attempts` int(11) NOT NULL, + `reconnects` int(11) NOT NULL, + `max_reconnect_interval` varchar(64) NOT NULL, + `connect_timeout` varchar(64) NOT NULL, + `reply_timeout` varchar(64) NOT NULL, + `tls` BOOLEAN NOT NULL, + `client_key` varchar(64) NOT NULL, + `client_certificate` varchar(64) NOT NULL, + `ca_certificate` varchar(64) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_dispatchers_hosts` (`tpid`,`tenant`, + `id`,`address`) +); + +-- +-- Table structure for table `versions` +-- + +DROP TABLE IF EXISTS versions; +CREATE TABLE versions ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `item` varchar(64) NOT NULL, + `version` int(11) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `id_item` (`id`,`item`) +); diff --git a/data/docker/integration/scripts/mysql/mysql_cdr_migration.sql b/data/docker/integration/scripts/mysql/mysql_cdr_migration.sql new file mode 100644 index 000000000..b44d1596d --- /dev/null +++ b/data/docker/integration/scripts/mysql/mysql_cdr_migration.sql @@ -0,0 +1,57 @@ +/* +This script will migrate CDRs from the old CGRateS tables to the new cdrs table +but it only migrate CDRs where the duration is > 0. +If you need CDRs also with duration is = 0 you can make the appropriate change in the line beginning WHERE cdrs_primary.usage +Also the script will process 10,000 CDRs before committing to save system resources +especially in systems where they are millions of CDRs to be migrated +You can increase or lower the value of step in the line after BEGIN below. + +You have to use 'CALL cgrates.migration();' to execute the script. If named other then default use that database name. +*/ + + +DELIMITER // + +CREATE PROCEDURE `migration`() +BEGIN + /* DECLARE variables */ + DECLARE max_cdrs bigint; + DECLARE start_id bigint; + DECLARE end_id bigint; + DECLARE step bigint; + /* Optimize table for performance */ + ALTER TABLE cdrs DISABLE KEYS; + SET autocommit=0; + SET unique_checks=0; + SET foreign_key_checks=0; + /* You must change the step var to commit every step rows inserted */ + SET step := 10000; + SET start_id := 0; + SET end_id := start_id + step; + SET max_cdrs = (select max(id) from rated_cdrs); + WHILE (start_id <= max_cdrs) DO + INSERT INTO + cdrs(cgrid,run_id,origin_host,source,origin_id,tor,request_type,tenant,category,account,subject,destination,setup_time,pdd,answer_time,`usage`,supplier,disconnect_cause,extra_fields,cost_source,cost,cost_details,extra_info, created_at, updated_at, deleted_at) + SELECT cdrs_primary.cgrid,rated_cdrs.runid as run_id,cdrs_primary.cdrhost as origin_host,cdrs_primary.cdrsource as source,cdrs_primary.accid as origin_id, cdrs_primary.tor,rated_cdrs.reqtype as request_type, rated_cdrs.tenant,rated_cdrs.category, rated_cdrs.account, rated_cdrs.subject, rated_cdrs.destination,rated_cdrs.setup_time,rated_cdrs.pdd,rated_cdrs.answer_time,rated_cdrs.`usage`,rated_cdrs.supplier,rated_cdrs.disconnect_cause,cdrs_extra.extra_fields,cost_details.cost_source,rated_cdrs.cost,cost_details.timespans as cost_details,rated_cdrs.extra_info,rated_cdrs.created_at,rated_cdrs.updated_at, rated_cdrs.deleted_at + FROM rated_cdrs + INNER JOIN cdrs_primary ON rated_cdrs.cgrid = cdrs_primary.cgrid + INNER JOIN cdrs_extra ON rated_cdrs.cgrid = cdrs_extra.cgrid + INNER JOIN cost_details ON rated_cdrs.cgrid = cost_details.cgrid + WHERE cdrs_primary.`usage` > '0' + AND not exists (select 1 from cdrs where cdrs.cgrid = cdrs_primary.cgrid AND cdrs.run_id=rated_cdrs.runid) + AND rated_cdrs.id >= start_id + AND rated_cdrs.id < end_id + GROUP BY cgrid, run_id, origin_id; + SET start_id = start_id + step; + SET end_id = end_id + step; + END WHILE; + /* SET Table for live usage */ + SET autocommit=1; + SET unique_checks=1; + SET foreign_key_checks=1; + ALTER TABLE cdrs ENABLE KEYS; + OPTIMIZE TABLE cdrs; +END // + +DELIMITER ; + diff --git a/data/docker/integration/scripts/mysql/setup_cgr_db.sh b/data/docker/integration/scripts/mysql/setup_cgr_db.sh new file mode 100755 index 000000000..8561b7e9d --- /dev/null +++ b/data/docker/integration/scripts/mysql/setup_cgr_db.sh @@ -0,0 +1,29 @@ +#! /usr/bin/env sh + +if test $# -lt 2; then + echo "" + echo "setup_cgr_db.sh []" + echo "" + exit 0 +fi + +host=$3 +if [ -z "$3" ]; then + host="localhost" +fi + +DIR="$(dirname "$(readlink -f "$0")")" + +mysql -u $1 -p$2 -h $host < "$DIR"/create_db_with_users.sql +cu=$? +mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_cdrs_tables.sql +cdrt=$? +mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_tariffplan_tables.sql +tpt=$? + +if [ $cu = 0 ] && [ $cdrt = 0 ] && [ $tpt = 0 ]; then + echo "\n\t+++ CGR-DB successfully set-up! +++\n" + exit 0 +fi + + diff --git a/data/docker/integration/scripts/mysql/setup_ers_db.sh b/data/docker/integration/scripts/mysql/setup_ers_db.sh new file mode 100755 index 000000000..455315e3c --- /dev/null +++ b/data/docker/integration/scripts/mysql/setup_ers_db.sh @@ -0,0 +1,25 @@ +#! /usr/bin/env sh + +if test $# -lt 2; then + echo "" + echo "setup_cgr_db.sh []" + echo "" + exit 0 +fi + +host=$3 +if [ -z "$3" ]; then + host="localhost" +fi + +DIR="$(dirname "$(readlink -f "$0")")" + +mysql -u $1 -p$2 -h $host < "$DIR"/create_ers_db.sql +cu=$? + +if [ $cu = 0 ]; then + echo "\n\t+++ CGR-DB successfully set-up! +++\n" + exit 0 +fi + + diff --git a/data/docker/integration/scripts/postgres/create_cdrs_tables.sql b/data/docker/integration/scripts/postgres/create_cdrs_tables.sql new file mode 100644 index 000000000..3ae7d5ccb --- /dev/null +++ b/data/docker/integration/scripts/postgres/create_cdrs_tables.sql @@ -0,0 +1,59 @@ +-- +-- Table structure for table `cdrs` +-- + +DROP TABLE IF EXISTS cdrs; +CREATE TABLE cdrs ( + id SERIAL PRIMARY KEY, + cgrid VARCHAR(40) NOT NULL, + run_id VARCHAR(64) NOT NULL, + origin_host VARCHAR(64) NOT NULL, + source VARCHAR(64) NOT NULL, + origin_id VARCHAR(128) NOT NULL, + tor VARCHAR(16) NOT NULL, + request_type VARCHAR(24) NOT NULL, + tenant VARCHAR(64) NOT NULL, + category VARCHAR(64) NOT NULL, + account VARCHAR(128) NOT NULL, + subject VARCHAR(128) NOT NULL, + destination VARCHAR(128) NOT NULL, + setup_time TIMESTAMP WITH TIME ZONE NOT NULL, + answer_time TIMESTAMP WITH TIME ZONE NULL, + usage BIGINT NOT NULL, + extra_fields jsonb NOT NULL, + cost_source VARCHAR(64) NOT NULL, + cost NUMERIC(20,4) DEFAULT NULL, + cost_details jsonb, + extra_info text, + created_at TIMESTAMP WITH TIME ZONE, + updated_at TIMESTAMP WITH TIME ZONE NULL, + deleted_at TIMESTAMP WITH TIME ZONE NULL, + UNIQUE (cgrid, run_id) +); +; +DROP INDEX IF EXISTS deleted_at_cp_idx; +CREATE INDEX deleted_at_cp_idx ON cdrs (deleted_at); + + +DROP TABLE IF EXISTS session_costs; +CREATE TABLE session_costs ( + id SERIAL PRIMARY KEY, + cgrid VARCHAR(40) NOT NULL, + run_id VARCHAR(64) NOT NULL, + origin_host VARCHAR(64) NOT NULL, + origin_id VARCHAR(128) NOT NULL, + cost_source VARCHAR(64) NOT NULL, + usage BIGINT NOT NULL, + cost_details jsonb, + created_at TIMESTAMP WITH TIME ZONE, + deleted_at TIMESTAMP WITH TIME ZONE NULL, + UNIQUE (cgrid, run_id) +); +DROP INDEX IF EXISTS cgrid_sessionscost_idx; +CREATE INDEX cgrid_sessionscost_idx ON session_costs (cgrid, run_id); +DROP INDEX IF EXISTS origin_sessionscost_idx; +CREATE INDEX origin_sessionscost_idx ON session_costs (origin_host, origin_id); +DROP INDEX IF EXISTS run_origin_sessionscost_idx; +CREATE INDEX run_origin_sessionscost_idx ON session_costs (run_id, origin_id); +DROP INDEX IF EXISTS deleted_at_sessionscost_idx; +CREATE INDEX deleted_at_sessionscost_idx ON session_costs (deleted_at); diff --git a/data/docker/integration/scripts/postgres/create_db_with_users.sh b/data/docker/integration/scripts/postgres/create_db_with_users.sh new file mode 100755 index 000000000..b193b3f5f --- /dev/null +++ b/data/docker/integration/scripts/postgres/create_db_with_users.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# +# Sample db and users creation. Replace here with your own details +# + +sudo -u postgres dropdb -e cgrates +sudo -u postgres dropuser -e cgrates +sudo -u postgres psql -c "CREATE USER cgrates password 'CGRateS.org';" +sudo -u postgres createdb -e -O cgrates cgrates diff --git a/data/docker/integration/scripts/postgres/create_ers_db.sh b/data/docker/integration/scripts/postgres/create_ers_db.sh new file mode 100755 index 000000000..17b760cda --- /dev/null +++ b/data/docker/integration/scripts/postgres/create_ers_db.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# extra DB for ers +sudo -u postgres dropdb -e cgrates2 +sudo -u postgres createdb -e -O cgrates cgrates2 diff --git a/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql b/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql new file mode 100644 index 000000000..0d0bca9b3 --- /dev/null +++ b/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql @@ -0,0 +1,478 @@ +-- +-- Table structure for table `tp_timings` +-- +DROP TABLE IF EXISTS tp_timings; +CREATE TABLE tp_timings ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + years VARCHAR(255) NOT NULL, + months VARCHAR(255) NOT NULL, + month_days VARCHAR(255) NOT NULL, + week_days VARCHAR(255) NOT NULL, + time VARCHAR(32) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag) +); +CREATE INDEX tptimings_tpid_idx ON tp_timings (tpid); +CREATE INDEX tptimings_idx ON tp_timings (tpid,tag); + +-- +-- Table structure for table `tp_destinations` +-- + +DROP TABLE IF EXISTS tp_destinations; +CREATE TABLE tp_destinations ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + prefix VARCHAR(24) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, prefix) +); +CREATE INDEX tpdests_tpid_idx ON tp_destinations (tpid); +CREATE INDEX tpdests_idx ON tp_destinations (tpid,tag); + +-- +-- Table structure for table `tp_rates` +-- + +DROP TABLE IF EXISTS tp_rates; +CREATE TABLE tp_rates ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + connect_fee NUMERIC(7,4) NOT NULL, + rate NUMERIC(10,4) NOT NULL, + rate_unit VARCHAR(16) NOT NULL, + rate_increment VARCHAR(16) NOT NULL, + group_interval_start VARCHAR(16) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, group_interval_start) +); +CREATE INDEX tprates_tpid_idx ON tp_rates (tpid); +CREATE INDEX tprates_idx ON tp_rates (tpid,tag); + +-- +-- Table structure for table `destination_rates` +-- + +DROP TABLE IF EXISTS tp_destination_rates; +CREATE TABLE tp_destination_rates ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + destinations_tag VARCHAR(64) NOT NULL, + rates_tag VARCHAR(64) NOT NULL, + rounding_method VARCHAR(255) NOT NULL, + rounding_decimals SMALLINT NOT NULL, + max_cost NUMERIC(7,4) NOT NULL, + max_cost_strategy VARCHAR(16) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag , destinations_tag) +); +CREATE INDEX tpdestrates_tpid_idx ON tp_destination_rates (tpid); +CREATE INDEX tpdestrates_idx ON tp_destination_rates (tpid,tag); + +-- +-- Table structure for table `tp_rating_plans` +-- + +DROP TABLE IF EXISTS tp_rating_plans; +CREATE TABLE tp_rating_plans ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + destrates_tag VARCHAR(64) NOT NULL, + timing_tag VARCHAR(64) NOT NULL, + weight NUMERIC(8,2) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, destrates_tag, timing_tag) +); +CREATE INDEX tpratingplans_tpid_idx ON tp_rating_plans (tpid); +CREATE INDEX tpratingplans_idx ON tp_rating_plans (tpid,tag); + + +-- +-- Table structure for table `tp_rate_profiles` +-- + +DROP TABLE IF EXISTS tp_rating_profiles; +CREATE TABLE tp_rating_profiles ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + loadid VARCHAR(64) NOT NULL, + tenant VARCHAR(64) NOT NULL, + category VARCHAR(32) NOT NULL, + subject VARCHAR(64) NOT NULL, + activation_time VARCHAR(26) NOT NULL, + rating_plan_tag VARCHAR(64) NOT NULL, + fallback_subjects VARCHAR(64), + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, loadid, tenant, category, subject, activation_time) +); +CREATE INDEX tpratingprofiles_tpid_idx ON tp_rating_profiles (tpid); +CREATE INDEX tpratingprofiles_idx ON tp_rating_profiles (tpid,loadid,tenant,category,subject); + +-- +-- Table structure for table `tp_shared_groups` +-- + +DROP TABLE IF EXISTS tp_shared_groups; +CREATE TABLE tp_shared_groups ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + account VARCHAR(64) NOT NULL, + strategy VARCHAR(24) NOT NULL, + rating_subject VARCHAR(24) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, account , strategy , rating_subject) +); +CREATE INDEX tpsharedgroups_tpid_idx ON tp_shared_groups (tpid); +CREATE INDEX tpsharedgroups_idx ON tp_shared_groups (tpid,tag); + +-- +-- Table structure for table `tp_actions` +-- + +DROP TABLE IF EXISTS tp_actions; +CREATE TABLE tp_actions ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + action VARCHAR(24) NOT NULL, + extra_parameters VARCHAR(256) NOT NULL, + filters VARCHAR(256) NOT NULL, + balance_tag VARCHAR(64) NOT NULL, + balance_type VARCHAR(24) NOT NULL, + categories VARCHAR(32) NOT NULL, + destination_tags VARCHAR(64) NOT NULL, + rating_subject VARCHAR(64) NOT NULL, + shared_groups VARCHAR(64) NOT NULL, + expiry_time VARCHAR(26) NOT NULL, + timing_tags VARCHAR(128) NOT NULL, + units VARCHAR(256) NOT NULL, + balance_weight VARCHAR(10) NOT NULL, + balance_blocker VARCHAR(5) NOT NULL, + balance_disabled VARCHAR(5) NOT NULL, + weight NUMERIC(8,2) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, action, balance_tag, balance_type, expiry_time, timing_tags, destination_tags, shared_groups, balance_weight, weight) +); +CREATE INDEX tpactions_tpid_idx ON tp_actions (tpid); +CREATE INDEX tpactions_idx ON tp_actions (tpid,tag); + +-- +-- Table structure for table `tp_action_timings` +-- + +DROP TABLE IF EXISTS tp_action_plans; +CREATE TABLE tp_action_plans ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + actions_tag VARCHAR(64) NOT NULL, + timing_tag VARCHAR(64) NOT NULL, + weight NUMERIC(8,2) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, actions_tag, timing_tag) +); +CREATE INDEX tpactionplans_tpid_idx ON tp_action_plans (tpid); +CREATE INDEX tpactionplans_idx ON tp_action_plans (tpid,tag); + +-- +-- Table structure for table tp_action_triggers +-- + +DROP TABLE IF EXISTS tp_action_triggers; +CREATE TABLE tp_action_triggers ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + unique_id VARCHAR(64) NOT NULL, + threshold_type VARCHAR(64) NOT NULL, + threshold_value NUMERIC(20,4) NOT NULL, + recurrent BOOLEAN NOT NULL, + min_sleep VARCHAR(16) NOT NULL, + expiry_time VARCHAR(26) NOT NULL, + activation_time VARCHAR(26) NOT NULL, + balance_tag VARCHAR(64) NOT NULL, + balance_type VARCHAR(24) NOT NULL, + balance_categories VARCHAR(32) NOT NULL, + balance_destination_tags VARCHAR(64) NOT NULL, + balance_rating_subject VARCHAR(64) NOT NULL, + balance_shared_groups VARCHAR(64) NOT NULL, + balance_expiry_time VARCHAR(26) NOT NULL, + balance_timing_tags VARCHAR(128) NOT NULL, + balance_weight VARCHAR(10) NOT NULL, + balance_blocker VARCHAR(5) NOT NULL, + balance_disabled VARCHAR(5) NOT NULL, + actions_tag VARCHAR(64) NOT NULL, + weight NUMERIC(8,2) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, balance_tag, balance_type, threshold_type, threshold_value, balance_destination_tags, actions_tag) +); +CREATE INDEX tpactiontrigers_tpid_idx ON tp_action_triggers (tpid); +CREATE INDEX tpactiontrigers_idx ON tp_action_triggers (tpid,tag); + +-- +-- Table structure for table tp_account_actions +-- + +DROP TABLE IF EXISTS tp_account_actions; +CREATE TABLE tp_account_actions ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + loadid VARCHAR(64) NOT NULL, + tenant VARCHAR(64) NOT NULL, + account VARCHAR(64) NOT NULL, + action_plan_tag VARCHAR(64), + action_triggers_tag VARCHAR(64), + allow_negative BOOLEAN NOT NULL, + disabled BOOLEAN NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, loadid, tenant, account) +); +CREATE INDEX tpaccountactions_tpid_idx ON tp_account_actions (tpid); +CREATE INDEX tpaccountactions_idx ON tp_account_actions (tpid,loadid,tenant,account); + + +-- +-- Table structure for table `tp_resources` +-- + +DROP TABLE IF EXISTS tp_resources; +CREATE TABLE tp_resources ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "usage_ttl" varchar(32) NOT NULL, + "limit" varchar(64) NOT NULL, + "allocation_message" varchar(64) NOT NULL, + "blocker" BOOLEAN NOT NULL, + "stored" BOOLEAN NOT NULL, + "weight" NUMERIC(8,2) NOT NULL, + "threshold_ids" varchar(64) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); +CREATE INDEX tp_resources_idx ON tp_resources (tpid); +CREATE INDEX tp_resources_unique ON tp_resources ("tpid", "tenant", "id", "filter_ids"); + + +-- +-- Table structure for table `tp_stats` +-- + +DROP TABLE IF EXISTS tp_stats; +CREATE TABLE tp_stats ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "queue_length" INTEGER NOT NULL, + "ttl" varchar(32) NOT NULL, + "min_items" INTEGER NOT NULL, + "metric_ids" VARCHAR(128) NOT NULL, + "metric_filter_ids" VARCHAR(128) NOT NULL, + "stored" BOOLEAN NOT NULL, + "blocker" BOOLEAN NOT NULL, + "weight" decimal(8,2) NOT NULL, + "threshold_ids" varchar(64) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); +CREATE INDEX tp_stats_idx ON tp_stats (tpid); +CREATE INDEX tp_stats_unique ON tp_stats ("tpid","tenant", "id", "filter_ids","metric_ids"); + +-- +-- Table structure for table `tp_threshold_cfgs` +-- + +DROP TABLE IF EXISTS tp_thresholds; +CREATE TABLE tp_thresholds ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "max_hits" INTEGER NOT NULL, + "min_hits" INTEGER NOT NULL, + "min_sleep" varchar(16) NOT NULL, + "blocker" BOOLEAN NOT NULL, + "weight" decimal(8,2) NOT NULL, + "action_ids" varchar(64) NOT NULL, + "async" BOOLEAN NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); +CREATE INDEX tp_thresholds_idx ON tp_thresholds (tpid); +CREATE INDEX tp_thresholds_unique ON tp_thresholds ("tpid","tenant", "id","filter_ids","action_ids"); + +-- +-- Table structure for table `tp_filter` +-- + +DROP TABLE IF EXISTS tp_filters; +CREATE TABLE tp_filters ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant" varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "type" varchar(16) NOT NULL, + "element" varchar(64) NOT NULL, + "values" varchar(256) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); + CREATE INDEX tp_filters_idx ON tp_filters (tpid); + CREATE INDEX tp_filters_unique ON tp_filters ("tpid","tenant", "id", "type", "element"); + +-- +-- Table structure for table `tp_routes` +-- + +DROP TABLE IF EXISTS tp_routes; +CREATE TABLE tp_routes ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "sorting" varchar(32) NOT NULL, + "sorting_parameters" varchar(64) NOT NULL, + "route_id" varchar(32) NOT NULL, + "route_filter_ids" varchar(64) NOT NULL, + "route_account_ids" varchar(64) NOT NULL, + "route_ratingplan_ids" varchar(64) NOT NULL, + "route_resource_ids" varchar(64) NOT NULL, + "route_stat_ids" varchar(64) NOT NULL, + "route_weight" decimal(8,2) NOT NULL, + "route_blocker" BOOLEAN NOT NULL, + "route_parameters" varchar(64) NOT NULL, + "weight" decimal(8,2) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); +CREATE INDEX tp_routes_idx ON tp_routes (tpid); +CREATE INDEX tp_routes_unique ON tp_routes ("tpid", "tenant", "id", + "filter_ids","route_id","route_filter_ids","route_account_ids", + "route_ratingplan_ids","route_resource_ids","route_stat_ids"); + + -- + -- Table structure for table `tp_attributes` + -- + + DROP TABLE IF EXISTS tp_attributes; + CREATE TABLE tp_attributes ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "contexts" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "attribute_filter_ids" varchar(64) NOT NULL, + "path" varchar(64) NOT NULL, + "type" varchar(64) NOT NULL, + "value" varchar(64) NOT NULL, + "blocker" BOOLEAN NOT NULL, + "weight" decimal(8,2) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE + ); + CREATE INDEX tp_attributes_ids ON tp_attributes (tpid); + CREATE INDEX tp_attributes_unique ON tp_attributes ("tpid", "tenant", "id", + "filter_ids","path","value"); + + -- + -- Table structure for table `tp_chargers` + -- + + DROP TABLE IF EXISTS tp_chargers; + CREATE TABLE tp_chargers ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "run_id" varchar(64) NOT NULL, + "attribute_ids" varchar(64) NOT NULL, + "weight" decimal(8,2) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE + ); + CREATE INDEX tp_chargers_ids ON tp_chargers (tpid); + CREATE INDEX tp_chargers_unique ON tp_chargers ("tpid", "tenant", "id", + "filter_ids","run_id","attribute_ids"); + + -- + -- Table structure for table `tp_dispatchers` + -- + + DROP TABLE IF EXISTS tp_dispatcher_profiles; + CREATE TABLE tp_dispatcher_profiles ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant" varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "subsystems" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "strategy" varchar(64) NOT NULL, + "strategy_parameters" varchar(64) NOT NULL, + "conn_id" varchar(64) NOT NULL, + "conn_filter_ids" varchar(64) NOT NULL, + "conn_weight" decimal(8,2) NOT NULL, + "conn_blocker" BOOLEAN NOT NULL, + "conn_parameters" varchar(64) NOT NULL, + "weight" decimal(8,2) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE + ); + CREATE INDEX tp_dispatcher_profiles_ids ON tp_dispatcher_profiles (tpid); + CREATE INDEX tp_dispatcher_profiles_unique ON tp_dispatcher_profiles ("tpid", "tenant", "id", + "filter_ids","strategy","conn_id","conn_filter_ids"); + +-- +-- Table structure for table `tp_dispatchers` +-- + + DROP TABLE IF EXISTS tp_dispatcher_hosts; + CREATE TABLE tp_dispatcher_hosts ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant" varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "address" varchar(64) NOT NULL, + "transport" varchar(64) NOT NULL, + "connect_attempts" INTEGER NOT NULL, + "reconnects" INTEGER NOT NULL, + "max_reconnect_interval" varchar(64) NOT NULL, + "connect_timeout" varchar(64) NOT NULL, + "reply_timeout" varchar(64) NOT NULL, + "tls" BOOLEAN NOT NULL, + "client_key" varchar(64) NOT NULL, + "client_certificate" varchar(64) NOT NULL, + "ca_certificate" varchar(64) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE + ); + CREATE INDEX tp_dispatchers_hosts_ids ON tp_dispatcher_hosts (tpid); + CREATE INDEX tp_dispatcher_hosts_unique ON tp_dispatcher_hosts ("tpid", "tenant", "id", + "address"); + + + +-- +-- Table structure for table `versions` +-- + +DROP TABLE IF EXISTS versions; +CREATE TABLE versions ( + "id" SERIAL PRIMARY KEY, + "item" varchar(64) NOT NULL, + "version" INTEGER NOT NULL, + UNIQUE ("id","item") +); diff --git a/data/docker/integration/scripts/postgres/pg_cdr_migration.sql b/data/docker/integration/scripts/postgres/pg_cdr_migration.sql new file mode 100644 index 000000000..5f40cb6d0 --- /dev/null +++ b/data/docker/integration/scripts/postgres/pg_cdr_migration.sql @@ -0,0 +1,42 @@ +/* +This script will migrate CDRs from the old CGRateS tables to the new cdrs table +but it only migrate CDRs where the duration is > 0. +If you need CDRs also with duration is = 0 you can make the appropriate change in the line beginning WHERE cdrs_primary.usage + +Also the script will process 10,000 CDRs before committing to save system resources +especially in systems where they are millions of CDRs to be migrated +You can increase or lower the value of step in the line after BEGIN below. +*/ + +DO $$ +DECLARE + max_cdrs bigint; + start_id bigint; + end_id bigint; + step bigint; +BEGIN + /* You must change the step var to commit every step rows inserted */ + step := 10000; + start_id := 0; + end_id := start_id + step; + select max(id) INTO max_cdrs from rated_cdrs; + WHILE start_id <= max_cdrs + LOOP + --RAISE NOTICE '%', (to_char(start_id, '99999999') || '-' || to_char(end_id, '99999999')); + INSERT INTO + cdrs(cgrid,run_id,origin_host,source,origin_id,tor,request_type,tenant,category,account,subject,destination,setup_time,pdd,answer_time,usage,supplier,disconnect_cause,extra_fields,cost_source,cost,cost_details,extra_info, created_at, updated_at, deleted_at) + SELECT cdrs_primary.cgrid,rated_cdrs.runid as run_id,cdrs_primary.cdrhost as origin_host,cdrs_primary.cdrsource as source,cdrs_primary.accid as origin_id, cdrs_primary.tor,rated_cdrs.reqtype as request_type, rated_cdrs.tenant,rated_cdrs.category, rated_cdrs.account, rated_cdrs.subject, rated_cdrs.destination,rated_cdrs.setup_time,rated_cdrs.pdd,rated_cdrs.answer_time,rated_cdrs.usage,rated_cdrs.supplier,rated_cdrs.disconnect_cause,cdrs_extra.extra_fields,cost_details.cost_source,rated_cdrs.cost,cost_details.timespans as cost_details,rated_cdrs.extra_info,rated_cdrs.created_at,rated_cdrs.updated_at, rated_cdrs.deleted_at + FROM rated_cdrs + INNER JOIN cdrs_primary ON rated_cdrs.cgrid = cdrs_primary.cgrid + INNER JOIN cdrs_extra ON rated_cdrs.cgrid = cdrs_extra.cgrid + INNER JOIN cost_details ON rated_cdrs.cgrid = cost_details.cgrid + WHERE cdrs_primary.usage > '0' + AND not exists (select 1 from cdrs c where c.cgrid = cdrs_primary.cgrid) + AND rated_cdrs.id >= start_id + AND rated_cdrs.id < end_id + ; + start_id = start_id + step; + end_id = end_id + step; + END LOOP; +END +$$; diff --git a/data/docker/integration/scripts/postgres/setup_cgr_db.sh b/data/docker/integration/scripts/postgres/setup_cgr_db.sh new file mode 100755 index 000000000..66d011763 --- /dev/null +++ b/data/docker/integration/scripts/postgres/setup_cgr_db.sh @@ -0,0 +1,30 @@ +#! /usr/bin/env sh + + +user=$1 +if [ -z "$1" ]; then + user="cgrates" +fi + +host=$2 +if [ -z "$2" ]; then + host="localhost" +fi + +DIR="$(dirname "$(readlink -f "$0")")" + +"$DIR"/create_db_with_users.sh + +export PGPASSWORD="CGRateS.org" + +psql -U $user -h $host -d cgrates -f "$DIR"/create_cdrs_tables.sql +cdrt=$? +psql -U $user -h $host -d cgrates -f "$DIR"/create_tariffplan_tables.sql +tpt=$? + +if [ $cdrt = 0 ] && [ $tpt = 0 ]; then + echo "\n\t+++ CGR-DB successfully set-up! +++\n" + exit 0 +fi + + diff --git a/recipients b/recipients index b49bf6477..a8c609354 100644 --- a/recipients +++ b/recipients @@ -1,5 +1 @@ -andrei.andronache@itsyscom.com -alexandru.tripon@itsyscom.com -ionut.boangiu@itsyscom.com -adrian.porosnicu@itsyscom.com -nickolas.filip@itsyscom.com +arber.katellari@itsyscom.com \ No newline at end of file