From 05ede32f0f647d73cda0626137875ce1815316de Mon Sep 17 00:00:00 2001 From: arberkatellari Date: Fri, 1 Sep 2023 11:22:23 -0400 Subject: [PATCH] Add Drone CI --- .drone.yml | 226 ++++ data/docker/integration/Dockerfile | 81 ++ data/docker/integration/conf/redis.conf | 1052 +++++++++++++++++ .../integration/conf/rsyslog.d/19-ftp.conf | 4 + .../integration/conf/rsyslog.d/20-user.conf | 4 + data/docker/integration/conf/rsyslogd.conf | 99 ++ .../docker/integration/conf/server.properties | 137 +++ data/docker/integration/docker-entrypoint.sh | 73 ++ data/docker/integration/main.yaml | 46 + data/docker/integration/scripts/mariadb-ep.sh | 356 ++++++ .../integration/scripts/mongo/create_user.js | 9 + .../integration/scripts/mongo/setup_cgr_db.sh | 14 + .../mysql/alter_cdr_tables_rc5_rc6.sql | 50 + .../scripts/mysql/create_cdrs_tables.sql | 52 + .../scripts/mysql/create_db_with_users.sql | 10 + .../scripts/mysql/create_ers_db.sql | 7 + .../mysql/create_tariffplan_tables.sql | 491 ++++++++ .../scripts/mysql/mysql_cdr_migration.sql | 57 + .../integration/scripts/mysql/setup_cgr_db.sh | 29 + .../integration/scripts/mysql/setup_ers_db.sh | 25 + .../scripts/postgres/create_cdrs_tables.sql | 59 + .../scripts/postgres/create_db_with_users.sh | 10 + .../scripts/postgres/create_ers_db.sh | 5 + .../postgres/create_tariffplan_tables.sql | 478 ++++++++ .../scripts/postgres/pg_cdr_migration.sql | 42 + .../scripts/postgres/setup_cgr_db.sh | 30 + data/docker/integration/scripts/service | 29 + recipients | 1 + 28 files changed, 3476 insertions(+) create mode 100644 .drone.yml create mode 100644 data/docker/integration/Dockerfile create mode 100644 data/docker/integration/conf/redis.conf create mode 100644 data/docker/integration/conf/rsyslog.d/19-ftp.conf create mode 100644 data/docker/integration/conf/rsyslog.d/20-user.conf create mode 100644 data/docker/integration/conf/rsyslogd.conf create mode 100644 data/docker/integration/conf/server.properties create mode 100755 data/docker/integration/docker-entrypoint.sh create mode 100644 data/docker/integration/main.yaml create mode 100755 data/docker/integration/scripts/mariadb-ep.sh create mode 100644 data/docker/integration/scripts/mongo/create_user.js create mode 100755 data/docker/integration/scripts/mongo/setup_cgr_db.sh create mode 100644 data/docker/integration/scripts/mysql/alter_cdr_tables_rc5_rc6.sql create mode 100644 data/docker/integration/scripts/mysql/create_cdrs_tables.sql create mode 100644 data/docker/integration/scripts/mysql/create_db_with_users.sql create mode 100644 data/docker/integration/scripts/mysql/create_ers_db.sql create mode 100644 data/docker/integration/scripts/mysql/create_tariffplan_tables.sql create mode 100644 data/docker/integration/scripts/mysql/mysql_cdr_migration.sql create mode 100755 data/docker/integration/scripts/mysql/setup_cgr_db.sh create mode 100755 data/docker/integration/scripts/mysql/setup_ers_db.sh create mode 100644 data/docker/integration/scripts/postgres/create_cdrs_tables.sql create mode 100755 data/docker/integration/scripts/postgres/create_db_with_users.sh create mode 100755 data/docker/integration/scripts/postgres/create_ers_db.sh create mode 100644 data/docker/integration/scripts/postgres/create_tariffplan_tables.sql create mode 100644 data/docker/integration/scripts/postgres/pg_cdr_migration.sql create mode 100755 data/docker/integration/scripts/postgres/setup_cgr_db.sh create mode 100755 data/docker/integration/scripts/service create mode 100644 recipients diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 000000000..8cb331bfb --- /dev/null +++ b/.drone.yml @@ -0,0 +1,226 @@ +kind: pipeline +type: docker +name: unit + +workspace: + path: /go/src/github.com/cgrates/cgrates/ + +steps: +- name: unit + pull: never + image: cgrates-integration + commands: + - go version + - ./test.sh +- name: notify + pull: never + image: drillster/drone-email + settings: + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail + host: smtp.gmail.com + username: + from_secret: email_username + password: + from_secret: email_password + subject: > + [{{ build.status }}] + {{ repo.owner }}/{{ repo.name }} + ({{ commit.branch }}) + body: > + Build number: # {{ build.number }} + Build link: {{ build.link }} + Build started: {{ build.started }} + Build ended: {{ build.finished }} + Commit author: {{ commit.author.name }} + Commit message: {{ commit.message }} + Commit branch: {{ commit.branch }} + Commit link: {{ commit.link }} + recipients_file: recipients + recipients_only: true + when: + status: failure + +--- +kind: pipeline +type: docker +name: integration-internal + + +workspace: + path: /go/src/github.com/cgrates/cgrates/ + +steps: +- name: integration-internal + pull: never + image: cgrates-integration + commands: + - data/docker/integration/docker-entrypoint.sh + - ./integration_test.sh -dbtype=*internal + failure: ignore +- name: notify + pull: never + image: drillster/drone-email + settings: + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail + host: smtp.gmail.com + username: + from_secret: email_username + password: + from_secret: email_password + subject: > + [{{ build.status }}] + {{ repo.owner }}/{{ repo.name }} + ({{ commit.branch }}) + body: > + Build number: # {{ build.number }} + Build link: {{ build.link }} + Build started: {{ build.started }} + Build ended: {{ build.finished }} + Commit author: {{ commit.author.name }} + Commit message: {{ commit.message }} + Commit branch: {{ commit.branch }} + Commit link: {{ commit.link }} + recipients_file: recipients + recipients_only: true + when: + status: failure + +--- +kind: pipeline +type: docker +name: integration-mysql + + +workspace: + path: /go/src/github.com/cgrates/cgrates/ + +steps: +- name: integration-mysql + pull: never + image: cgrates-integration + commands: + - data/docker/integration/docker-entrypoint.sh + - ./integration_test.sh -dbtype=*mysql + failure: ignore +- name: notify + pull: never + image: drillster/drone-email + settings: + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail + host: smtp.gmail.com + username: + from_secret: email_username + password: + from_secret: email_password + subject: > + [{{ build.status }}] + {{ repo.owner }}/{{ repo.name }} + ({{ commit.branch }}) + body: > + Build number: # {{ build.number }} + Build link: {{ build.link }} + Build started: {{ build.started }} + Build ended: {{ build.finished }} + Commit author: {{ commit.author.name }} + Commit message: {{ commit.message }} + Commit branch: {{ commit.branch }} + Commit link: {{ commit.link }} + recipients_file: recipients + recipients_only: true + when: + status: failure + +--- +kind: pipeline +type: docker +name: integration-mongo + + +workspace: + path: /go/src/github.com/cgrates/cgrates/ + +steps: +- name: integration-mongo + pull: never + image: cgrates-integration + commands: + - data/docker/integration/docker-entrypoint.sh + - ./integration_test.sh -dbtype=*mongo + failure: ignore +- name: notify + pull: never + image: drillster/drone-email + settings: + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail + host: smtp.gmail.com + username: + from_secret: email_username + password: + from_secret: email_password + subject: > + [{{ build.status }}] + {{ repo.owner }}/{{ repo.name }} + ({{ commit.branch }}) + body: > + Build number: # {{ build.number }} + Build link: {{ build.link }} + Build started: {{ build.started }} + Build ended: {{ build.finished }} + Commit author: {{ commit.author.name }} + Commit message: {{ commit.message }} + Commit branch: {{ commit.branch }} + Commit link: {{ commit.link }} + recipients_file: recipients + recipients_only: true + when: + status: failure + +--- +kind: pipeline +type: docker +name: integration-postgres + + +workspace: + path: /go/src/github.com/cgrates/cgrates/ + +steps: +- name: integration-postgres + pull: never + image: cgrates-integration + commands: + - data/docker/integration/docker-entrypoint.sh + - ./integration_test.sh -dbtype=*postgres + failure: ignore +- name: notify + pull: never + image: drillster/drone-email + settings: + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail + host: smtp.gmail.com + username: + from_secret: email_username + password: + from_secret: email_password + subject: > + [{{ build.status }}] + {{ repo.owner }}/{{ repo.name }} + ({{ commit.branch }}) + body: > + Build number: # {{ build.number }} + Build link: {{ build.link }} + Build started: {{ build.started }} + Build ended: {{ build.finished }} + Commit author: {{ commit.author.name }} + Commit message: {{ commit.message }} + Commit branch: {{ commit.branch }} + Commit link: {{ commit.link }} + recipients_file: recipients + recipients_only: true + when: + status: failure \ No newline at end of file diff --git a/data/docker/integration/Dockerfile b/data/docker/integration/Dockerfile new file mode 100644 index 000000000..f915c73c1 --- /dev/null +++ b/data/docker/integration/Dockerfile @@ -0,0 +1,81 @@ +FROM ubuntu:jammy + +RUN groupadd -r mongodb && useradd -r -g mongodb mongodb +RUN groupadd -r redis && useradd -r -g redis redis +RUN groupadd -r kafka && useradd -r -g kafka kafka + +RUN mkdir -p /usr/share/cgrates/conf /usr/share/man/man1 /var/spool/cgrates/ers/in /var/spool/cgrates/ers/out /var/spool/cgrates/cdre/csv /var/spool/cgrates/cdre/fwv /var/spool/cgrates/tpe /var/spool/cgrates/failed_posts /var/spool/cgrates/analyzers /run /data/configdb /data/db /kafka /logs + + +ENV DEBIAN_FRONTEND=noninteractive + + +RUN apt-get update && apt-get install -y ansible + +# Install Erlang packages +RUN apt-get install -y erlang-base erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key erlang-runtime-tools erlang-snmp erlang-ssl erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl + +RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk-headless neovim net-tools + +RUN curl -fsSL https://pgp.mongodb.com/server-7.0.asc | gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor +RUN echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list + +RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +RUN apt-get update +RUN apt-get install -y mongodb-org postgresql +# ADD THIS AFTER postgresql rabbitmq-server + +WORKDIR /kafka + +RUN wget "https://archive.apache.org/dist/kafka/2.1.0/kafka_2.11-2.1.0.tgz" +RUN tar -xvzf ./kafka_2.11-2.1.0.tgz --strip 1 +RUN rm kafka_2.11-2.1.0.tgz +COPY ./conf/server.properties /kafka/config/server.properties +WORKDIR / + +RUN set -ex; \ + rm -rf /var/lib/apt/lists/*; \ +# purge and re-create /var/lib/mysql with appropriate ownership + rm -rf /var/lib/mysql; \ + mkdir -p /var/lib/mysql /var/run/mysqld; \ + chown -R mysql:mysql /var/lib/mysql /var/run/mysqld; \ +# ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime + chmod 777 /var/run/mysqld; \ +# comment out a few problematic configuration values + find /etc/mysql/ -name '*.cnf' -print0 \ + | xargs -0 grep -lZE '^(bind-address|log|user\s)' \ + | xargs -rt -0 sed -Ei 's/^(bind-address|log|user\s)/#&/'; \ +# don't reverse lookup hostnames, they are usually another container + echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf + +# Clean up +RUN apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN touch /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log +RUN chmod 777 /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log + +COPY main.yaml /integration_tests.yaml + +COPY roles /roles + +RUN wget -O go.tgz "https://storage.googleapis.com/golang/go1.21.0.linux-amd64.tar.gz" --progress=dot:giga +RUN tar -C /usr/local -xzf go.tgz +RUN rm go.tgz + +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" +RUN go version + +COPY ./scripts /scripts +COPY ./scripts/service /usr/local/bin/service + +COPY ./conf/rsyslogd.conf /etc/rsyslogd.conf +COPY ./conf/rsyslog.d /etc/rsyslog.d +COPY ./conf/redis.conf /etc/redis/redis.conf + +COPY ./docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] \ No newline at end of file diff --git a/data/docker/integration/conf/redis.conf b/data/docker/integration/conf/redis.conf new file mode 100644 index 000000000..a97dc2cb2 --- /dev/null +++ b/data/docker/integration/conf/redis.conf @@ -0,0 +1,1052 @@ +# Redis configuration file example. +# +# Note that in order to read the configuration file, Redis must be +# started with the file path as first argument: +# +# ./redis-server /path/to/redis.conf + +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis servers but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################## NETWORK ##################################### + +# By default, if no "bind" configuration directive is specified, Redis listens +# for connections from all the network interfaces available on the server. +# It is possible to listen to just one or multiple selected interfaces using +# the "bind" configuration directive, followed by one or more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 +# bind 127.0.0.1 ::1 +# +# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the +# internet, binding to all the interfaces is dangerous and will expose the +# instance to everybody on the internet. So by default we uncomment the +# following bind directive, that will force Redis to listen only into +# the IPv4 lookback interface address (this means Redis will be able to +# accept connections only from clients running into the same computer it +# is running). +# +# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES +# JUST COMMENT THE FOLLOWING LINE. +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +bind 127.0.0.1 + +# Protected mode is a layer of security protection, in order to avoid that +# Redis instances left open on the internet are accessed and exploited. +# +# When protected mode is on and if: +# +# 1) The server is not binding explicitly to a set of addresses using the +# "bind" directive. +# 2) No password is configured. +# +# The server only accepts connections from clients connecting from the +# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain +# sockets. +# +# By default protected mode is enabled. You should disable it only if +# you are sure you want clients from other hosts to connect to Redis +# even if no authentication is configured, nor a specific set of interfaces +# are explicitly listed using the "bind" directive. +protected-mode yes + +# Accept connections on the specified port, default is 6379 (IANA #815344). +# If port 0 is specified Redis will not listen on a TCP socket. +port 6379 + +# TCP listen() backlog. +# +# In high requests-per-second environments you need an high backlog in order +# to avoid slow clients connections issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to get the desired effect. +tcp-backlog 511 + +# Unix socket. +# +# Specify the path for the Unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +# unixsocket /var/run/redis/redis.sock +# unixsocketperm 700 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Take the connection alive from the point of view of network +# equipment in the middle. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 300 seconds, which is the new +# Redis default starting with Redis 3.2.1. +tcp-keepalive 300 + +################################# GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize yes + +# If you run Redis from upstart or systemd, Redis can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous liveness pings back to your supervisor. +supervised no + +# If a pid file is specified, Redis writes it where specified at startup +# and removes it at exit. +# +# When the server runs non daemonized, no pid file is created if none is +# specified in the configuration. When the server is daemonized, the pid file +# is used even if not specified, defaulting to "/var/run/redis.pid". +# +# Creating a pid file is best effort: if Redis is not able to create it +# nothing bad happens, the server will start and run normally. +pidfile /var/run/redis/redis-server.pid + +# Specify the server verbosity level. +# This can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /var/log/redis/redis-server.log + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +################################ SNAPSHOTTING ################################ +# +# Save the DB on disk: +# +# save +# +# Will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# In the example below the behaviour will be to save: +# after 900 sec (15 min) if at least 1 key changed +# after 300 sec (5 min) if at least 10 keys changed +# after 60 sec if at least 10000 keys changed +# +# Note: you can disable saving completely by commenting out all "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# +# save "" + +save 900 1 +save 300 10 +save 60 10000 + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in a hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# disaster will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usual even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + +# Compress string objects using LZF when dump .rdb databases? +# For default that's set to 'yes' as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# The filename where to dump the DB +dbfilename dump.rdb + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# The Append Only File will also be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir /var/lib/redis + +################################# REPLICATION ################################# + +# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# another Redis server. A few things to understand ASAP about Redis replication. +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of slaves. +# 2) Redis slaves are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition slaves automatically try to reconnect to masters +# and resynchronize with them. +# +# slaveof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth + +# When a slave loses its connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Replication SYNC strategy: disk or socket. +# +# ------------------------------------------------------- +# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY +# ------------------------------------------------------- +# +# New slaves and reconnecting slaves that are not able to continue the replication +# process just receiving differences, need to do what is called a "full +# synchronization". An RDB file is transmitted from the master to the slaves. +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the slaves incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to slave sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more slaves +# can be queued and served with the RDB file as soon as the current child producing +# the RDB file finishes its work. With diskless replication instead once +# the transfer starts, new slaves arriving will be queued and a new transfer +# will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple slaves +# will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that transfers the RDB via socket +# to the slaves. +# +# This is important since once the transfer starts, it is not possible to serve +# new slaves arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more slaves arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of slave. +# 2) Master timeout from the point of view of slaves (data, pings). +# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +# repl-timeout 60 + +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# slave data when slaves are disconnected for some time, so that when a slave +# wants to reconnect again, often a full resync is not needed, but a partial +# resync is enough, just passing the portion of data the slave missed while +# disconnected. +# +# The bigger the replication backlog, the longer the time the slave can be +# disconnected and later be able to perform a partial resynchronization. +# +# The backlog is only allocated once there is at least a slave connected. +# +# repl-backlog-size 1mb + +# After a master has no longer connected slaves for some time, the backlog +# will be freed. The following option configures the amount of seconds that +# need to elapse, starting from the time the last slave disconnected, for +# the backlog buffer to be freed. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The slave priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the slave as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# It is possible for a master to stop accepting writes if there are less than +# N slaves connected, having a lag less or equal than M seconds. +# +# The N slaves need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the slave, that is usually sent every second. +# +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough slaves +# are available, to the specified number of seconds. +# +# For example to require at least 3 slaves with a lag <= 10 seconds use: +# +# min-slaves-to-write 3 +# min-slaves-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-slaves-to-write is set to 0 (feature disabled) and +# min-slaves-max-lag is set to 10. + +# A Redis master is able to list the address and port of the attached +# slaves in different ways. For example the "INFO replication" section +# offers this information, which is used, among other tools, by +# Redis Sentinel in order to discover slave instances. +# Another place where this info is available is in the output of the +# "ROLE" command of a masteer. +# +# The listed IP and address normally reported by a slave is obtained +# in the following way: +# +# IP: The address is auto detected by checking the peer address +# of the socket used by the slave to connect with the master. +# +# Port: The port is communicated by the slave during the replication +# handshake, and is normally the port that the slave is using to +# list for connections. +# +# However when port forwarding or Network Address Translation (NAT) is +# used, the slave may be actually reachable via different IP and port +# pairs. The following two options can be used by a slave in order to +# report to its master a specific set of IP and port, so that both INFO +# and ROLE will report those values. +# +# There is no need to use both the options if you need to override just +# the port or the IP address. +# +# slave-announce-ip 5.5.5.5 +# slave-announce-port 1234 + +################################## SECURITY ################################### + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running redis-server. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to slaves may cause problems. + +################################### LIMITS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# maxclients 10000 + +# Don't use more memory than the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# according to the eviction policy selected (see maxmemory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# a hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). +# +# maxmemory + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select among five behaviors: +# +# volatile-lru -> remove the key with an expire set using an LRU algorithm +# allkeys-lru -> remove any key according to the LRU algorithm +# volatile-random -> remove a random key with an expire set +# allkeys-random -> remove a random key, any key +# volatile-ttl -> remove the key with the nearest expire time (minor TTL) +# noeviction -> don't expire at all, just return an error on write operations +# +# Note: with any of the above policies, Redis will return an error on write +# operations, when there are no suitable keys for eviction. +# +# At the date of writing these commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +# maxmemory-policy noeviction + +# LRU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can tune it for speed or +# accuracy. For default Redis will check five keys and pick the one that was +# used less recently, you can change the sample size using the following +# configuration directive. +# +# The default of 5 produces good enough results. 10 Approximates very closely +# true LRU but costs a bit more CPU. 3 is very fast but not very accurate. +# +# maxmemory-samples 5 + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. + +appendonly no + +# The name of the append only file (default: "appendonly.aof") + +appendfilename "appendonly.aof" + +# The fsync() call tells the Operating System to actually write data on disk +# instead of waiting for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log. Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec", as that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. + +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (if no rewrite has happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceeds the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################ REDIS CLUSTER ############################### +# +# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# WARNING EXPERIMENTAL: Redis Cluster is considered to be stable code, however +# in order to mark it as "mature" we need to wait for a non trivial percentage +# of users to deploy it in production. +# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +# +# Normal Redis instances can't be part of a Redis Cluster; only nodes that are +# started as cluster nodes can. In order to start a Redis instance as a +# cluster node enable the cluster support uncommenting the following: +# +# cluster-enabled yes + +# Every cluster node has a cluster configuration file. This file is not +# intended to be edited by hand. It is created and updated by Redis nodes. +# Every Redis Cluster node requires a different cluster configuration file. +# Make sure that instances running in the same system do not have +# overlapping cluster configuration file names. +# +# cluster-config-file nodes-6379.conf + +# Cluster node timeout is the amount of milliseconds a node must be unreachable +# for it to be considered in failure state. +# Most other internal time limits are multiple of the node timeout. +# +# cluster-node-timeout 15000 + +# A slave of a failing master will avoid to start a failover if its data +# looks too old. +# +# There is no simple way for a slave to actually have a exact measure of +# its "data age", so the following two checks are performed: +# +# 1) If there are multiple slaves able to failover, they exchange messages +# in order to try to give an advantage to the slave with the best +# replication offset (more data from the master processed). +# Slaves will try to get their rank by offset, and apply to the start +# of the failover a delay proportional to their rank. +# +# 2) Every single slave computes the time of the last interaction with +# its master. This can be the last ping or command received (if the master +# is still in the "connected" state), or the time that elapsed since the +# disconnection with the master (if the replication link is currently down). +# If the last interaction is too old, the slave will not try to failover +# at all. +# +# The point "2" can be tuned by user. Specifically a slave will not perform +# the failover if, since the last interaction with the master, the time +# elapsed is greater than: +# +# (node-timeout * slave-validity-factor) + repl-ping-slave-period +# +# So for example if node-timeout is 30 seconds, and the slave-validity-factor +# is 10, and assuming a default repl-ping-slave-period of 10 seconds, the +# slave will not try to failover if it was not able to talk with the master +# for longer than 310 seconds. +# +# A large slave-validity-factor may allow slaves with too old data to failover +# a master, while a too small value may prevent the cluster from being able to +# elect a slave at all. +# +# For maximum availability, it is possible to set the slave-validity-factor +# to a value of 0, which means, that slaves will always try to failover the +# master regardless of the last time they interacted with the master. +# (However they'll always try to apply a delay proportional to their +# offset rank). +# +# Zero is the only value able to guarantee that when all the partitions heal +# the cluster will always be able to continue. +# +# cluster-slave-validity-factor 10 + +# Cluster slaves are able to migrate to orphaned masters, that are masters +# that are left without working slaves. This improves the cluster ability +# to resist to failures as otherwise an orphaned master can't be failed over +# in case of failure if it has no working slaves. +# +# Slaves migrate to orphaned masters only if there are still at least a +# given number of other working slaves for their old master. This number +# is the "migration barrier". A migration barrier of 1 means that a slave +# will migrate only if there is at least 1 other working slave for its master +# and so forth. It usually reflects the number of slaves you want for every +# master in your cluster. +# +# Default is 1 (slaves migrate only if their masters remain with at least +# one slave). To disable migration just set it to a very large value. +# A value of 0 can be set but is useful only for debugging and dangerous +# in production. +# +# cluster-migration-barrier 1 + +# By default Redis Cluster nodes stop accepting queries if they detect there +# is at least an hash slot uncovered (no available node is serving it). +# This way if the cluster is partially down (for example a range of hash slots +# are no longer covered) all the cluster becomes, eventually, unavailable. +# It automatically returns available as soon as all the slots are covered again. +# +# However sometimes you want the subset of the cluster which is working, +# to continue to accept queries for the part of the key space that is still +# covered. In order to do so, just set the cluster-require-full-coverage +# option to no. +# +# cluster-require-full-coverage yes + +# In order to setup your cluster make sure to read the documentation +# available at http://redis.io web site. + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enabled at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# EVENT NOTIFICATION ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at http://redis.io/topics/notifications +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# +# The "notify-keyspace-events" takes as argument a string that is composed +# of zero or multiple characters. The empty string means that notifications +# are disabled. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 + +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happen to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into a hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# actively rehash the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply from time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients including MONITOR clients +# slave -> slave clients +# pubsub -> clients subscribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled by setting them to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeout, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are performed with the same frequency, but Redis checks for +# tasks to perform according to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes diff --git a/data/docker/integration/conf/rsyslog.d/19-ftp.conf b/data/docker/integration/conf/rsyslog.d/19-ftp.conf new file mode 100644 index 000000000..e4e1534c7 --- /dev/null +++ b/data/docker/integration/conf/rsyslog.d/19-ftp.conf @@ -0,0 +1,4 @@ +ftp.* { + /proc/self/fd/2 + stop +} diff --git a/data/docker/integration/conf/rsyslog.d/20-user.conf b/data/docker/integration/conf/rsyslog.d/20-user.conf new file mode 100644 index 000000000..549cc1451 --- /dev/null +++ b/data/docker/integration/conf/rsyslog.d/20-user.conf @@ -0,0 +1,4 @@ +local1.* { + /proc/self/fd/2 + stop +} diff --git a/data/docker/integration/conf/rsyslogd.conf b/data/docker/integration/conf/rsyslogd.conf new file mode 100644 index 000000000..9dd19113b --- /dev/null +++ b/data/docker/integration/conf/rsyslogd.conf @@ -0,0 +1,99 @@ + +# /etc/rsyslog.conf Configuration file for rsyslog. +# +# For more information see +# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html + + +################# +#### MODULES #### +################# + +module(load="imuxsock") +input(type="imuxsock" Socket="/var/run/rsyslog/dev/log" CreatePath="on") + +# provides TCP syslog reception +module(load="imtcp") +input(type="imtcp" port="514") + +# provides UDP syslog reception +module(load="imudp") +input(type="imudp" port="514") + + +########################### +#### GLOBAL DIRECTIVES #### +########################### + +# +# Use traditional timestamp format. +# To enable high precision timestamps, comment out the following line. +# +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat + +# +# Set the default permissions for all log files. +# +$FileOwner root +$FileGroup adm +$FileCreateMode 0640 +$DirCreateMode 0755 +$Umask 0022 + +# +# Where to place spool and state files +# +$WorkDirectory /var/spool/rsyslog + +# +# Include all config files in /etc/rsyslog.d/ +# +$IncludeConfig /etc/rsyslog.d/*.conf + + +############### +#### RULES #### +############### + +# +# First some standard log files. Log by facility. +# +auth,authpriv.* /var/log/auth.log +*.*;auth,authpriv.none -/var/log/syslog +#cron.* /var/log/cron.log +daemon.* -/var/log/daemon.log +kern.* -/var/log/kern.log +lpr.* -/var/log/lpr.log +mail.* -/var/log/mail.log +user.* -/var/log/user.log + +# +# Logging for the mail system. Split it up so that +# it is easy to write scripts to parse these files. +# +mail.info -/var/log/mail.info +mail.warn -/var/log/mail.warn +mail.err /var/log/mail.err + +# +# Logging for INN news system. +# +news.crit /var/log/news/news.crit +news.err /var/log/news/news.err +news.notice -/var/log/news/news.notice + +# +# Some "catch-all" log files. +# +*.=debug;\ + auth,authpriv.none;\ + news.none;mail.none -/var/log/debug +*.=info;*.=notice;*.=warn;\ + auth,authpriv.none;\ + cron,daemon.none;\ + mail,news.none -/var/log/messages + +# +# Emergencies are sent to everybody logged in. +# +*.emerg :omusrmsg:* diff --git a/data/docker/integration/conf/server.properties b/data/docker/integration/conf/server.properties new file mode 100644 index 000000000..521fb74a4 --- /dev/null +++ b/data/docker/integration/conf/server.properties @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# see kafka.server.KafkaConfig for additional details and defaults + +############################# Server Basics ############################# + +# The id of the broker. This must be set to a unique integer for each broker. +broker.id=0 + +############################# Socket Server Settings ############################# + +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 +#listeners=PLAINTEXT://:9092 + +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 + +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + +# The number of threads that the server uses for receiving requests from the network and sending responses to the network +num.network.threads=1 + +# The number of threads that the server uses for processing requests, which may include disk I/O +num.io.threads=1 + +# The send buffer (SO_SNDBUF) used by the socket server +socket.send.buffer.bytes=102400 + +# The receive buffer (SO_RCVBUF) used by the socket server +socket.receive.buffer.bytes=102400 + +# The maximum size of a request that the socket server will accept (protection against OOM) +socket.request.max.bytes=50000000 + + +############################# Log Basics ############################# + +# A comma separated list of directories under which to store log files +log.dirs=/tmp/kafka-logs + +# The default number of log partitions per topic. More partitions allow greater +# parallelism for consumption, but this will also result in more files across +# the brokers. +num.partitions=1 + +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + +############################# Log Flush Policy ############################# + +# Messages are immediately written to the filesystem but by default we only fsync() to sync +# the OS cache lazily. The following configurations control the flush of data to disk. +# There are a few important trade-offs here: +# 1. Durability: Unflushed data may be lost if you are not using replication. +# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. +# The settings below allow one to configure the flush policy to flush data after a period of time or +# every N messages (or both). This can be done globally and overridden on a per-topic basis. + +# The number of messages to accept before forcing a flush of data to disk +#log.flush.interval.messages=10000 + +# The maximum amount of time a message can sit in a log before we force a flush +#log.flush.interval.ms=1000 + +############################# Log Retention Policy ############################# + +# The following configurations control the disposal of log segments. The policy can +# be set to delete segments after a period of time, or after a given size has accumulated. +# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens +# from the end of the log. + +# The minimum age of a log file to be eligible for deletion due to age +log.retention.hours=5 + +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. +log.retention.bytes=20971520 + +# The maximum size of a log segment file. When this size is reached a new log segment will be created. +log.segment.bytes=20971520 + +# The interval at which log segments are checked to see if they can be deleted according +# to the retention policies +log.retention.check.interval.ms=300000 + +############################# Zookeeper ############################# + +# Zookeeper connection string (see zookeeper docs for details). +# This is a comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". +# You can also append an optional chroot string to the urls to specify the +# root directory for all kafka znodes. +zookeeper.connect=localhost:2181 + +# Timeout in ms for connecting to zookeeper +zookeeper.connection.timeout.ms=6000 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 +delete.topic.enable = true diff --git a/data/docker/integration/docker-entrypoint.sh b/data/docker/integration/docker-entrypoint.sh new file mode 100755 index 000000000..8efaa8b0b --- /dev/null +++ b/data/docker/integration/docker-entrypoint.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -ev + +ansible-playbook /integration_tests.yaml -i localhost, -c local -e "ansible_user=root clone_repository=true" -vvv + +# # Create symbolic links +ln -s "/go/src/github.com/cgrates/cgrates/data" "/usr/share/cgrates" +ln -s "/go/bin/cgr-engine" "/usr/bin/cgr-engine" +ln -s "/go/bin/cgr-loader" "/usr/bin/cgr-loader" +ln -s "/go/bin/cgr-migrator" "/usr/bin/cgr-migrator" +ln -s "/go/bin/cgr-console" "/usr/bin/cgr-console" +ln -s "/go/bin/cgr-tester" "/usr/bin/cgr-tester" + +# start basic subsystems +# export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M" +# /kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties +# /kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties + +rsyslogd -f /etc/rsyslogd.conf +version=$(ls /var/lib/postgresql) +pg_ctlcluster $version main start & +mongod --bind_ip 127.0.0.1 --logpath /logs/mongodb.log & +redis-server /etc/redis/redis.conf & +MYSQL_ROOT_PASSWORD="CGRateS.org" /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mariadb-ep.sh mysqld +# rabbitmq-server > /logs/rabbitmq.log 2>&1 & + + +# START_TIMEOUT=600 + +# start_timeout_exceeded=false +# count=0 +# step=10 +# while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do +# echo "waiting for kafka to be ready" +# sleep $step; +# count=$((count + step)) +# if [ $count -gt $START_TIMEOUT ]; then +# start_timeout_exceeded=true +# break +# fi +# done + +# if $start_timeout_exceeded; then +# echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" +# exit 1 +# fi + +# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates +# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs + + + +# gosu postgres psql -c "CREATE USER cgrates password 'CGRateS.org';" > /dev/null 2>&1 +# gosu postgres createdb -e -O cgrates cgrates > /dev/null 2>&1 + + + +# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_cdrs_tables.sql >/dev/null 2>&1 +# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql >/dev/null 2>&1 + + +# mongo --quiet /cgrates/data/docker/integration/scripts/create_user.js >/dev/null 2>&1 +echo "Starting..." +sleep 60 # Pause for 1 min +echo "Resuming..." +mysql -u root -pCGRateS.org -h localhost < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_db_with_users.sql +mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_cdrs_tables.sql +mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql + +cp -r /go/src/github.com/cgrates/cgrates/data/. /usr/share/cgrates + +# Set versions +# cgr-migrator -exec=*set_versions -config_path=/usr/share/cgrates/conf/samples/tutmysql \ No newline at end of file diff --git a/data/docker/integration/main.yaml b/data/docker/integration/main.yaml new file mode 100644 index 000000000..dffaafca1 --- /dev/null +++ b/data/docker/integration/main.yaml @@ -0,0 +1,46 @@ +--- +- hosts: all + vars: + golang_gopath: "/go" + clone_repository: true + cgrates_dir: "{{ golang_gopath }}/src/github.com/cgrates/cgrates" + golang_install_dir: /usr/local/go + git_version: "v0.10" + cgrates_dependencies: + - git + - redis-server + - mariadb-server + - make + - gcc + + tasks: + - name: Install CGRateS dependencies + become: yes + ansible.builtin.package: + name: "{{ cgrates_dependencies }}" + state: present + update_cache: yes + cache_valid_time: 86400 + + - name: Create cgrates directory + ansible.builtin.file: + state: directory + mode: "u=rwx,go=rx" + owner: "{{ ansible_user }}" + dest: "{{ cgrates_dir }}" + when: clone_repository | bool + + - name: Git clone cgrates + ansible.builtin.git: + repo: https://github.com/cgrates/cgrates.git + dest: "{{ cgrates_dir }}" + update: yes + force: yes + version: "{{ git_version }}" + when: clone_repository | bool + + - name: Build cgrates + ansible.builtin.shell: + cmd: bash -lc "sh {{ cgrates_dir }}/build.sh" + args: + chdir: "{{ cgrates_dir }}" diff --git a/data/docker/integration/scripts/mariadb-ep.sh b/data/docker/integration/scripts/mariadb-ep.sh new file mode 100755 index 000000000..be0f2d142 --- /dev/null +++ b/data/docker/integration/scripts/mariadb-ep.sh @@ -0,0 +1,356 @@ +#!/bin/bash +set -eo pipefail +shopt -s nullglob + +# logging functions +mysql_log() { + # return + local type="$1"; shift + printf '%s [%s] [Entrypoint]: %s\n' "$(date --rfc-3339=seconds)" "$type" "$*" +} +mysql_note() { + mysql_log Note "$@" +} +mysql_warn() { + mysql_log Warn "$@" >&2 +} +mysql_error() { + mysql_log ERROR "$@" >&2 + exit 1 +} + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + mysql_error "Both $var and $fileVar are set (but are exclusive)" + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions +docker_process_init_files() { + # mysql here for backwards compatibility "${mysql[@]}" + mysql=( docker_process_sql ) + + echo + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + mysql_note "$0: running $f" + "$f" + else + mysql_note "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) mysql_note "$0: running $f"; docker_process_sql < "$f"; echo ;; + *.sql.gz) mysql_note "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; + *.sql.xz) mysql_note "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; + *) mysql_warn "$0: ignoring $f" ;; + esac + echo + done +} + +mysql_check_config() { + local toRun=( "$@" --verbose --help --log-bin-index="$(mktemp -u)" ) errors + if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then + mysql_error $'mysqld failed while attempting to check config\n\tcommand was: '"${toRun[*]}"$'\n\t'"$errors" + fi +} + +# Fetch value from server config +# We use mysqld --verbose --help instead of my_print_defaults because the +# latter only show values present in config files, and not server defaults +mysql_get_config() { + local conf="$1"; shift + "$@" --verbose --help --log-bin-index="$(mktemp -u)" 2>/dev/null \ + | awk -v conf="$conf" '$1 == conf && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }' + # match "datadir /some/path with/spaces in/it here" but not "--xyz=abc\n datadir (xyz)" +} + +# Do a temporary startup of the MySQL server, for init purposes +docker_temp_server_start() { + "$@" --skip-networking --socket="${SOCKET}" & + mysql_note "Waiting for server startup" + local i + for i in {30..0}; do + # only use the root password if the database has already been initializaed + # so that it won't try to fill in a password file when it hasn't been set yet + extraArgs=() + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + extraArgs+=( '--dont-use-mysql-root-password' ) + fi + if docker_process_sql "${extraArgs[@]}" --database=mysql <<<'SELECT 1' &> /dev/null; then + break + fi + sleep 1 + done + if [ "$i" = 0 ]; then + mysql_error "Unable to start server." + fi +} + +# Stop the server. When using a local socket file mysqladmin will block until +# the shutdown is complete. +docker_temp_server_stop() { + if ! mysqladmin --defaults-extra-file=<( _mysql_passfile ) shutdown -uroot --socket="${SOCKET}"; then + mysql_error "Unable to shut down server." + fi +} + +# Verify that the minimally required password settings are set for new databases. +docker_verify_minimum_env() { + if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then + mysql_error $'Database is uninitialized and password option is not specified\n\tYou need to specify one of MYSQL_ROOT_PASSWORD, MYSQL_ALLOW_EMPTY_PASSWORD and MYSQL_RANDOM_ROOT_PASSWORD' + fi +} + +# creates folders for the database +# also ensures permission for user mysql of run as root +docker_create_db_directories() { + local user; user="$(id -u)" + + # TODO other directories that are used by default? like /var/lib/mysql-files + # see https://github.com/docker-library/mysql/issues/562 + mkdir -p "$DATADIR" + + if [ "$user" = "0" ]; then + # this will cause less disk access than `chown -R` + find "$DATADIR" \! -user mysql -exec chown mysql '{}' + + fi +} + +# initializes the database directory +docker_init_database_dir() { + mysql_note "Initializing database files" + installArgs=( --datadir="$DATADIR" --rpm --auth-root-authentication-method=normal ) + if { mysql_install_db --help || :; } | grep -q -- '--skip-test-db'; then + # 10.3+ + installArgs+=( --skip-test-db ) + fi + # "Other options are passed to mysqld." (so we pass all "mysqld" arguments directly here) + mysql_install_db "${installArgs[@]}" "${@:2}" + mysql_note "Database files initialized" +} + +# Loads various settings that are used elsewhere in the script +# This should be called after mysql_check_config, but before any other functions +docker_setup_env() { + # Get config + declare -g DATADIR SOCKET + DATADIR="$(mysql_get_config 'datadir' "$@")" + SOCKET="$(mysql_get_config 'socket' "$@")" + + # Initialize values that might be stored in a file + file_env 'MYSQL_ROOT_HOST' '%' + file_env 'MYSQL_DATABASE' + file_env 'MYSQL_USER' + file_env 'MYSQL_PASSWORD' + file_env 'MYSQL_ROOT_PASSWORD' + + declare -g DATABASE_ALREADY_EXISTS + if [ -d "$DATADIR/mysql" ]; then + DATABASE_ALREADY_EXISTS='true' + fi +} + +# Execute sql script, passed via stdin +# usage: docker_process_sql [--dont-use-mysql-root-password] [mysql-cli-args] +# ie: docker_process_sql --database=mydb <<<'INSERT ...' +# ie: docker_process_sql --dont-use-mysql-root-password --database=mydb /dev/null + + docker_init_database_dir "$@" + + mysql_note "Starting temporary server" + docker_temp_server_start "$@" + mysql_note "Temporary server started." + + docker_setup_db + # docker_process_init_files /docker-entrypoint-initdb.d/* + + mysql_note "Stopping temporary server" + docker_temp_server_stop + mysql_note "Temporary server stopped" + + echo + mysql_note "MySQL init process done. Ready for start up." + echo + fi + fi + exec "$@" > /logs/mariadb.log 2>&1 & +} + +# If we are sourced from elsewhere, don't perform any further actions +if ! _is_sourced; then + _main "$@" +fi diff --git a/data/docker/integration/scripts/mongo/create_user.js b/data/docker/integration/scripts/mongo/create_user.js new file mode 100644 index 000000000..169461df6 --- /dev/null +++ b/data/docker/integration/scripts/mongo/create_user.js @@ -0,0 +1,9 @@ +db = db.getSiblingDB('cgrates') +db.createUser( + { + user: "cgrates", + pwd: "CGRateS.org", + roles: [ { role: "dbAdmin", db: "cgrates" } ] + } +) + diff --git a/data/docker/integration/scripts/mongo/setup_cgr_db.sh b/data/docker/integration/scripts/mongo/setup_cgr_db.sh new file mode 100755 index 000000000..4cbc93516 --- /dev/null +++ b/data/docker/integration/scripts/mongo/setup_cgr_db.sh @@ -0,0 +1,14 @@ +#! /usr/bin/env sh + + +mongo --quiet create_user.js +cu=$? + +if [ $cu = 0 ]; then + echo "" + echo "\t+++ CGR-DB successfully set-up! +++" + echo "" + exit 0 +fi + + diff --git a/data/docker/integration/scripts/mysql/alter_cdr_tables_rc5_rc6.sql b/data/docker/integration/scripts/mysql/alter_cdr_tables_rc5_rc6.sql new file mode 100644 index 000000000..e3ddad60b --- /dev/null +++ b/data/docker/integration/scripts/mysql/alter_cdr_tables_rc5_rc6.sql @@ -0,0 +1,50 @@ +USE `cgrates`; + +ALTER TABLE `cdrs_primary` + CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first , + CHANGE `cgrid` `cgrid` char(40) NOT NULL after `id` , + ADD COLUMN `pdd` decimal(12,9) NOT NULL after `setup_time` , + CHANGE `answer_time` `answer_time` datetime NULL after `pdd` , + ADD COLUMN `supplier` varchar(128) NOT NULL after `usage` , + ADD COLUMN `disconnect_cause` varchar(64) NOT NULL after `supplier` , + ADD COLUMN `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP after `disconnect_cause` , + ADD COLUMN `deleted_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at` , + ADD KEY `answer_time_idx`(`answer_time`) , + ADD KEY `deleted_at_idx`(`deleted_at`) , + DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ; + +ALTER TABLE `cdrs_extra` + CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first , + CHANGE `cgrid` `cgrid` char(40) NOT NULL after `id` , + ADD COLUMN `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP after `extra_fields` , + ADD COLUMN `deleted_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at`, + ADD UNIQUE KEY `cgrid`(`cgrid`) , + ADD KEY `deleted_at_idx`(`deleted_at`) , + DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ; + +ALTER TABLE `cost_details` + CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first , + CHANGE `cost_source` `cost_source` varchar(64) NOT NULL after `timespans` , + ADD COLUMN `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP after `cost_source` , + ADD COLUMN `updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at` , + ADD COLUMN `deleted_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `updated_at` , + DROP COLUMN `cost_time` , + ADD KEY `deleted_at_idx`(`deleted_at`) , + DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ; + +ALTER TABLE `rated_cdrs` + CHANGE COLUMN tbid `id` int(11) NOT NULL auto_increment first , + CHANGE `cgrid` `cgrid` char(40) NOT NULL after `id` , + CHANGE `category` `category` varchar(32) NOT NULL after `tenant` , + ADD COLUMN `pdd` decimal(12,9) NOT NULL after `setup_time` , + CHANGE `answer_time` `answer_time` datetime NULL after `pdd` , + ADD COLUMN `supplier` varchar(128) NOT NULL after `usage` , + ADD COLUMN `disconnect_cause` varchar(64) NOT NULL after `supplier` , + CHANGE `cost` `cost` decimal(20,4) NULL after `disconnect_cause` , + ADD COLUMN `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP after `extra_info` , + ADD COLUMN `updated_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `created_at` , + ADD COLUMN `deleted_at` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' after `updated_at` , + DROP COLUMN `mediation_time` , + ADD KEY `deleted_at_idx`(`deleted_at`) , + DROP KEY `PRIMARY`, ADD PRIMARY KEY(`id`) ; + diff --git a/data/docker/integration/scripts/mysql/create_cdrs_tables.sql b/data/docker/integration/scripts/mysql/create_cdrs_tables.sql new file mode 100644 index 000000000..a563b1bac --- /dev/null +++ b/data/docker/integration/scripts/mysql/create_cdrs_tables.sql @@ -0,0 +1,52 @@ +-- +-- Table structure for table `cdrs` +-- + +DROP TABLE IF EXISTS cdrs; +CREATE TABLE cdrs ( + id int(11) NOT NULL AUTO_INCREMENT, + cgrid varchar(40) NOT NULL, + run_id varchar(64) NOT NULL, + origin_host varchar(64) NOT NULL, + source varchar(64) NOT NULL, + origin_id varchar(128) NOT NULL, + tor varchar(16) NOT NULL, + request_type varchar(24) NOT NULL, + tenant varchar(64) NOT NULL, + category varchar(64) NOT NULL, + account varchar(128) NOT NULL, + subject varchar(128) NOT NULL, + destination varchar(128) NOT NULL, + setup_time datetime NOT NULL, + answer_time datetime NULL, + `usage` BIGINT NOT NULL, + extra_fields text NOT NULL, + cost_source varchar(64) NOT NULL, + cost DECIMAL(20,4) NOT NULL, + cost_details MEDIUMTEXT, + extra_info text, + created_at TIMESTAMP NULL, + updated_at TIMESTAMP NULL, + deleted_at TIMESTAMP NULL, + PRIMARY KEY (id), + UNIQUE KEY cdrrun (cgrid, run_id) +); + +DROP TABLE IF EXISTS session_costs; +CREATE TABLE session_costs ( + id int(11) NOT NULL AUTO_INCREMENT, + cgrid varchar(40) NOT NULL, + run_id varchar(64) NOT NULL, + origin_host varchar(64) NOT NULL, + origin_id varchar(128) NOT NULL, + cost_source varchar(64) NOT NULL, + `usage` BIGINT NOT NULL, + cost_details MEDIUMTEXT, + created_at TIMESTAMP NULL, + deleted_at TIMESTAMP NULL, + PRIMARY KEY (`id`), + UNIQUE KEY costid (cgrid, run_id), + KEY origin_idx (origin_host, origin_id), + KEY run_origin_idx (run_id, origin_id), + KEY deleted_at_idx (deleted_at) +); diff --git a/data/docker/integration/scripts/mysql/create_db_with_users.sql b/data/docker/integration/scripts/mysql/create_db_with_users.sql new file mode 100644 index 000000000..3a9e88304 --- /dev/null +++ b/data/docker/integration/scripts/mysql/create_db_with_users.sql @@ -0,0 +1,10 @@ + +-- +-- Sample db and users creation. Replace here with your own details +-- + +DROP DATABASE IF EXISTS cgrates; +CREATE DATABASE cgrates; +CREATE USER IF NOT EXISTS 'cgrates'@'localhost' IDENTIFIED BY 'CGRateS.org'; +GRANT ALL PRIVILEGES ON cgrates.* TO 'cgrates'@'localhost' WITH GRANT OPTION; +FLUSH PRIVILEGES; diff --git a/data/docker/integration/scripts/mysql/create_ers_db.sql b/data/docker/integration/scripts/mysql/create_ers_db.sql new file mode 100644 index 000000000..99a6bc15e --- /dev/null +++ b/data/docker/integration/scripts/mysql/create_ers_db.sql @@ -0,0 +1,7 @@ + +-- +-- extra DB for ees and ers +DROP DATABASE IF EXISTS cgrates2; +CREATE DATABASE cgrates2; + +GRANT ALL on cgrates2.* TO 'cgrates'@'localhost' IDENTIFIED BY 'CGRateS.org'; diff --git a/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql b/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql new file mode 100644 index 000000000..c184be757 --- /dev/null +++ b/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql @@ -0,0 +1,491 @@ +-- +-- Table structure for table `tp_timings` +-- +DROP TABLE IF EXISTS `tp_timings`; +CREATE TABLE `tp_timings` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `years` varchar(255) NOT NULL, + `months` varchar(255) NOT NULL, + `month_days` varchar(255) NOT NULL, + `week_days` varchar(255) NOT NULL, + `time` varchar(32) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_tmid` (`tpid`,`tag`), + UNIQUE KEY `tpid_tag` (`tpid`,`tag`) +); + +-- +-- Table structure for table `tp_destinations` +-- + +DROP TABLE IF EXISTS `tp_destinations`; +CREATE TABLE `tp_destinations` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `prefix` varchar(24) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_dstid` (`tpid`,`tag`), + UNIQUE KEY `tpid_dest_prefix` (`tpid`,`tag`,`prefix`) +); + +-- +-- Table structure for table `tp_rates` +-- + +DROP TABLE IF EXISTS `tp_rates`; +CREATE TABLE `tp_rates` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `connect_fee` decimal(7,4) NOT NULL, + `rate` decimal(10,4) NOT NULL, + `rate_unit` varchar(16) NOT NULL, + `rate_increment` varchar(16) NOT NULL, + `group_interval_start` varchar(16) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `unique_tprate` (`tpid`,`tag`,`group_interval_start`), + KEY `tpid` (`tpid`), + KEY `tpid_rtid` (`tpid`,`tag`) +); + +-- +-- Table structure for table `destination_rates` +-- + +DROP TABLE IF EXISTS `tp_destination_rates`; +CREATE TABLE `tp_destination_rates` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `destinations_tag` varchar(64) NOT NULL, + `rates_tag` varchar(64) NOT NULL, + `rounding_method` varchar(255) NOT NULL, + `rounding_decimals` tinyint(4) NOT NULL, + `max_cost` decimal(7,4) NOT NULL, + `max_cost_strategy` varchar(16) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_drid` (`tpid`,`tag`), + UNIQUE KEY `tpid_drid_dstid` (`tpid`,`tag`,`destinations_tag`) +); + +-- +-- Table structure for table `tp_rating_plans` +-- + +DROP TABLE IF EXISTS `tp_rating_plans`; +CREATE TABLE `tp_rating_plans` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `destrates_tag` varchar(64) NOT NULL, + `timing_tag` varchar(64) NOT NULL, + `weight` DECIMAL(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_rpl` (`tpid`,`tag`), + UNIQUE KEY `tpid_rplid_destrates_timings_weight` (`tpid`,`tag`,`destrates_tag`,`timing_tag`) +); + +-- +-- Table structure for table `tp_rate_profiles` +-- + +DROP TABLE IF EXISTS `tp_rating_profiles`; +CREATE TABLE `tp_rating_profiles` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `loadid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `category` varchar(32) NOT NULL, + `subject` varchar(64) NOT NULL, + `activation_time` varchar(26) NOT NULL, + `rating_plan_tag` varchar(64) NOT NULL, + `fallback_subjects` varchar(64), + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + KEY `tpid_loadid` (`tpid`, `loadid`), + UNIQUE KEY `tpid_loadid_tenant_category_subj_atime` (`tpid`,`loadid`, `tenant`,`category`,`subject`,`activation_time`) +); + +-- +-- Table structure for table `tp_shared_groups` +-- + +DROP TABLE IF EXISTS `tp_shared_groups`; +CREATE TABLE `tp_shared_groups` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `account` varchar(64) NOT NULL, + `strategy` varchar(24) NOT NULL, + `rating_subject` varchar(24) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_shared_group` (`tpid`,`tag`,`account`,`strategy`,`rating_subject`) +); + +-- +-- Table structure for table `tp_actions` +-- + +DROP TABLE IF EXISTS `tp_actions`; +CREATE TABLE `tp_actions` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `action` varchar(24) NOT NULL, + `extra_parameters` varchar(256) NOT NULL, + `filters` varchar(256) NOT NULL, + `balance_tag` varchar(64) NOT NULL, + `balance_type` varchar(24) NOT NULL, + `categories` varchar(32) NOT NULL, + `destination_tags` varchar(64) NOT NULL, + `rating_subject` varchar(64) NOT NULL, + `shared_groups` varchar(64) NOT NULL, + `expiry_time` varchar(26) NOT NULL, + `timing_tags` varchar(128) NOT NULL, + `units` varchar(256) NOT NULL, + `balance_weight` varchar(10) NOT NULL, + `balance_blocker` varchar(5) NOT NULL, + `balance_disabled` varchar(24) NOT NULL, + `weight` DECIMAL(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_action` (`tpid`,`tag`,`action`,`balance_tag`,`balance_type`,`expiry_time`,`timing_tags`,`destination_tags`,`shared_groups`,`balance_weight`,`weight`) +); + +-- +-- Table structure for table `tp_action_timings` +-- + +DROP TABLE IF EXISTS `tp_action_plans`; +CREATE TABLE `tp_action_plans` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `actions_tag` varchar(64) NOT NULL, + `timing_tag` varchar(64) NOT NULL, + `weight` DECIMAL(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_action_schedule` (`tpid`,`tag`,`actions_tag`,`timing_tag`) +); + +-- +-- Table structure for table `tp_action_triggers` +-- + +DROP TABLE IF EXISTS `tp_action_triggers`; +CREATE TABLE `tp_action_triggers` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tag` varchar(64) NOT NULL, + `unique_id` varchar(64) NOT NULL, + `threshold_type` char(64) NOT NULL, + `threshold_value` DECIMAL(20,4) NOT NULL, + `recurrent` BOOLEAN NOT NULL, + `min_sleep` varchar(16) NOT NULL, + `expiry_time` varchar(26) NOT NULL, + `activation_time` varchar(26) NOT NULL, + `balance_tag` varchar(64) NOT NULL, + `balance_type` varchar(24) NOT NULL, + `balance_categories` varchar(32) NOT NULL, + `balance_destination_tags` varchar(64) NOT NULL, + `balance_rating_subject` varchar(64) NOT NULL, + `balance_shared_groups` varchar(64) NOT NULL, + `balance_expiry_time` varchar(26) NOT NULL, + `balance_timing_tags` varchar(128) NOT NULL, + `balance_weight` varchar(10) NOT NULL, + `balance_blocker` varchar(5) NOT NULL, + `balance_disabled` varchar(5) NOT NULL, + `actions_tag` varchar(64) NOT NULL, + `weight` DECIMAL(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_trigger_definition` (`tpid`,`tag`,`balance_tag`,`balance_type`,`threshold_type`,`threshold_value`,`balance_destination_tags`,`actions_tag`) +); + +-- +-- Table structure for table `tp_account_actions` +-- + +DROP TABLE IF EXISTS `tp_account_actions`; +CREATE TABLE `tp_account_actions` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `loadid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `account` varchar(64) NOT NULL, + `action_plan_tag` varchar(64), + `action_triggers_tag` varchar(64), + `allow_negative` BOOLEAN NOT NULL, + `disabled` BOOLEAN NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`id`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_account` (`tpid`,`loadid`,`tenant`,`account`) +); + +-- +-- Table structure for table `tp_resources` +-- + +DROP TABLE IF EXISTS tp_resources; +CREATE TABLE tp_resources ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `usage_ttl` varchar(32) NOT NULL, + `limit` varchar(64) NOT NULL, + `allocation_message` varchar(64) NOT NULL, + `blocker` BOOLEAN NOT NULL, + `stored` BOOLEAN NOT NULL, + `weight` decimal(8,2) NOT NULL, + `threshold_ids` varchar(64) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_resource` (`tpid`,`tenant`, `id`,`filter_ids` ) +); + +-- +-- Table structure for table `tp_stats` +-- + +DROP TABLE IF EXISTS tp_stats; +CREATE TABLE tp_stats ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `queue_length` int(11) NOT NULL, + `ttl` varchar(32) NOT NULL, + `min_items` int(11) NOT NULL, + `metric_ids` varchar(128) NOT NULL, + `metric_filter_ids` varchar(64) NOT NULL, + `stored` BOOLEAN NOT NULL, + `blocker` BOOLEAN NOT NULL, + `weight` decimal(8,2) NOT NULL, + `threshold_ids` varchar(64) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_stats` (`tpid`, `tenant`, `id`, `filter_ids`,`metric_ids`) +); + +-- +-- Table structure for table `tp_threshold_cfgs` +-- + +DROP TABLE IF EXISTS tp_thresholds; +CREATE TABLE tp_thresholds ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `max_hits` int(11) NOT NULL, + `min_hits` int(11) NOT NULL, + `min_sleep` varchar(16) NOT NULL, + `blocker` BOOLEAN NOT NULL, + `weight` decimal(8,2) NOT NULL, + `action_ids` varchar(64) NOT NULL, + `async` BOOLEAN NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_thresholds` (`tpid`,`tenant`, `id`,`filter_ids`,`action_ids`) +); + +-- +-- Table structure for table `tp_filter` +-- + +DROP TABLE IF EXISTS tp_filters; +CREATE TABLE tp_filters ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `type` varchar(16) NOT NULL, + `element` varchar(64) NOT NULL, + `values` varchar(256) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_filters` (`tpid`,`tenant`, `id`, `type`, `element`) +); + +-- +-- Table structure for table `tp_routes` +-- + + +DROP TABLE IF EXISTS tp_routes; +CREATE TABLE tp_routes ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `sorting` varchar(32) NOT NULL, + `sorting_parameters` varchar(64) NOT NULL, + `route_id` varchar(32) NOT NULL, + `route_filter_ids` varchar(64) NOT NULL, + `route_account_ids` varchar(64) NOT NULL, + `route_ratingplan_ids` varchar(64) NOT NULL, + `route_rate_profile_ids` varchar(64) NOT NULL, + `route_resource_ids` varchar(64) NOT NULL, + `route_stat_ids` varchar(64) NOT NULL, + `route_weight` decimal(8,2) NOT NULL, + `route_blocker` BOOLEAN NOT NULL, + `route_parameters` varchar(64) NOT NULL, + `weight` decimal(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_routes` (`tpid`,`tenant`, + `id`,`filter_ids`,`route_id`,`route_filter_ids`,`route_account_ids`, + `route_ratingplan_ids`,`route_resource_ids`,`route_stat_ids` ) +); + +-- +-- Table structure for table `tp_attributes` +-- + +DROP TABLE IF EXISTS tp_attributes; +CREATE TABLE tp_attributes ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `contexts` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `attribute_filter_ids` varchar(64) NOT NULL, + `path` varchar(64) NOT NULL, + `type` varchar(64) NOT NULL, + `value` varchar(64) NOT NULL, + `blocker` BOOLEAN NOT NULL, + `weight` decimal(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_attributes` (`tpid`,`tenant`, + `id`,`filter_ids`,`path`,`value` ) +); + +-- +-- Table structure for table `tp_chargers` +-- + +DROP TABLE IF EXISTS tp_chargers; +CREATE TABLE tp_chargers ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `run_id` varchar(64) NOT NULL, + `attribute_ids` varchar(64) NOT NULL, + `weight` decimal(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_chargers` (`tpid`,`tenant`, + `id`,`filter_ids`,`run_id`,`attribute_ids`) +); + +-- +-- Table structure for table `tp_dispatchers` +-- + +DROP TABLE IF EXISTS tp_dispatcher_profiles; +CREATE TABLE tp_dispatcher_profiles ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `subsystems` varchar(64) NOT NULL, + `filter_ids` varchar(64) NOT NULL, + `activation_interval` varchar(64) NOT NULL, + `strategy` varchar(64) NOT NULL, + `strategy_parameters` varchar(64) NOT NULL, + `conn_id` varchar(64) NOT NULL, + `conn_filter_ids` varchar(64) NOT NULL, + `conn_weight` decimal(8,2) NOT NULL, + `conn_blocker` BOOLEAN NOT NULL, + `conn_parameters` varchar(64) NOT NULL, + `weight` decimal(8,2) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_dispatcher_profiles` (`tpid`,`tenant`, + `id`,`filter_ids`,`strategy`,`conn_id`,`conn_filter_ids`) +); + +-- +-- Table structure for table `tp_dispatchers` +-- + +DROP TABLE IF EXISTS tp_dispatcher_hosts; +CREATE TABLE tp_dispatcher_hosts ( + `pk` int(11) NOT NULL AUTO_INCREMENT, + `tpid` varchar(64) NOT NULL, + `tenant` varchar(64) NOT NULL, + `id` varchar(64) NOT NULL, + `address` varchar(64) NOT NULL, + `transport` varchar(64) NOT NULL, + `connect_attempts` int(11) NOT NULL, + `reconnects` int(11) NOT NULL, + `max_reconnect_interval` varchar(64) NOT NULL, + `connect_timeout` varchar(64) NOT NULL, + `reply_timeout` varchar(64) NOT NULL, + `tls` BOOLEAN NOT NULL, + `client_key` varchar(64) NOT NULL, + `client_certificate` varchar(64) NOT NULL, + `ca_certificate` varchar(64) NOT NULL, + `created_at` TIMESTAMP, + PRIMARY KEY (`pk`), + KEY `tpid` (`tpid`), + UNIQUE KEY `unique_tp_dispatchers_hosts` (`tpid`,`tenant`, + `id`,`address`) +); + +-- +-- Table structure for table `versions` +-- + +DROP TABLE IF EXISTS versions; +CREATE TABLE versions ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `item` varchar(64) NOT NULL, + `version` int(11) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `id_item` (`id`,`item`) +); diff --git a/data/docker/integration/scripts/mysql/mysql_cdr_migration.sql b/data/docker/integration/scripts/mysql/mysql_cdr_migration.sql new file mode 100644 index 000000000..b44d1596d --- /dev/null +++ b/data/docker/integration/scripts/mysql/mysql_cdr_migration.sql @@ -0,0 +1,57 @@ +/* +This script will migrate CDRs from the old CGRateS tables to the new cdrs table +but it only migrate CDRs where the duration is > 0. +If you need CDRs also with duration is = 0 you can make the appropriate change in the line beginning WHERE cdrs_primary.usage +Also the script will process 10,000 CDRs before committing to save system resources +especially in systems where they are millions of CDRs to be migrated +You can increase or lower the value of step in the line after BEGIN below. + +You have to use 'CALL cgrates.migration();' to execute the script. If named other then default use that database name. +*/ + + +DELIMITER // + +CREATE PROCEDURE `migration`() +BEGIN + /* DECLARE variables */ + DECLARE max_cdrs bigint; + DECLARE start_id bigint; + DECLARE end_id bigint; + DECLARE step bigint; + /* Optimize table for performance */ + ALTER TABLE cdrs DISABLE KEYS; + SET autocommit=0; + SET unique_checks=0; + SET foreign_key_checks=0; + /* You must change the step var to commit every step rows inserted */ + SET step := 10000; + SET start_id := 0; + SET end_id := start_id + step; + SET max_cdrs = (select max(id) from rated_cdrs); + WHILE (start_id <= max_cdrs) DO + INSERT INTO + cdrs(cgrid,run_id,origin_host,source,origin_id,tor,request_type,tenant,category,account,subject,destination,setup_time,pdd,answer_time,`usage`,supplier,disconnect_cause,extra_fields,cost_source,cost,cost_details,extra_info, created_at, updated_at, deleted_at) + SELECT cdrs_primary.cgrid,rated_cdrs.runid as run_id,cdrs_primary.cdrhost as origin_host,cdrs_primary.cdrsource as source,cdrs_primary.accid as origin_id, cdrs_primary.tor,rated_cdrs.reqtype as request_type, rated_cdrs.tenant,rated_cdrs.category, rated_cdrs.account, rated_cdrs.subject, rated_cdrs.destination,rated_cdrs.setup_time,rated_cdrs.pdd,rated_cdrs.answer_time,rated_cdrs.`usage`,rated_cdrs.supplier,rated_cdrs.disconnect_cause,cdrs_extra.extra_fields,cost_details.cost_source,rated_cdrs.cost,cost_details.timespans as cost_details,rated_cdrs.extra_info,rated_cdrs.created_at,rated_cdrs.updated_at, rated_cdrs.deleted_at + FROM rated_cdrs + INNER JOIN cdrs_primary ON rated_cdrs.cgrid = cdrs_primary.cgrid + INNER JOIN cdrs_extra ON rated_cdrs.cgrid = cdrs_extra.cgrid + INNER JOIN cost_details ON rated_cdrs.cgrid = cost_details.cgrid + WHERE cdrs_primary.`usage` > '0' + AND not exists (select 1 from cdrs where cdrs.cgrid = cdrs_primary.cgrid AND cdrs.run_id=rated_cdrs.runid) + AND rated_cdrs.id >= start_id + AND rated_cdrs.id < end_id + GROUP BY cgrid, run_id, origin_id; + SET start_id = start_id + step; + SET end_id = end_id + step; + END WHILE; + /* SET Table for live usage */ + SET autocommit=1; + SET unique_checks=1; + SET foreign_key_checks=1; + ALTER TABLE cdrs ENABLE KEYS; + OPTIMIZE TABLE cdrs; +END // + +DELIMITER ; + diff --git a/data/docker/integration/scripts/mysql/setup_cgr_db.sh b/data/docker/integration/scripts/mysql/setup_cgr_db.sh new file mode 100755 index 000000000..8561b7e9d --- /dev/null +++ b/data/docker/integration/scripts/mysql/setup_cgr_db.sh @@ -0,0 +1,29 @@ +#! /usr/bin/env sh + +if test $# -lt 2; then + echo "" + echo "setup_cgr_db.sh []" + echo "" + exit 0 +fi + +host=$3 +if [ -z "$3" ]; then + host="localhost" +fi + +DIR="$(dirname "$(readlink -f "$0")")" + +mysql -u $1 -p$2 -h $host < "$DIR"/create_db_with_users.sql +cu=$? +mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_cdrs_tables.sql +cdrt=$? +mysql -u $1 -p$2 -h $host -D cgrates < "$DIR"/create_tariffplan_tables.sql +tpt=$? + +if [ $cu = 0 ] && [ $cdrt = 0 ] && [ $tpt = 0 ]; then + echo "\n\t+++ CGR-DB successfully set-up! +++\n" + exit 0 +fi + + diff --git a/data/docker/integration/scripts/mysql/setup_ers_db.sh b/data/docker/integration/scripts/mysql/setup_ers_db.sh new file mode 100755 index 000000000..455315e3c --- /dev/null +++ b/data/docker/integration/scripts/mysql/setup_ers_db.sh @@ -0,0 +1,25 @@ +#! /usr/bin/env sh + +if test $# -lt 2; then + echo "" + echo "setup_cgr_db.sh []" + echo "" + exit 0 +fi + +host=$3 +if [ -z "$3" ]; then + host="localhost" +fi + +DIR="$(dirname "$(readlink -f "$0")")" + +mysql -u $1 -p$2 -h $host < "$DIR"/create_ers_db.sql +cu=$? + +if [ $cu = 0 ]; then + echo "\n\t+++ CGR-DB successfully set-up! +++\n" + exit 0 +fi + + diff --git a/data/docker/integration/scripts/postgres/create_cdrs_tables.sql b/data/docker/integration/scripts/postgres/create_cdrs_tables.sql new file mode 100644 index 000000000..3ae7d5ccb --- /dev/null +++ b/data/docker/integration/scripts/postgres/create_cdrs_tables.sql @@ -0,0 +1,59 @@ +-- +-- Table structure for table `cdrs` +-- + +DROP TABLE IF EXISTS cdrs; +CREATE TABLE cdrs ( + id SERIAL PRIMARY KEY, + cgrid VARCHAR(40) NOT NULL, + run_id VARCHAR(64) NOT NULL, + origin_host VARCHAR(64) NOT NULL, + source VARCHAR(64) NOT NULL, + origin_id VARCHAR(128) NOT NULL, + tor VARCHAR(16) NOT NULL, + request_type VARCHAR(24) NOT NULL, + tenant VARCHAR(64) NOT NULL, + category VARCHAR(64) NOT NULL, + account VARCHAR(128) NOT NULL, + subject VARCHAR(128) NOT NULL, + destination VARCHAR(128) NOT NULL, + setup_time TIMESTAMP WITH TIME ZONE NOT NULL, + answer_time TIMESTAMP WITH TIME ZONE NULL, + usage BIGINT NOT NULL, + extra_fields jsonb NOT NULL, + cost_source VARCHAR(64) NOT NULL, + cost NUMERIC(20,4) DEFAULT NULL, + cost_details jsonb, + extra_info text, + created_at TIMESTAMP WITH TIME ZONE, + updated_at TIMESTAMP WITH TIME ZONE NULL, + deleted_at TIMESTAMP WITH TIME ZONE NULL, + UNIQUE (cgrid, run_id) +); +; +DROP INDEX IF EXISTS deleted_at_cp_idx; +CREATE INDEX deleted_at_cp_idx ON cdrs (deleted_at); + + +DROP TABLE IF EXISTS session_costs; +CREATE TABLE session_costs ( + id SERIAL PRIMARY KEY, + cgrid VARCHAR(40) NOT NULL, + run_id VARCHAR(64) NOT NULL, + origin_host VARCHAR(64) NOT NULL, + origin_id VARCHAR(128) NOT NULL, + cost_source VARCHAR(64) NOT NULL, + usage BIGINT NOT NULL, + cost_details jsonb, + created_at TIMESTAMP WITH TIME ZONE, + deleted_at TIMESTAMP WITH TIME ZONE NULL, + UNIQUE (cgrid, run_id) +); +DROP INDEX IF EXISTS cgrid_sessionscost_idx; +CREATE INDEX cgrid_sessionscost_idx ON session_costs (cgrid, run_id); +DROP INDEX IF EXISTS origin_sessionscost_idx; +CREATE INDEX origin_sessionscost_idx ON session_costs (origin_host, origin_id); +DROP INDEX IF EXISTS run_origin_sessionscost_idx; +CREATE INDEX run_origin_sessionscost_idx ON session_costs (run_id, origin_id); +DROP INDEX IF EXISTS deleted_at_sessionscost_idx; +CREATE INDEX deleted_at_sessionscost_idx ON session_costs (deleted_at); diff --git a/data/docker/integration/scripts/postgres/create_db_with_users.sh b/data/docker/integration/scripts/postgres/create_db_with_users.sh new file mode 100755 index 000000000..b193b3f5f --- /dev/null +++ b/data/docker/integration/scripts/postgres/create_db_with_users.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# +# Sample db and users creation. Replace here with your own details +# + +sudo -u postgres dropdb -e cgrates +sudo -u postgres dropuser -e cgrates +sudo -u postgres psql -c "CREATE USER cgrates password 'CGRateS.org';" +sudo -u postgres createdb -e -O cgrates cgrates diff --git a/data/docker/integration/scripts/postgres/create_ers_db.sh b/data/docker/integration/scripts/postgres/create_ers_db.sh new file mode 100755 index 000000000..17b760cda --- /dev/null +++ b/data/docker/integration/scripts/postgres/create_ers_db.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# extra DB for ers +sudo -u postgres dropdb -e cgrates2 +sudo -u postgres createdb -e -O cgrates cgrates2 diff --git a/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql b/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql new file mode 100644 index 000000000..0d0bca9b3 --- /dev/null +++ b/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql @@ -0,0 +1,478 @@ +-- +-- Table structure for table `tp_timings` +-- +DROP TABLE IF EXISTS tp_timings; +CREATE TABLE tp_timings ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + years VARCHAR(255) NOT NULL, + months VARCHAR(255) NOT NULL, + month_days VARCHAR(255) NOT NULL, + week_days VARCHAR(255) NOT NULL, + time VARCHAR(32) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag) +); +CREATE INDEX tptimings_tpid_idx ON tp_timings (tpid); +CREATE INDEX tptimings_idx ON tp_timings (tpid,tag); + +-- +-- Table structure for table `tp_destinations` +-- + +DROP TABLE IF EXISTS tp_destinations; +CREATE TABLE tp_destinations ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + prefix VARCHAR(24) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, prefix) +); +CREATE INDEX tpdests_tpid_idx ON tp_destinations (tpid); +CREATE INDEX tpdests_idx ON tp_destinations (tpid,tag); + +-- +-- Table structure for table `tp_rates` +-- + +DROP TABLE IF EXISTS tp_rates; +CREATE TABLE tp_rates ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + connect_fee NUMERIC(7,4) NOT NULL, + rate NUMERIC(10,4) NOT NULL, + rate_unit VARCHAR(16) NOT NULL, + rate_increment VARCHAR(16) NOT NULL, + group_interval_start VARCHAR(16) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, group_interval_start) +); +CREATE INDEX tprates_tpid_idx ON tp_rates (tpid); +CREATE INDEX tprates_idx ON tp_rates (tpid,tag); + +-- +-- Table structure for table `destination_rates` +-- + +DROP TABLE IF EXISTS tp_destination_rates; +CREATE TABLE tp_destination_rates ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + destinations_tag VARCHAR(64) NOT NULL, + rates_tag VARCHAR(64) NOT NULL, + rounding_method VARCHAR(255) NOT NULL, + rounding_decimals SMALLINT NOT NULL, + max_cost NUMERIC(7,4) NOT NULL, + max_cost_strategy VARCHAR(16) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag , destinations_tag) +); +CREATE INDEX tpdestrates_tpid_idx ON tp_destination_rates (tpid); +CREATE INDEX tpdestrates_idx ON tp_destination_rates (tpid,tag); + +-- +-- Table structure for table `tp_rating_plans` +-- + +DROP TABLE IF EXISTS tp_rating_plans; +CREATE TABLE tp_rating_plans ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + destrates_tag VARCHAR(64) NOT NULL, + timing_tag VARCHAR(64) NOT NULL, + weight NUMERIC(8,2) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, destrates_tag, timing_tag) +); +CREATE INDEX tpratingplans_tpid_idx ON tp_rating_plans (tpid); +CREATE INDEX tpratingplans_idx ON tp_rating_plans (tpid,tag); + + +-- +-- Table structure for table `tp_rate_profiles` +-- + +DROP TABLE IF EXISTS tp_rating_profiles; +CREATE TABLE tp_rating_profiles ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + loadid VARCHAR(64) NOT NULL, + tenant VARCHAR(64) NOT NULL, + category VARCHAR(32) NOT NULL, + subject VARCHAR(64) NOT NULL, + activation_time VARCHAR(26) NOT NULL, + rating_plan_tag VARCHAR(64) NOT NULL, + fallback_subjects VARCHAR(64), + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, loadid, tenant, category, subject, activation_time) +); +CREATE INDEX tpratingprofiles_tpid_idx ON tp_rating_profiles (tpid); +CREATE INDEX tpratingprofiles_idx ON tp_rating_profiles (tpid,loadid,tenant,category,subject); + +-- +-- Table structure for table `tp_shared_groups` +-- + +DROP TABLE IF EXISTS tp_shared_groups; +CREATE TABLE tp_shared_groups ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + account VARCHAR(64) NOT NULL, + strategy VARCHAR(24) NOT NULL, + rating_subject VARCHAR(24) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, account , strategy , rating_subject) +); +CREATE INDEX tpsharedgroups_tpid_idx ON tp_shared_groups (tpid); +CREATE INDEX tpsharedgroups_idx ON tp_shared_groups (tpid,tag); + +-- +-- Table structure for table `tp_actions` +-- + +DROP TABLE IF EXISTS tp_actions; +CREATE TABLE tp_actions ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + action VARCHAR(24) NOT NULL, + extra_parameters VARCHAR(256) NOT NULL, + filters VARCHAR(256) NOT NULL, + balance_tag VARCHAR(64) NOT NULL, + balance_type VARCHAR(24) NOT NULL, + categories VARCHAR(32) NOT NULL, + destination_tags VARCHAR(64) NOT NULL, + rating_subject VARCHAR(64) NOT NULL, + shared_groups VARCHAR(64) NOT NULL, + expiry_time VARCHAR(26) NOT NULL, + timing_tags VARCHAR(128) NOT NULL, + units VARCHAR(256) NOT NULL, + balance_weight VARCHAR(10) NOT NULL, + balance_blocker VARCHAR(5) NOT NULL, + balance_disabled VARCHAR(5) NOT NULL, + weight NUMERIC(8,2) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, action, balance_tag, balance_type, expiry_time, timing_tags, destination_tags, shared_groups, balance_weight, weight) +); +CREATE INDEX tpactions_tpid_idx ON tp_actions (tpid); +CREATE INDEX tpactions_idx ON tp_actions (tpid,tag); + +-- +-- Table structure for table `tp_action_timings` +-- + +DROP TABLE IF EXISTS tp_action_plans; +CREATE TABLE tp_action_plans ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + actions_tag VARCHAR(64) NOT NULL, + timing_tag VARCHAR(64) NOT NULL, + weight NUMERIC(8,2) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, actions_tag, timing_tag) +); +CREATE INDEX tpactionplans_tpid_idx ON tp_action_plans (tpid); +CREATE INDEX tpactionplans_idx ON tp_action_plans (tpid,tag); + +-- +-- Table structure for table tp_action_triggers +-- + +DROP TABLE IF EXISTS tp_action_triggers; +CREATE TABLE tp_action_triggers ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + tag VARCHAR(64) NOT NULL, + unique_id VARCHAR(64) NOT NULL, + threshold_type VARCHAR(64) NOT NULL, + threshold_value NUMERIC(20,4) NOT NULL, + recurrent BOOLEAN NOT NULL, + min_sleep VARCHAR(16) NOT NULL, + expiry_time VARCHAR(26) NOT NULL, + activation_time VARCHAR(26) NOT NULL, + balance_tag VARCHAR(64) NOT NULL, + balance_type VARCHAR(24) NOT NULL, + balance_categories VARCHAR(32) NOT NULL, + balance_destination_tags VARCHAR(64) NOT NULL, + balance_rating_subject VARCHAR(64) NOT NULL, + balance_shared_groups VARCHAR(64) NOT NULL, + balance_expiry_time VARCHAR(26) NOT NULL, + balance_timing_tags VARCHAR(128) NOT NULL, + balance_weight VARCHAR(10) NOT NULL, + balance_blocker VARCHAR(5) NOT NULL, + balance_disabled VARCHAR(5) NOT NULL, + actions_tag VARCHAR(64) NOT NULL, + weight NUMERIC(8,2) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, tag, balance_tag, balance_type, threshold_type, threshold_value, balance_destination_tags, actions_tag) +); +CREATE INDEX tpactiontrigers_tpid_idx ON tp_action_triggers (tpid); +CREATE INDEX tpactiontrigers_idx ON tp_action_triggers (tpid,tag); + +-- +-- Table structure for table tp_account_actions +-- + +DROP TABLE IF EXISTS tp_account_actions; +CREATE TABLE tp_account_actions ( + id SERIAL PRIMARY KEY, + tpid VARCHAR(64) NOT NULL, + loadid VARCHAR(64) NOT NULL, + tenant VARCHAR(64) NOT NULL, + account VARCHAR(64) NOT NULL, + action_plan_tag VARCHAR(64), + action_triggers_tag VARCHAR(64), + allow_negative BOOLEAN NOT NULL, + disabled BOOLEAN NOT NULL, + created_at TIMESTAMP WITH TIME ZONE, + UNIQUE (tpid, loadid, tenant, account) +); +CREATE INDEX tpaccountactions_tpid_idx ON tp_account_actions (tpid); +CREATE INDEX tpaccountactions_idx ON tp_account_actions (tpid,loadid,tenant,account); + + +-- +-- Table structure for table `tp_resources` +-- + +DROP TABLE IF EXISTS tp_resources; +CREATE TABLE tp_resources ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "usage_ttl" varchar(32) NOT NULL, + "limit" varchar(64) NOT NULL, + "allocation_message" varchar(64) NOT NULL, + "blocker" BOOLEAN NOT NULL, + "stored" BOOLEAN NOT NULL, + "weight" NUMERIC(8,2) NOT NULL, + "threshold_ids" varchar(64) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); +CREATE INDEX tp_resources_idx ON tp_resources (tpid); +CREATE INDEX tp_resources_unique ON tp_resources ("tpid", "tenant", "id", "filter_ids"); + + +-- +-- Table structure for table `tp_stats` +-- + +DROP TABLE IF EXISTS tp_stats; +CREATE TABLE tp_stats ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "queue_length" INTEGER NOT NULL, + "ttl" varchar(32) NOT NULL, + "min_items" INTEGER NOT NULL, + "metric_ids" VARCHAR(128) NOT NULL, + "metric_filter_ids" VARCHAR(128) NOT NULL, + "stored" BOOLEAN NOT NULL, + "blocker" BOOLEAN NOT NULL, + "weight" decimal(8,2) NOT NULL, + "threshold_ids" varchar(64) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); +CREATE INDEX tp_stats_idx ON tp_stats (tpid); +CREATE INDEX tp_stats_unique ON tp_stats ("tpid","tenant", "id", "filter_ids","metric_ids"); + +-- +-- Table structure for table `tp_threshold_cfgs` +-- + +DROP TABLE IF EXISTS tp_thresholds; +CREATE TABLE tp_thresholds ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "max_hits" INTEGER NOT NULL, + "min_hits" INTEGER NOT NULL, + "min_sleep" varchar(16) NOT NULL, + "blocker" BOOLEAN NOT NULL, + "weight" decimal(8,2) NOT NULL, + "action_ids" varchar(64) NOT NULL, + "async" BOOLEAN NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); +CREATE INDEX tp_thresholds_idx ON tp_thresholds (tpid); +CREATE INDEX tp_thresholds_unique ON tp_thresholds ("tpid","tenant", "id","filter_ids","action_ids"); + +-- +-- Table structure for table `tp_filter` +-- + +DROP TABLE IF EXISTS tp_filters; +CREATE TABLE tp_filters ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant" varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "type" varchar(16) NOT NULL, + "element" varchar(64) NOT NULL, + "values" varchar(256) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); + CREATE INDEX tp_filters_idx ON tp_filters (tpid); + CREATE INDEX tp_filters_unique ON tp_filters ("tpid","tenant", "id", "type", "element"); + +-- +-- Table structure for table `tp_routes` +-- + +DROP TABLE IF EXISTS tp_routes; +CREATE TABLE tp_routes ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "sorting" varchar(32) NOT NULL, + "sorting_parameters" varchar(64) NOT NULL, + "route_id" varchar(32) NOT NULL, + "route_filter_ids" varchar(64) NOT NULL, + "route_account_ids" varchar(64) NOT NULL, + "route_ratingplan_ids" varchar(64) NOT NULL, + "route_resource_ids" varchar(64) NOT NULL, + "route_stat_ids" varchar(64) NOT NULL, + "route_weight" decimal(8,2) NOT NULL, + "route_blocker" BOOLEAN NOT NULL, + "route_parameters" varchar(64) NOT NULL, + "weight" decimal(8,2) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE +); +CREATE INDEX tp_routes_idx ON tp_routes (tpid); +CREATE INDEX tp_routes_unique ON tp_routes ("tpid", "tenant", "id", + "filter_ids","route_id","route_filter_ids","route_account_ids", + "route_ratingplan_ids","route_resource_ids","route_stat_ids"); + + -- + -- Table structure for table `tp_attributes` + -- + + DROP TABLE IF EXISTS tp_attributes; + CREATE TABLE tp_attributes ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "contexts" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "attribute_filter_ids" varchar(64) NOT NULL, + "path" varchar(64) NOT NULL, + "type" varchar(64) NOT NULL, + "value" varchar(64) NOT NULL, + "blocker" BOOLEAN NOT NULL, + "weight" decimal(8,2) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE + ); + CREATE INDEX tp_attributes_ids ON tp_attributes (tpid); + CREATE INDEX tp_attributes_unique ON tp_attributes ("tpid", "tenant", "id", + "filter_ids","path","value"); + + -- + -- Table structure for table `tp_chargers` + -- + + DROP TABLE IF EXISTS tp_chargers; + CREATE TABLE tp_chargers ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant"varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "run_id" varchar(64) NOT NULL, + "attribute_ids" varchar(64) NOT NULL, + "weight" decimal(8,2) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE + ); + CREATE INDEX tp_chargers_ids ON tp_chargers (tpid); + CREATE INDEX tp_chargers_unique ON tp_chargers ("tpid", "tenant", "id", + "filter_ids","run_id","attribute_ids"); + + -- + -- Table structure for table `tp_dispatchers` + -- + + DROP TABLE IF EXISTS tp_dispatcher_profiles; + CREATE TABLE tp_dispatcher_profiles ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant" varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "subsystems" varchar(64) NOT NULL, + "filter_ids" varchar(64) NOT NULL, + "activation_interval" varchar(64) NOT NULL, + "strategy" varchar(64) NOT NULL, + "strategy_parameters" varchar(64) NOT NULL, + "conn_id" varchar(64) NOT NULL, + "conn_filter_ids" varchar(64) NOT NULL, + "conn_weight" decimal(8,2) NOT NULL, + "conn_blocker" BOOLEAN NOT NULL, + "conn_parameters" varchar(64) NOT NULL, + "weight" decimal(8,2) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE + ); + CREATE INDEX tp_dispatcher_profiles_ids ON tp_dispatcher_profiles (tpid); + CREATE INDEX tp_dispatcher_profiles_unique ON tp_dispatcher_profiles ("tpid", "tenant", "id", + "filter_ids","strategy","conn_id","conn_filter_ids"); + +-- +-- Table structure for table `tp_dispatchers` +-- + + DROP TABLE IF EXISTS tp_dispatcher_hosts; + CREATE TABLE tp_dispatcher_hosts ( + "pk" SERIAL PRIMARY KEY, + "tpid" varchar(64) NOT NULL, + "tenant" varchar(64) NOT NULL, + "id" varchar(64) NOT NULL, + "address" varchar(64) NOT NULL, + "transport" varchar(64) NOT NULL, + "connect_attempts" INTEGER NOT NULL, + "reconnects" INTEGER NOT NULL, + "max_reconnect_interval" varchar(64) NOT NULL, + "connect_timeout" varchar(64) NOT NULL, + "reply_timeout" varchar(64) NOT NULL, + "tls" BOOLEAN NOT NULL, + "client_key" varchar(64) NOT NULL, + "client_certificate" varchar(64) NOT NULL, + "ca_certificate" varchar(64) NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE + ); + CREATE INDEX tp_dispatchers_hosts_ids ON tp_dispatcher_hosts (tpid); + CREATE INDEX tp_dispatcher_hosts_unique ON tp_dispatcher_hosts ("tpid", "tenant", "id", + "address"); + + + +-- +-- Table structure for table `versions` +-- + +DROP TABLE IF EXISTS versions; +CREATE TABLE versions ( + "id" SERIAL PRIMARY KEY, + "item" varchar(64) NOT NULL, + "version" INTEGER NOT NULL, + UNIQUE ("id","item") +); diff --git a/data/docker/integration/scripts/postgres/pg_cdr_migration.sql b/data/docker/integration/scripts/postgres/pg_cdr_migration.sql new file mode 100644 index 000000000..5f40cb6d0 --- /dev/null +++ b/data/docker/integration/scripts/postgres/pg_cdr_migration.sql @@ -0,0 +1,42 @@ +/* +This script will migrate CDRs from the old CGRateS tables to the new cdrs table +but it only migrate CDRs where the duration is > 0. +If you need CDRs also with duration is = 0 you can make the appropriate change in the line beginning WHERE cdrs_primary.usage + +Also the script will process 10,000 CDRs before committing to save system resources +especially in systems where they are millions of CDRs to be migrated +You can increase or lower the value of step in the line after BEGIN below. +*/ + +DO $$ +DECLARE + max_cdrs bigint; + start_id bigint; + end_id bigint; + step bigint; +BEGIN + /* You must change the step var to commit every step rows inserted */ + step := 10000; + start_id := 0; + end_id := start_id + step; + select max(id) INTO max_cdrs from rated_cdrs; + WHILE start_id <= max_cdrs + LOOP + --RAISE NOTICE '%', (to_char(start_id, '99999999') || '-' || to_char(end_id, '99999999')); + INSERT INTO + cdrs(cgrid,run_id,origin_host,source,origin_id,tor,request_type,tenant,category,account,subject,destination,setup_time,pdd,answer_time,usage,supplier,disconnect_cause,extra_fields,cost_source,cost,cost_details,extra_info, created_at, updated_at, deleted_at) + SELECT cdrs_primary.cgrid,rated_cdrs.runid as run_id,cdrs_primary.cdrhost as origin_host,cdrs_primary.cdrsource as source,cdrs_primary.accid as origin_id, cdrs_primary.tor,rated_cdrs.reqtype as request_type, rated_cdrs.tenant,rated_cdrs.category, rated_cdrs.account, rated_cdrs.subject, rated_cdrs.destination,rated_cdrs.setup_time,rated_cdrs.pdd,rated_cdrs.answer_time,rated_cdrs.usage,rated_cdrs.supplier,rated_cdrs.disconnect_cause,cdrs_extra.extra_fields,cost_details.cost_source,rated_cdrs.cost,cost_details.timespans as cost_details,rated_cdrs.extra_info,rated_cdrs.created_at,rated_cdrs.updated_at, rated_cdrs.deleted_at + FROM rated_cdrs + INNER JOIN cdrs_primary ON rated_cdrs.cgrid = cdrs_primary.cgrid + INNER JOIN cdrs_extra ON rated_cdrs.cgrid = cdrs_extra.cgrid + INNER JOIN cost_details ON rated_cdrs.cgrid = cost_details.cgrid + WHERE cdrs_primary.usage > '0' + AND not exists (select 1 from cdrs c where c.cgrid = cdrs_primary.cgrid) + AND rated_cdrs.id >= start_id + AND rated_cdrs.id < end_id + ; + start_id = start_id + step; + end_id = end_id + step; + END LOOP; +END +$$; diff --git a/data/docker/integration/scripts/postgres/setup_cgr_db.sh b/data/docker/integration/scripts/postgres/setup_cgr_db.sh new file mode 100755 index 000000000..66d011763 --- /dev/null +++ b/data/docker/integration/scripts/postgres/setup_cgr_db.sh @@ -0,0 +1,30 @@ +#! /usr/bin/env sh + + +user=$1 +if [ -z "$1" ]; then + user="cgrates" +fi + +host=$2 +if [ -z "$2" ]; then + host="localhost" +fi + +DIR="$(dirname "$(readlink -f "$0")")" + +"$DIR"/create_db_with_users.sh + +export PGPASSWORD="CGRateS.org" + +psql -U $user -h $host -d cgrates -f "$DIR"/create_cdrs_tables.sql +cdrt=$? +psql -U $user -h $host -d cgrates -f "$DIR"/create_tariffplan_tables.sql +tpt=$? + +if [ $cdrt = 0 ] && [ $tpt = 0 ]; then + echo "\n\t+++ CGR-DB successfully set-up! +++\n" + exit 0 +fi + + diff --git a/data/docker/integration/scripts/service b/data/docker/integration/scripts/service new file mode 100755 index 000000000..f15921848 --- /dev/null +++ b/data/docker/integration/scripts/service @@ -0,0 +1,29 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then + exit 1 +fi + +if [ $1 != "rabbitmq-server" ]; then + exit 1 +fi + +case "$2" in + "restart") + rabbitmqctl stop >/logs/rabbitmq.log 2>&1 + rabbitmq-server >/logs/rabbitmq.log 2>&1 & + sleep 5s + echo "Done restart" + exit 0;; + "start") + rabbitmq-server >/logs/rabbitmq.log 2>&1 & + sleep 5s + echo "Done start" + exit 0;; + "stop") + rabbitmqctl stop >/logs/rabbitmq.log 2>&1 + echo "Done stop" + exit 0;; + *) + exit 1;; +esac diff --git a/recipients b/recipients new file mode 100644 index 000000000..a8c609354 --- /dev/null +++ b/recipients @@ -0,0 +1 @@ +arber.katellari@itsyscom.com \ No newline at end of file