From c04e28cb569c483dd9fd4cd205ea5ddae8c1bcab Mon Sep 17 00:00:00 2001 From: arberkatellari Date: Fri, 1 Sep 2023 10:07:44 -0400 Subject: [PATCH] Renew Drone CI --- .drone.yaml => .drone.yml | 54 +++++++------ data/docker/integration/Dockerfile | 49 ++++++------ data/docker/integration/docker-entrypoint.sh | 83 +++++++++++--------- data/docker/integration/main.yaml | 46 +++++++++++ recipients | 2 +- 5 files changed, 152 insertions(+), 82 deletions(-) rename .drone.yaml => .drone.yml (81%) create mode 100644 data/docker/integration/main.yaml diff --git a/.drone.yaml b/.drone.yml similarity index 81% rename from .drone.yaml rename to .drone.yml index 66ec1979c..8cb331bfb 100644 --- a/.drone.yaml +++ b/.drone.yml @@ -2,22 +2,22 @@ kind: pipeline type: docker name: unit - workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: unit pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - go version - ./test.sh - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username @@ -48,20 +48,22 @@ name: integration-internal workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: integration-internal pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - data/docker/integration/docker-entrypoint.sh - ./integration_test.sh -dbtype=*internal + failure: ignore - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username @@ -83,7 +85,7 @@ steps: recipients_file: recipients recipients_only: true when: - status: failure + status: failure --- kind: pipeline @@ -92,20 +94,22 @@ name: integration-mysql workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: integration-mysql pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - data/docker/integration/docker-entrypoint.sh - ./integration_test.sh -dbtype=*mysql + failure: ignore - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username @@ -136,20 +140,22 @@ name: integration-mongo workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: integration-mongo pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - data/docker/integration/docker-entrypoint.sh - ./integration_test.sh -dbtype=*mongo + failure: ignore - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username @@ -180,20 +186,22 @@ name: integration-postgres workspace: - path: /cgrates + path: /go/src/github.com/cgrates/cgrates/ steps: - name: integration-postgres pull: never - image: cgrates-it-entrypoint + image: cgrates-integration commands: - - docker-entrypoint.sh + - data/docker/integration/docker-entrypoint.sh - ./integration_test.sh -dbtype=*postgres + failure: ignore - name: notify pull: never image: drillster/drone-email settings: - from: cgrates.test01@gmail.com + from.address: cgrates.dronebot@gmail.com + from.name: DroneEmail host: smtp.gmail.com username: from_secret: email_username @@ -215,4 +223,4 @@ steps: recipients_file: recipients recipients_only: true when: - status: failure + status: failure \ No newline at end of file diff --git a/data/docker/integration/Dockerfile b/data/docker/integration/Dockerfile index 4251dfd6c..f915c73c1 100644 --- a/data/docker/integration/Dockerfile +++ b/data/docker/integration/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster-slim +FROM ubuntu:jammy RUN groupadd -r mongodb && useradd -r -g mongodb mongodb RUN groupadd -r redis && useradd -r -g redis redis @@ -6,27 +6,26 @@ RUN groupadd -r kafka && useradd -r -g kafka kafka RUN mkdir -p /usr/share/cgrates/conf /usr/share/man/man1 /var/spool/cgrates/ers/in /var/spool/cgrates/ers/out /var/spool/cgrates/cdre/csv /var/spool/cgrates/cdre/fwv /var/spool/cgrates/tpe /var/spool/cgrates/failed_posts /var/spool/cgrates/analyzers /run /data/configdb /data/db /kafka /logs -RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections -# Install necessary libs -RUN apt-get update +ENV DEBIAN_FRONTEND=noninteractive + + +RUN apt-get update && apt-get install -y ansible + +# Install Erlang packages +RUN apt-get install -y erlang-base erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key erlang-runtime-tools erlang-snmp erlang-ssl erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl + RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk-headless neovim net-tools -RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - -RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list +RUN curl -fsSL https://pgp.mongodb.com/server-7.0.asc | gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor +RUN echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list -RUN wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list - -RUN wget https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb -RUN dpkg -i erlang-solutions_1.0_all.deb -RUN apt-get update -RUN apt-get install -y erlang erlang-nox -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash +RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN apt-get update -RUN apt-get install -y mongodb-org postgresql rabbitmq-server - +RUN apt-get install -y mongodb-org postgresql +# ADD THIS AFTER postgresql rabbitmq-server WORKDIR /kafka @@ -36,7 +35,6 @@ RUN rm kafka_2.11-2.1.0.tgz COPY ./conf/server.properties /kafka/config/server.properties WORKDIR / - RUN set -ex; \ rm -rf /var/lib/apt/lists/*; \ # purge and re-create /var/lib/mysql with appropriate ownership @@ -52,8 +50,18 @@ RUN set -ex; \ # don't reverse lookup hostnames, they are usually another container echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf +# Clean up +RUN apt-get clean && \ + rm -rf /var/lib/apt/lists/* -RUN wget -O go.tgz "https://storage.googleapis.com/golang/go1.18.linux-amd64.tar.gz" --progress=dot:giga +RUN touch /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log +RUN chmod 777 /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log + +COPY main.yaml /integration_tests.yaml + +COPY roles /roles + +RUN wget -O go.tgz "https://storage.googleapis.com/golang/go1.21.0.linux-amd64.tar.gz" --progress=dot:giga RUN tar -C /usr/local -xzf go.tgz RUN rm go.tgz @@ -62,11 +70,6 @@ ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" RUN go version - -RUN touch /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log -RUN chmod 777 /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log - - COPY ./scripts /scripts COPY ./scripts/service /usr/local/bin/service diff --git a/data/docker/integration/docker-entrypoint.sh b/data/docker/integration/docker-entrypoint.sh index 55a8426c6..8efaa8b0b 100755 --- a/data/docker/integration/docker-entrypoint.sh +++ b/data/docker/integration/docker-entrypoint.sh @@ -1,60 +1,73 @@ #!/bin/bash set -ev +ansible-playbook /integration_tests.yaml -i localhost, -c local -e "ansible_user=root clone_repository=true" -vvv + +# # Create symbolic links +ln -s "/go/src/github.com/cgrates/cgrates/data" "/usr/share/cgrates" +ln -s "/go/bin/cgr-engine" "/usr/bin/cgr-engine" +ln -s "/go/bin/cgr-loader" "/usr/bin/cgr-loader" +ln -s "/go/bin/cgr-migrator" "/usr/bin/cgr-migrator" +ln -s "/go/bin/cgr-console" "/usr/bin/cgr-console" +ln -s "/go/bin/cgr-tester" "/usr/bin/cgr-tester" + # start basic subsystems -export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M" -/kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties -/kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties +# export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M" +# /kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties +# /kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties rsyslogd -f /etc/rsyslogd.conf version=$(ls /var/lib/postgresql) pg_ctlcluster $version main start & mongod --bind_ip 127.0.0.1 --logpath /logs/mongodb.log & redis-server /etc/redis/redis.conf & -MYSQL_ROOT_PASSWORD="CGRateS.org" /scripts/mariadb-ep.sh mysqld > /logs/mariadb_script.log 2>&1 -rabbitmq-server > /logs/rabbitmq.log 2>&1 & +MYSQL_ROOT_PASSWORD="CGRateS.org" /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mariadb-ep.sh mysqld +# rabbitmq-server > /logs/rabbitmq.log 2>&1 & -START_TIMEOUT=600 +# START_TIMEOUT=600 -start_timeout_exceeded=false -count=0 -step=10 -while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do - echo "waiting for kafka to be ready" - sleep $step; - count=$((count + step)) - if [ $count -gt $START_TIMEOUT ]; then - start_timeout_exceeded=true - break - fi -done +# start_timeout_exceeded=false +# count=0 +# step=10 +# while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do +# echo "waiting for kafka to be ready" +# sleep $step; +# count=$((count + step)) +# if [ $count -gt $START_TIMEOUT ]; then +# start_timeout_exceeded=true +# break +# fi +# done -if $start_timeout_exceeded; then - echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" - exit 1 -fi +# if $start_timeout_exceeded; then +# echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" +# exit 1 +# fi -/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates -/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs +# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates +# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs -gosu postgres psql -c "CREATE USER cgrates password 'CGRateS.org';" > /dev/null 2>&1 -gosu postgres createdb -e -O cgrates cgrates > /dev/null 2>&1 +# gosu postgres psql -c "CREATE USER cgrates password 'CGRateS.org';" > /dev/null 2>&1 +# gosu postgres createdb -e -O cgrates cgrates > /dev/null 2>&1 -PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /scripts/postgres/create_cdrs_tables.sql >/dev/null 2>&1 -PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /scripts/postgres/create_tariffplan_tables.sql >/dev/null 2>&1 +# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_cdrs_tables.sql >/dev/null 2>&1 +# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql >/dev/null 2>&1 -mongo --quiet /scripts/create_user.js >/dev/null 2>&1 +# mongo --quiet /cgrates/data/docker/integration/scripts/create_user.js >/dev/null 2>&1 +echo "Starting..." +sleep 60 # Pause for 1 min +echo "Resuming..." +mysql -u root -pCGRateS.org -h localhost < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_db_with_users.sql +mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_cdrs_tables.sql +mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql +cp -r /go/src/github.com/cgrates/cgrates/data/. /usr/share/cgrates -mysql -u root -pCGRateS.org -h localhost < /scripts/mysql/create_db_with_users.sql > /dev/null 2>&1 -mysql -u root -pCGRateS.org -h localhost < /scripts/mysql/create_db_with_users_extra.sql > /dev/null 2>&1 -mysql -u root -pCGRateS.org -h localhost -D cgrates < /scripts/mysql/create_cdrs_tables.sql > /dev/null 2>&1 -mysql -u root -pCGRateS.org -h localhost -D cgrates < /scripts/mysql/create_tariffplan_tables.sql > /dev/null 2>&1 - -cp -r data/. /usr/share/cgrates \ No newline at end of file +# Set versions +# cgr-migrator -exec=*set_versions -config_path=/usr/share/cgrates/conf/samples/tutmysql \ No newline at end of file diff --git a/data/docker/integration/main.yaml b/data/docker/integration/main.yaml new file mode 100644 index 000000000..f826d1386 --- /dev/null +++ b/data/docker/integration/main.yaml @@ -0,0 +1,46 @@ +--- +- hosts: all + vars: + golang_gopath: "/go" + clone_repository: true + cgrates_dir: "{{ golang_gopath }}/src/github.com/cgrates/cgrates" + golang_install_dir: /usr/local/go + git_version: "master" + cgrates_dependencies: + - git + - redis-server + - mariadb-server + - make + - gcc + + tasks: + - name: Install CGRateS dependencies + become: yes + ansible.builtin.package: + name: "{{ cgrates_dependencies }}" + state: present + update_cache: yes + cache_valid_time: 86400 + + - name: Create cgrates directory + ansible.builtin.file: + state: directory + mode: "u=rwx,go=rx" + owner: "{{ ansible_user }}" + dest: "{{ cgrates_dir }}" + when: clone_repository | bool + + - name: Git clone cgrates + ansible.builtin.git: + repo: https://github.com/cgrates/cgrates.git + dest: "{{ cgrates_dir }}" + update: yes + force: yes + version: "{{ git_version }}" + when: clone_repository | bool + + - name: Build cgrates + ansible.builtin.shell: + cmd: bash -lc "sh {{ cgrates_dir }}/build.sh" + args: + chdir: "{{ cgrates_dir }}" diff --git a/recipients b/recipients index 8a8ab6c60..a8c609354 100644 --- a/recipients +++ b/recipients @@ -1 +1 @@ -nickolas.filip@itsyscom.com +arber.katellari@itsyscom.com \ No newline at end of file