Renew Drone CI

2026-02-11 18:16:24 +05:00 · 2023-09-01 10:07:44 -04:00
parent 47af22c724
commit c04e28cb56
5 changed files with 152 additions and 82 deletions
--- a/.drone.yaml
+++ b/.drone.yaml
@@ -2,22 +2,22 @@ kind: pipeline
 type: docker
 name: unit

-
 workspace:
-  path: /cgrates
+  path: /go/src/github.com/cgrates/cgrates/

 steps:
 - name: unit
  pull: never
-  image: cgrates-it-entrypoint
+  image: cgrates-integration
  commands:
-  - docker-entrypoint.sh
+  - go version
  - ./test.sh
 - name: notify
  pull: never
  image: drillster/drone-email
  settings:
-    from: cgrates.test01@gmail.com
+    from.address: cgrates.dronebot@gmail.com
+    from.name: DroneEmail
    host: smtp.gmail.com
    username: 
      from_secret: email_username
@@ -48,20 +48,22 @@ name: integration-internal


 workspace:
-  path: /cgrates
+  path: /go/src/github.com/cgrates/cgrates/

 steps:
 - name: integration-internal
  pull: never
-  image: cgrates-it-entrypoint
+  image: cgrates-integration
  commands:
-  - docker-entrypoint.sh
+  - data/docker/integration/docker-entrypoint.sh
  - ./integration_test.sh -dbtype=*internal
+  failure: ignore
 - name: notify
  pull: never
  image: drillster/drone-email
  settings:
-    from: cgrates.test01@gmail.com
+    from.address: cgrates.dronebot@gmail.com
+    from.name: DroneEmail
    host: smtp.gmail.com
    username: 
      from_secret: email_username
@@ -83,7 +85,7 @@ steps:
    recipients_file: recipients
    recipients_only: true
  when:
-    status: failure 
+    status: failure

 ---
 kind: pipeline
@@ -92,20 +94,22 @@ name: integration-mysql


 workspace:
-  path: /cgrates
+  path: /go/src/github.com/cgrates/cgrates/

 steps:
 - name: integration-mysql
  pull: never
-  image: cgrates-it-entrypoint
+  image: cgrates-integration
  commands:
-  - docker-entrypoint.sh
+  - data/docker/integration/docker-entrypoint.sh
  - ./integration_test.sh -dbtype=*mysql
+  failure: ignore
 - name: notify
  pull: never
  image: drillster/drone-email
  settings:
-    from: cgrates.test01@gmail.com
+    from.address: cgrates.dronebot@gmail.com
+    from.name: DroneEmail
    host: smtp.gmail.com
    username: 
      from_secret: email_username
@@ -136,20 +140,22 @@ name: integration-mongo


 workspace:
-  path: /cgrates
+  path: /go/src/github.com/cgrates/cgrates/

 steps:
 - name: integration-mongo
  pull: never
-  image: cgrates-it-entrypoint
+  image: cgrates-integration
  commands:
-  - docker-entrypoint.sh
+  - data/docker/integration/docker-entrypoint.sh
  - ./integration_test.sh -dbtype=*mongo
+  failure: ignore
 - name: notify
  pull: never
  image: drillster/drone-email
  settings:
-    from: cgrates.test01@gmail.com
+    from.address: cgrates.dronebot@gmail.com
+    from.name: DroneEmail
    host: smtp.gmail.com
    username: 
      from_secret: email_username
@@ -180,20 +186,22 @@ name: integration-postgres


 workspace:
-  path: /cgrates
+  path: /go/src/github.com/cgrates/cgrates/

 steps:
 - name: integration-postgres
  pull: never
-  image: cgrates-it-entrypoint
+  image: cgrates-integration
  commands:
-  - docker-entrypoint.sh
+  - data/docker/integration/docker-entrypoint.sh
  - ./integration_test.sh -dbtype=*postgres
+  failure: ignore
 - name: notify
  pull: never
  image: drillster/drone-email
  settings:
-    from: cgrates.test01@gmail.com
+    from.address: cgrates.dronebot@gmail.com
+    from.name: DroneEmail
    host: smtp.gmail.com
    username: 
      from_secret: email_username
@@ -215,4 +223,4 @@ steps:
    recipients_file: recipients
    recipients_only: true
  when:
-    status: failure
+    status: failure
--- a/data/docker/integration/Dockerfile
+++ b/data/docker/integration/Dockerfile
@@ -1,4 +1,4 @@
-FROM  debian:buster-slim
+FROM  ubuntu:jammy

 RUN groupadd -r mongodb && useradd -r -g mongodb mongodb
 RUN groupadd -r redis && useradd -r -g redis redis 
@@ -6,27 +6,26 @@ RUN groupadd -r kafka && useradd -r -g kafka kafka

 RUN mkdir -p /usr/share/cgrates/conf /usr/share/man/man1 /var/spool/cgrates/ers/in /var/spool/cgrates/ers/out /var/spool/cgrates/cdre/csv /var/spool/cgrates/cdre/fwv /var/spool/cgrates/tpe /var/spool/cgrates/failed_posts /var/spool/cgrates/analyzers /run /data/configdb /data/db /kafka /logs

-RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

-# Install necessary libs
-RUN apt-get update 
+ENV DEBIAN_FRONTEND=noninteractive
+
+
+RUN apt-get update && apt-get install -y ansible
+
+# Install Erlang packages
+RUN apt-get install -y erlang-base erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key erlang-runtime-tools erlang-snmp erlang-ssl erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl
+
 RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk-headless neovim net-tools

-RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add -
-RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list
+RUN curl -fsSL https://pgp.mongodb.com/server-7.0.asc | gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg  --dearmor
+RUN echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list

-RUN wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
-
-RUN wget https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
-RUN dpkg -i erlang-solutions_1.0_all.deb
-RUN apt-get update
-RUN apt-get install -y erlang erlang-nox
-RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash
+RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc |  apt-key add -

 RUN apt-get update
-RUN apt-get install -y mongodb-org postgresql rabbitmq-server
-
+RUN apt-get install -y mongodb-org postgresql
+# ADD THIS AFTER postgresql rabbitmq-server

 WORKDIR /kafka

@@ -36,7 +35,6 @@ RUN rm kafka_2.11-2.1.0.tgz
 COPY ./conf/server.properties /kafka/config/server.properties
 WORKDIR /

-
 RUN set -ex; \
 	rm -rf /var/lib/apt/lists/*; \
 # purge and re-create /var/lib/mysql with appropriate ownership
@@ -52,8 +50,18 @@ RUN set -ex; \
 # don't reverse lookup hostnames, they are usually another container
 	echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf

+# Clean up
+RUN apt-get clean && \
+    rm -rf /var/lib/apt/lists/*

-RUN wget -O go.tgz "https://storage.googleapis.com/golang/go1.18.linux-amd64.tar.gz" --progress=dot:giga
+RUN touch /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log 
+RUN chmod 777  /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log 
+
+COPY main.yaml /integration_tests.yaml
+
+COPY roles /roles
+
+RUN wget -O go.tgz "https://storage.googleapis.com/golang/go1.21.0.linux-amd64.tar.gz" --progress=dot:giga
 RUN tar -C /usr/local -xzf go.tgz
 RUN rm go.tgz

@@ -62,11 +70,6 @@ ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
 RUN	go version

-
-RUN touch /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log 
-RUN chmod 777  /logs/mariadb.log /logs/mariadb_script.log /logs/rabbitmq.log 
-
-
 COPY ./scripts /scripts
 COPY ./scripts/service /usr/local/bin/service

--- a/data/docker/integration/docker-entrypoint.sh
+++ b/data/docker/integration/docker-entrypoint.sh
@@ -1,60 +1,73 @@
 #!/bin/bash
 set -ev

+ansible-playbook /integration_tests.yaml -i localhost, -c local -e "ansible_user=root clone_repository=true" -vvv
+
+# # Create symbolic links
+ln -s "/go/src/github.com/cgrates/cgrates/data" "/usr/share/cgrates"
+ln -s "/go/bin/cgr-engine" "/usr/bin/cgr-engine"
+ln -s "/go/bin/cgr-loader" "/usr/bin/cgr-loader"
+ln -s "/go/bin/cgr-migrator" "/usr/bin/cgr-migrator"
+ln -s "/go/bin/cgr-console" "/usr/bin/cgr-console"
+ln -s "/go/bin/cgr-tester" "/usr/bin/cgr-tester"
+
 # start basic subsystems
-export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M"
-/kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties
-/kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties
+# export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M"
+# /kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties
+# /kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties

 rsyslogd -f /etc/rsyslogd.conf 
 version=$(ls /var/lib/postgresql)
 pg_ctlcluster $version main start &
 mongod --bind_ip 127.0.0.1  --logpath /logs/mongodb.log &
 redis-server /etc/redis/redis.conf &
-MYSQL_ROOT_PASSWORD="CGRateS.org" /scripts/mariadb-ep.sh mysqld > /logs/mariadb_script.log 2>&1
-rabbitmq-server > /logs/rabbitmq.log  2>&1 &
+MYSQL_ROOT_PASSWORD="CGRateS.org" /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mariadb-ep.sh mysqld
+# rabbitmq-server > /logs/rabbitmq.log  2>&1 &


-START_TIMEOUT=600
+# START_TIMEOUT=600

-start_timeout_exceeded=false
-count=0
-step=10
-while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do
-    echo "waiting for kafka to be ready"
-    sleep $step;
-    count=$((count + step))
-    if [ $count -gt $START_TIMEOUT ]; then
-        start_timeout_exceeded=true
-        break
-    fi
-done
+# start_timeout_exceeded=false
+# count=0
+# step=10
+# while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do
+#     echo "waiting for kafka to be ready"
+#     sleep $step;
+#     count=$((count + step))
+#     if [ $count -gt $START_TIMEOUT ]; then
+#         start_timeout_exceeded=true
+#         break
+#     fi
+# done

-if $start_timeout_exceeded; then
-    echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
-    exit 1
-fi
+# if $start_timeout_exceeded; then
+#     echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
+#     exit 1
+# fi

-/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates
-/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs
+# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates
+# /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs



-gosu postgres psql  -c "CREATE USER cgrates password 'CGRateS.org';"  > /dev/null 2>&1
-gosu postgres createdb -e -O cgrates cgrates > /dev/null 2>&1
+# gosu postgres psql  -c "CREATE USER cgrates password 'CGRateS.org';"  > /dev/null 2>&1
+# gosu postgres createdb -e -O cgrates cgrates > /dev/null 2>&1



-PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /scripts/postgres/create_cdrs_tables.sql >/dev/null 2>&1
-PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /scripts/postgres/create_tariffplan_tables.sql >/dev/null 2>&1
+# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_cdrs_tables.sql >/dev/null 2>&1
+# PGPASSWORD="CGRateS.org" psql -U "cgrates" -h "localhost" -d cgrates -f /cgrates/data/docker/integration/scripts/postgres/create_tariffplan_tables.sql >/dev/null 2>&1


-mongo --quiet /scripts/create_user.js >/dev/null 2>&1
+# mongo --quiet /cgrates/data/docker/integration/scripts/create_user.js >/dev/null 2>&1
+echo "Starting..."
+sleep 60  # Pause for 1 min
+echo "Resuming..."
+mysql -u root -pCGRateS.org -h localhost < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_db_with_users.sql 
+mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_cdrs_tables.sql
+mysql -u root -pCGRateS.org -h localhost -D cgrates < /go/src/github.com/cgrates/cgrates/data/docker/integration/scripts/mysql/create_tariffplan_tables.sql

+cp -r /go/src/github.com/cgrates/cgrates/data/. /usr/share/cgrates

-mysql -u root -pCGRateS.org -h localhost < /scripts/mysql/create_db_with_users.sql > /dev/null 2>&1
-mysql -u root -pCGRateS.org -h localhost < /scripts/mysql/create_db_with_users_extra.sql > /dev/null 2>&1
-mysql -u root -pCGRateS.org -h localhost -D cgrates < /scripts/mysql/create_cdrs_tables.sql > /dev/null 2>&1
-mysql -u root -pCGRateS.org -h localhost -D cgrates < /scripts/mysql/create_tariffplan_tables.sql > /dev/null 2>&1
-
-cp -r data/. /usr/share/cgrates
+# Set versions
+# cgr-migrator -exec=*set_versions -config_path=/usr/share/cgrates/conf/samples/tutmysql
--- a/data/docker/integration/main.yaml
+++ b/data/docker/integration/main.yaml
@@ -0,0 +1,46 @@
+---
+- hosts: all
+  vars:
+    golang_gopath: "/go"
+    clone_repository: true
+    cgrates_dir: "{{ golang_gopath }}/src/github.com/cgrates/cgrates"
+    golang_install_dir: /usr/local/go
+    git_version: "master"
+    cgrates_dependencies:
+      - git
+      - redis-server
+      - mariadb-server
+      - make
+      - gcc
+
+  tasks:
+    - name: Install CGRateS dependencies
+      become: yes
+      ansible.builtin.package:
+        name: "{{ cgrates_dependencies }}"
+        state: present
+        update_cache: yes
+        cache_valid_time: 86400
+
+    - name: Create cgrates directory
+      ansible.builtin.file:
+        state: directory
+        mode: "u=rwx,go=rx"
+        owner: "{{ ansible_user }}"
+        dest: "{{ cgrates_dir }}"
+      when: clone_repository | bool
+
+    - name: Git clone cgrates
+      ansible.builtin.git:
+        repo: https://github.com/cgrates/cgrates.git
+        dest: "{{ cgrates_dir }}"
+        update: yes
+        force: yes
+        version: "{{ git_version }}"
+      when: clone_repository | bool
+
+    - name: Build cgrates
+      ansible.builtin.shell:
+        cmd: bash -lc "sh {{ cgrates_dir }}/build.sh"
+      args:
+        chdir: "{{ cgrates_dir }}"
--- a/2
+++ b/2
@@ -1 +1 @@
-nickolas.filip@itsyscom.com
+arber.katellari@itsyscom.com