Updated drone ansible script

data/ansible/drone/.env.j2

@@ -1,11 +1,12 @@
 DRONE_GITHUB_CLIENT_ID={{ drone_github_client_id }}
 DRONE_GITHUB_CLIENT_SECRET={{ drone_github_client_secret }}
-DRONE_SERVER_PROTO={{ drone_server_protocol }}
+DRONE_SERVER_PROTO=http
+DRONE_SERVER_PROXY_PROTO=https
 DRONE_SERVER_HOST={{ drone_server_host }}
 DRONE_VALIDATE_PLUGIN_ENDPOINT=http://{{ drone_validate_plugin_endpoint }}
 DRONE_VALIDATE_PLUGIN_SECRET={{ drone_secret.stdout }} 
 DRONE_RPC_SECRET={{ drone_secret.stdout }} 
 DRONE_USER_CREATE=username:{{ drone_user_create }},admin:true,token:{{ drone_token.stdout }}
-DRONE_USER_FILTER={{ drone_user_filter }}
-DRONE_TLS_AUTOCERT=true
+DRONE_USER_FILTER={{ drone_user_filter | join(',') }}
+DRONE_SERVER_PORT=":80"
 DRONE_LOGS_DEBUG=true 

data/ansible/drone/main.yaml

@@ -1,6 +1,6 @@
 ---
 - name: Check and set python version on APT server
-  hosts: local
+  hosts: ci
   remote_user: '{{ user }}'
   gather_facts: false
   tasks:
@@ -11,7 +11,7 @@
         fi
       become: true
 
-- hosts: local
+- hosts: ci
   vars:
     ###############################################################
     ##################### Golang Vars #############################
@@ -35,13 +35,18 @@
     #################################################################
     drone_dir: "/home/{{ user }}/go/src/github.com/drone/drone"
 
-    drone_server_protocol: https
-    drone_server_host: ci.cgrates.org
+    drone_server_host: ci.cgrates.net
     drone_validate_plugin_endpoint: 127.0.0.1:3001
     docker_runner_capacity: 20
 
     drone_user_create: cgrates
-    drone_user_filter: cgrates,danbogos,TeoV,Trial97,porosnicuadrian,andronache98
+    drone_user_filter: 
+      - cgrates
+      - danbogos
+      - TeoV
+      - Trial97
+      - porosnicuadrian
+      - andronache98
     ###############################################################
     # CGRateS vars
     dependencies:
@@ -263,7 +268,7 @@
 
     - name:  Start drone-runner
       become: yes
-      shell: docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e DRONE_RPC_PROTO="{{ drone_server_protocol }}" -e DRONE_RPC_HOST="{{ drone_server_host }}" -e DRONE_RPC_SECRET="{{ drone_secret.stdout }}" -e DRONE_RUNNER_CAPACITY={{ docker_runner_capacity }} -e DRONE_RUNNER_NAME=cgrates -p 3000:3000 --restart always --name runner drone/drone-runner-docker:1
+      shell: docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e DRONE_RPC_PROTO="https" -e DRONE_RPC_HOST="{{ drone_server_host }}" -e DRONE_RPC_SECRET="{{ drone_secret.stdout }}" -e DRONE_RUNNER_CAPACITY={{ docker_runner_capacity }} -e DRONE_RUNNER_NAME=cgrates -p 3000:3000 --restart always --name runner drone/drone-runner-docker:1
       when: continerList2.stdout_lines|length == 0
 
     - name: Creating credentials file
@@ -279,12 +284,20 @@
     - name:  add secrets
       become: yes
       shell: |
-        drone-cli orgsecret add cgrates credentials @/home/{{ user }}/credentials.json --allow-pull-request;
-        drone-cli orgsecret add cgrates token @/home/{{ user }}/token.json --allow-pull-request;
-        rm /home/{{ user }}/credentials.json;
-        rm /home/{{ user }}/token.json; 
+        drone-cli orgsecret add {{ item }} credentials @/home/{{ user }}/credentials.json --allow-pull-request;
+        drone-cli orgsecret add {{ item }} token @/home/{{ user }}/token.json --allow-pull-request;
       environment:
         PATH: "{{ lookup('env','PATH') }}:{{ golang_gopath }}/bin:/usr/local/go/bin:{{ ansible_env.PATH }}"
-        DRONE_SERVER: "{{ drone_server_protocol }}://{{ drone_server_host }}"
+        DRONE_SERVER: "https://{{ drone_server_host }}"
         DRONE_TOKEN: "{{ drone_token.stdout }}"
+      with_items: '{{ drone_user_filter }}'
 
+    - name: Removeing credentials file
+      file:
+        path: "/home/{{ user }}/credentials.json"
+        state: absent
+    
+    - name: Removeing token file 
+      file:
+        path: "/home/{{ user }}/token.json"
+        state: absent

data/docker/integration/Dockerfile

@@ -10,7 +10,7 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selectio
 
 # Install necessary libs
 RUN apt-get update 
-RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk-headless neovim
+RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk-headless neovim net-tools
 
 RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add -
 RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list

data/docker/integration/conf/server.properties

@@ -39,10 +39,10 @@ broker.id=0
 #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
 
 # The number of threads that the server uses for receiving requests from the network and sending responses to the network
-num.network.threads=3
+num.network.threads=1
 
 # The number of threads that the server uses for processing requests, which may include disk I/O
-num.io.threads=8
+num.io.threads=1
 
 # The send buffer (SO_SNDBUF) used by the socket server
 socket.send.buffer.bytes=102400
@@ -51,7 +51,7 @@ socket.send.buffer.bytes=102400
 socket.receive.buffer.bytes=102400
 
 # The maximum size of a request that the socket server will accept (protection against OOM)
-socket.request.max.bytes=104857600
+socket.request.max.bytes=50000000
 
 
 ############################# Log Basics #############################
@@ -100,14 +100,14 @@ transaction.state.log.min.isr=1
 # from the end of the log.
 
 # The minimum age of a log file to be eligible for deletion due to age
-log.retention.hours=168
+log.retention.hours=5
 
 # A size-based retention policy for logs. Segments are pruned from the log unless the remaining
 # segments drop below log.retention.bytes. Functions independently of log.retention.hours.
-#log.retention.bytes=1073741824
+log.retention.bytes=20971520
 
 # The maximum size of a log segment file. When this size is reached a new log segment will be created.
-log.segment.bytes=1073741824
+log.segment.bytes=20971520
 
 # The interval at which log segments are checked to see if they can be deleted according
 # to the retention policies

data/docker/integration/docker-entrypoint.sh

@@ -2,6 +2,7 @@
 set -ev
 
 # start basic subsystems
+export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M"
 /kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties
 /kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties
 
@@ -13,6 +14,26 @@ MYSQL_ROOT_PASSWORD="CGRateS.org" /scripts/mariadb-ep.sh mysqld > /logs/mariadb_
 rabbitmq-server > /logs/rabbitmq.log  2>&1 &
 
 
+START_TIMEOUT=600
+
+start_timeout_exceeded=false
+count=0
+step=10
+while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do
+    echo "waiting for kafka to be ready"
+    sleep $step;
+    count=$((count + step))
+    if [ $count -gt $START_TIMEOUT ]; then
+        start_timeout_exceeded=true
+        break
+    fi
+done
+
+if $start_timeout_exceeded; then
+    echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
+    exit 1
+fi
+
 /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates
 /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs