diff --git a/data/ansible/drone/.env.j2 b/data/ansible/drone/.env.j2 index 97e0b24fb..010800442 100644 --- a/data/ansible/drone/.env.j2 +++ b/data/ansible/drone/.env.j2 @@ -1,11 +1,12 @@ DRONE_GITHUB_CLIENT_ID={{ drone_github_client_id }} DRONE_GITHUB_CLIENT_SECRET={{ drone_github_client_secret }} -DRONE_SERVER_PROTO={{ drone_server_protocol }} +DRONE_SERVER_PROTO=http +DRONE_SERVER_PROXY_PROTO=https DRONE_SERVER_HOST={{ drone_server_host }} DRONE_VALIDATE_PLUGIN_ENDPOINT=http://{{ drone_validate_plugin_endpoint }} DRONE_VALIDATE_PLUGIN_SECRET={{ drone_secret.stdout }} DRONE_RPC_SECRET={{ drone_secret.stdout }} DRONE_USER_CREATE=username:{{ drone_user_create }},admin:true,token:{{ drone_token.stdout }} -DRONE_USER_FILTER={{ drone_user_filter }} -DRONE_TLS_AUTOCERT=true +DRONE_USER_FILTER={{ drone_user_filter | join(',') }} +DRONE_SERVER_PORT=":80" DRONE_LOGS_DEBUG=true diff --git a/data/ansible/drone/main.yaml b/data/ansible/drone/main.yaml index 16fcbf6c0..fa9fea503 100644 --- a/data/ansible/drone/main.yaml +++ b/data/ansible/drone/main.yaml @@ -1,6 +1,6 @@ --- - name: Check and set python version on APT server - hosts: local + hosts: ci remote_user: '{{ user }}' gather_facts: false tasks: @@ -11,7 +11,7 @@ fi become: true -- hosts: local +- hosts: ci vars: ############################################################### ##################### Golang Vars ############################# @@ -35,13 +35,18 @@ ################################################################# drone_dir: "/home/{{ user }}/go/src/github.com/drone/drone" - drone_server_protocol: https - drone_server_host: ci.cgrates.org + drone_server_host: ci.cgrates.net drone_validate_plugin_endpoint: 127.0.0.1:3001 docker_runner_capacity: 20 drone_user_create: cgrates - drone_user_filter: cgrates,danbogos,TeoV,Trial97,porosnicuadrian,andronache98 + drone_user_filter: + - cgrates + - danbogos + - TeoV + - Trial97 + - porosnicuadrian + - andronache98 ############################################################### # CGRateS vars dependencies: @@ -263,7 +268,7 @@ - name: Start drone-runner become: yes - shell: docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e DRONE_RPC_PROTO="{{ drone_server_protocol }}" -e DRONE_RPC_HOST="{{ drone_server_host }}" -e DRONE_RPC_SECRET="{{ drone_secret.stdout }}" -e DRONE_RUNNER_CAPACITY={{ docker_runner_capacity }} -e DRONE_RUNNER_NAME=cgrates -p 3000:3000 --restart always --name runner drone/drone-runner-docker:1 + shell: docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e DRONE_RPC_PROTO="https" -e DRONE_RPC_HOST="{{ drone_server_host }}" -e DRONE_RPC_SECRET="{{ drone_secret.stdout }}" -e DRONE_RUNNER_CAPACITY={{ docker_runner_capacity }} -e DRONE_RUNNER_NAME=cgrates -p 3000:3000 --restart always --name runner drone/drone-runner-docker:1 when: continerList2.stdout_lines|length == 0 - name: Creating credentials file @@ -279,12 +284,20 @@ - name: add secrets become: yes shell: | - drone-cli orgsecret add cgrates credentials @/home/{{ user }}/credentials.json --allow-pull-request; - drone-cli orgsecret add cgrates token @/home/{{ user }}/token.json --allow-pull-request; - rm /home/{{ user }}/credentials.json; - rm /home/{{ user }}/token.json; + drone-cli orgsecret add {{ item }} credentials @/home/{{ user }}/credentials.json --allow-pull-request; + drone-cli orgsecret add {{ item }} token @/home/{{ user }}/token.json --allow-pull-request; environment: PATH: "{{ lookup('env','PATH') }}:{{ golang_gopath }}/bin:/usr/local/go/bin:{{ ansible_env.PATH }}" - DRONE_SERVER: "{{ drone_server_protocol }}://{{ drone_server_host }}" + DRONE_SERVER: "https://{{ drone_server_host }}" DRONE_TOKEN: "{{ drone_token.stdout }}" + with_items: '{{ drone_user_filter }}' + - name: Removeing credentials file + file: + path: "/home/{{ user }}/credentials.json" + state: absent + + - name: Removeing token file + file: + path: "/home/{{ user }}/token.json" + state: absent \ No newline at end of file diff --git a/data/docker/integration/Dockerfile b/data/docker/integration/Dockerfile index 5155b17f2..7cdc8087d 100644 --- a/data/docker/integration/Dockerfile +++ b/data/docker/integration/Dockerfile @@ -10,7 +10,7 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selectio # Install necessary libs RUN apt-get update -RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk-headless neovim +RUN apt-get install -y apt-utils wget gnupg gnupg2 apt-transport-https curl redis-server git build-essential rsyslog procps gosu "mariadb-server" mariadb-backup socat default-jdk-headless neovim net-tools RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list diff --git a/data/docker/integration/conf/server.properties b/data/docker/integration/conf/server.properties index 49a156219..521fb74a4 100644 --- a/data/docker/integration/conf/server.properties +++ b/data/docker/integration/conf/server.properties @@ -39,10 +39,10 @@ broker.id=0 #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL # The number of threads that the server uses for receiving requests from the network and sending responses to the network -num.network.threads=3 +num.network.threads=1 # The number of threads that the server uses for processing requests, which may include disk I/O -num.io.threads=8 +num.io.threads=1 # The send buffer (SO_SNDBUF) used by the socket server socket.send.buffer.bytes=102400 @@ -51,7 +51,7 @@ socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 # The maximum size of a request that the socket server will accept (protection against OOM) -socket.request.max.bytes=104857600 +socket.request.max.bytes=50000000 ############################# Log Basics ############################# @@ -100,14 +100,14 @@ transaction.state.log.min.isr=1 # from the end of the log. # The minimum age of a log file to be eligible for deletion due to age -log.retention.hours=168 +log.retention.hours=5 # A size-based retention policy for logs. Segments are pruned from the log unless the remaining # segments drop below log.retention.bytes. Functions independently of log.retention.hours. -#log.retention.bytes=1073741824 +log.retention.bytes=20971520 # The maximum size of a log segment file. When this size is reached a new log segment will be created. -log.segment.bytes=1073741824 +log.segment.bytes=20971520 # The interval at which log segments are checked to see if they can be deleted according # to the retention policies diff --git a/data/docker/integration/docker-entrypoint.sh b/data/docker/integration/docker-entrypoint.sh index 469690624..c8b9aa57f 100755 --- a/data/docker/integration/docker-entrypoint.sh +++ b/data/docker/integration/docker-entrypoint.sh @@ -2,6 +2,7 @@ set -ev # start basic subsystems +export KAFKA_HEAP_OPTS="-Xmx100M -Xms100M" /kafka/bin/zookeeper-server-start.sh -daemon /kafka/config/zookeeper.properties /kafka/bin/kafka-server-start.sh -daemon /kafka/config/server.properties @@ -13,6 +14,26 @@ MYSQL_ROOT_PASSWORD="CGRateS.org" /scripts/mariadb-ep.sh mysqld > /logs/mariadb_ rabbitmq-server > /logs/rabbitmq.log 2>&1 & +START_TIMEOUT=600 + +start_timeout_exceeded=false +count=0 +step=10 +while netstat -lnt | awk '$4 ~ /:9092$/ {exit 1}'; do + echo "waiting for kafka to be ready" + sleep $step; + count=$((count + step)) + if [ $count -gt $START_TIMEOUT ]; then + start_timeout_exceeded=true + break + fi +done + +if $start_timeout_exceeded; then + echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)" + exit 1 +fi + /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates /kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic cgrates_cdrs