CI: redo Travis (unit) testing using Docker images

Author: kares

.travis.yml

@@ -1,21 +1,16 @@
 ---
-sudo: false
-language: ruby
-cache: bundler
+sudo: required
+language: minimal
 matrix:
   include:
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=master
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=7.0
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=6.7
-  - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=6.6
-  - rvm: jruby-1.7.27
-    env: LOGSTASH_BRANCH=5.6
+  - env: ELASTIC_STACK_VERSION=5.x
+  - env: ELASTIC_STACK_VERSION=6.x
+  - env: ELASTIC_STACK_VERSION=7.x
+  - env: ELASTIC_STACK_VERSION=7.x SNAPSHOT=true
+  - env: ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
   fast_finish: true
-install: true
-script: ci/build.sh
-jdk: openjdk8
-before_install: gem install bundler -v '< 2'
+#allow_failures:
+#  - env: ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
+#  - env: ELASTIC_STACK_VERSION=7.x SNAPSHOT=true
+install: ci/unit/docker-setup.sh
+script: ci/unit/docker-run.sh

ci/Dockerfile

@@ -0,0 +1,18 @@
+ARG ELASTIC_STACK_VERSION
+FROM docker.elastic.co/logstash/logstash:$ELASTIC_STACK_VERSION
+COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile
+COPY --chown=logstash:logstash *.gemspec /usr/share/plugins/plugin/
+RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml
+ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin"
+ENV LOGSTASH_SOURCE="1"
+ENV ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION
+# DISTRIBUTION="default" (by default) or "oss"
+ARG DISTRIBUTION
+ENV DISTRIBUTION=$DISTRIBUTION
+# INTEGRATION="true" while integration testing (false-y by default)
+ARG INTEGRATION
+ENV INTEGRATION=$INTEGRATION
+RUN gem install bundler -v '< 2'
+WORKDIR /usr/share/plugins/plugin
+RUN bundle install --with test ci
+COPY --chown=logstash:logstash . /usr/share/plugins/plugin

ci/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# This is intended to be run inside the docker container as the command of the docker-compose.
+set -ex
+
+CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+docker-compose -f $CURRENT_DIR/docker-compose.yml up --exit-code-from logstash

ci/docker-run.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh`
+# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable.
+set -e
+
+VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/master/ci/logstash_releases.json"
+
+if [ -z "${ELASTIC_STACK_VERSION}" ]; then
+    echo "Please set the ELASTIC_STACK_VERSION environment variable"
+    echo "For example: export ELASTIC_STACK_VERSION=7.x"
+    exit 1
+fi
+
+echo "Fetching versions from $VERSION_URL"
+VERSIONS=$(curl $VERSION_URL)
+
+if [[ "$SNAPSHOT" = "true" ]]; then
+  ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"')
+  echo $ELASTIC_STACK_RETRIEVED_VERSION
+else
+  ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"')
+fi
+
+if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then
+  # remove starting and trailing double quotes
+  ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}"
+  ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}"
+  echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}"
+  export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION
+fi
+
+if [[ "$DISTRIBUTION" = "oss" ]]; then
+  DISTRIBUTION_SUFFIX="-oss"
+else
+  DISTRIBUTION_SUFFIX=""
+fi
+
+echo "Testing against version: $ELASTIC_STACK_VERSION (distribution: ${DISTRIBUTION:-"default"})"
+
+if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then
+    cd /tmp
+
+    jq=".build.projects.\"logstash\".packages.\"logstash$DISTRIBUTION_SUFFIX-$ELASTIC_STACK_VERSION-docker-image.tar.gz\".url"
+    result=$(curl --silent https://artifacts-api.elastic.co/v1/versions/$ELASTIC_STACK_VERSION/builds/latest | jq -r $jq)
+    echo $result
+    curl $result > logstash-docker-image.tar.gz
+    tar xfvz logstash-docker-image.tar.gz repositories
+    echo "Loading docker image: "
+    cat repositories
+    docker load < logstash-docker-image.tar.gz
+    rm logstash-docker-image.tar.gz
+    cd -
+fi
+
+if [ -f Gemfile.lock ]; then
+    rm Gemfile.lock
+fi
+
+CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") # e.g. "ci/unit"
+
+docker-compose -f "$CURRENT_DIR/docker-compose.yml" down
+docker-compose -f "$CURRENT_DIR/docker-compose.yml" build logstash
+
+#docker-compose -f "$CURRENT_DIR/docker-compose.yml" up --exit-code-from logstash --force-recreate

ci/docker-setup.sh

@@ -0,0 +1 @@
+../Dockerfile

ci/setup.sh

@@ -0,0 +1,19 @@
+version: '3'
+
+# run tests:  cd ci/unit; docker-compose up --build --force-recreate
+# manual:  cd ci/unit; docker-compose run logstash bash
+services:
+
+  logstash:
+    build:
+      context: ../../
+      dockerfile: ci/unit/Dockerfile
+      args:
+        - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION
+        - DISTRIBUTION=${DISTRIBUTION:-default}
+        #- INTEGRATION=false
+    command: jruby -rbundler/setup -S rspec -fd
+    env_file: docker.env
+    environment:
+      - SPEC_OPTS
+    tty: true

ci/unit/Dockerfile

@@ -0,0 +1 @@
+../docker-run.sh

ci/unit/docker-compose.yml

@@ -0,0 +1 @@
+../docker-setup.sh

ci/unit/docker-run.sh

@@ -0,0 +1 @@
+LS_JAVA_OPTS="-Xms256m -Xmx256m -XX:MaxMetaspaceSize=256m"

ci/unit/docker-setup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# This is intended to be run inside the docker container as the command of the docker-compose.
+set -ex
+
+jruby -rbundler/setup -S rspec -fd