Skip to content

Commit 9f02f7e

Browse files
geoandiocanel
geoand
authored and
iocanel
committed
Add integration tests
These tests are small Spring Cloud Kubernetes application that are deployed inside a Kubernetes cluster launched with microk8s. The deployment of the applications is done with the Fabric8 Maven Plugin
1 parent 9504dc5 commit 9f02f7e

File tree

26 files changed

+791
-5
lines changed

26 files changed

+791
-5
lines changed

‎.circleci/config.yml

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ jobs:
2020
branches:
2121
ignore:
2222
- gh-pages
23-
docker:
24-
- image: circleci/openjdk:8-jdk-node-browsers
23+
machine: true
2524
environment:
2625
_JAVA_OPTIONS: "-Xms1024m -Xmx2048m"
2726
_SERVICE_OCCURENCE: 5
@@ -41,18 +40,61 @@ jobs:
4140
- ~/.m2
4241
key: spring-cloud-kubernetes-{{ .Branch }}-{{ checksum "pom.xml" }}
4342
- run:
44-
name: test
43+
name: Run regular tests
4544
command: |
4645
./mvnw -s .settings.xml clean install -Dservice.occurence=${_SERVICE_OCCURENCE} #org.jacoco:jacoco-maven-plugin:prepare-agent install -U -P sonar -nsu --batch-mode -Dmaven.test.redirectTestOutputToFile=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
4746
mkdir -p $HOME/artifacts/junit/
4847
find . -type f -regex ".*/spring-cloud-*.*/target/*.*" -exec cp {} $HOME/artifacts/ \;
4948
find . -type f -regex ".*/target/.*-reports/.*" -exec cp {} $HOME/artifacts/junit/ \;
5049
bash <(curl -s https://codecov.io/bash)
50+
- run:
51+
name: Install Snap
52+
command: |
53+
sudo apt update
54+
sudo apt install snapd
55+
- run:
56+
name: Launch Kubernetes with microk8s
57+
command: |
58+
sudo snap install microk8s --classic --channel=1.11/stable
59+
60+
# wait until a k8s node is ready
61+
sleep 10
62+
n=0
63+
until [ $n -ge 10 ]
64+
do
65+
(/snap/bin/microk8s.kubectl get no | grep -z "Ready") && break
66+
n=$[$n+1]
67+
sleep 20
68+
done
69+
70+
echo "Kubernetes cluster launched"
71+
72+
# Allow intra-pod communication
73+
sudo iptables -P FORWARD ACCEPT
74+
75+
/snap/bin/microk8s.enable dns registry
76+
77+
# wait until the registry is up and running
78+
sleep 10
79+
n=0
80+
until [ $n -ge 10 ]
81+
do
82+
(/snap/bin/microk8s.kubectl get pod --namespace=container-registry | grep -z "Running") && break
83+
n=$[$n+1]
84+
sleep 10
85+
done
86+
87+
echo "Kubernetes Container Registry enabled"
88+
- run:
89+
name: Run integration tests
90+
command: |
91+
cd spring-cloud-kubernetes-integration-tests
92+
/snap/bin/microk8s.kubectl config view --raw > /tmp/kubeconfig
93+
./deploy_test_undeploy_all.sh
5194
- store_test_results:
5295
path: $HOME/artifacts/junit/
5396
- store_artifacts:
5497
path: $HOME/artifacts/
5598
notify:
5699
webhooks:
57100
- url: https://webhooks.gitter.im/e/22e6bb4eb945dd61ba54
58-

‎pom.xml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
<fabric8.maven.plugin.version>3.5.37</fabric8.maven.plugin.version>
7272
<gmavenplus-plugin.version>1.6</gmavenplus-plugin.version>
7373
<groovy.version>2.4.12</groovy.version>
74+
<restassured.version>3.0.2</restassured.version>
7475
<spock-spring.version>1.1-groovy-2.4</spock-spring.version>
7576
</properties>
7677

@@ -86,12 +87,13 @@
8687
<module>spring-cloud-starter-kubernetes-all</module>
8788
<module>spring-cloud-kubernetes-examples</module>
8889
<module>spring-cloud-kubernetes-leader</module>
90+
<module>spring-cloud-kubernetes-integration-tests</module>
8991
<module>docs</module>
9092
</modules>
9193

9294
<dependencyManagement>
9395
<dependencies>
94-
96+
9597
<dependency>
9698
<groupId>org.springframework.cloud</groupId>
9799
<artifactId>spring-cloud-kubernetes-dependencies</artifactId>

‎spring-cloud-kubernetes-integration-tests/README.md

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# Purpose
2+
3+
Demonstrate how integration tests can be run against a local cluster setup with [microk8s](https://microk8s.io/)
4+
The Kubernetes resources necessary to get the test application
5+
onto the cluster are created using the [Fabric8 Maven Plugin](https://maven.fabric8.io/)
6+
7+
# Instructions
8+
9+
## Install microk8s
10+
11+
```bash
12+
sudo snap install microk8s --classic --channel=1.11/stable
13+
sleep 10
14+
microk8s.enable dns registry
15+
```
16+
17+
Ensure everything is running by inspecting the output of:
18+
19+
```bash
20+
microk8s.kubectl get all --all-namespaces
21+
```
22+
23+
It should look something like:
24+
25+
```
26+
NAMESPACE NAME READY STATUS RESTARTS AGE
27+
container-registry pod/registry-6bc95dfd76-274lc 1/1 Running 0 40s
28+
kube-system pod/hostpath-provisioner-9979c7f64-f96tw 1/1 Running 0 41s
29+
kube-system pod/kube-dns-864b8bdc77-68kmn 2/3 Running 0 41s
30+
31+
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
32+
container-registry service/registry NodePort 10.152.183.175 <none> 5000:32000/TCP 50s
33+
default service/kubernetes ClusterIP 10.152.183.1 <none> 443/TCP 1m
34+
kube-system service/kube-dns ClusterIP 10.152.183.10 <none> 53/UDP,53/TCP 56s
35+
36+
NAMESPACE NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
37+
container-registry deployment.apps/registry 1 1 1 1 50s
38+
kube-system deployment.apps/hostpath-provisioner 1 1 1 1 51s
39+
kube-system deployment.apps/kube-dns 1 1 1 0 56s
40+
41+
NAMESPACE NAME DESIRED CURRENT READY AGE
42+
container-registry replicaset.apps/registry-6bc95dfd76 1 1 1 41s
43+
kube-system replicaset.apps/hostpath-provisioner-9979c7f64 1 1 1 41s
44+
kube-system replicaset.apps/kube-dns-864b8bdc77 1 1 0 41s
45+
```
46+
47+
## Setup environment for FMP to work
48+
49+
One premise for the Fabric8 Maven Plugin to work is that is can read the proper Kubernetes Config file.
50+
We export the corresponding config file for the cluster microk8s sets up to temp file which will
51+
be used later
52+
53+
```bash
54+
microk8s.kubectl config view --raw > /tmp/kubeconfig
55+
```
56+
57+
## Deploy application
58+
59+
**The following commands assume that they are executed inside the maven module of each specific test**
60+
61+
Since FMP is based on the Fabric8 Kubernetes Client, we can leverage the `KUBECONFIG` environment variable
62+
to make FMP aware of the Kubernetes Config file we created above
63+
64+
```bash
65+
export KUBECONFIG=/tmp/kubeconfig
66+
../../mvnw clean package fabric8:build fabric8:push fabric8:deploy -Pfmp
67+
```
68+
69+
Make sure the application was deployed be executing:
70+
71+
```bash
72+
microk8s.kubectl get pod -l app=sb-fmp-microk8s
73+
```
74+
75+
It should look something like:
76+
77+
```
78+
NAME READY STATUS RESTARTS AGE
79+
simple-core-5fbb7646dc-66t7b 1/1 Running 0 57s
80+
```
81+
82+
The integration tests can be run against the service which is deployed inside the cluster by executing:
83+
84+
```bash
85+
../../mvnw verify -Pfmp,it -Dfabric8.skip
86+
```
87+
88+
This integration tests runs locally (making it easily debuggable), and interacts with the deployed service
89+
using the exposed port.
90+
By leveraging [Aquillian Cube Kubernetes](http://arquillian.org/arquillian-cube/#_kubernetes), it's able to setup and teardown resources needed for each test
91+
92+
To undeploy the application from the cluster simply execute:
93+
94+
```bash
95+
../../mvnw fabric8:undeploy -Pfmp
96+
```
97+
98+
## Run all tests
99+
100+
By executing
101+
102+
```bash
103+
deploy_test_undeploy_all.sh
104+
```
105+
106+
each one of the test applications will be deployed to the cluster and the integration tests will be executed
107+
108+

‎spring-cloud-kubernetes-integration-tests/deploy_test_undeploy_all.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
# The script launches and tests each test application - one at a time
6+
7+
for D in $(find . -maxdepth 1 -mindepth 1 -not -path '*/\.*' -type d); do
8+
pushd ${D} > /dev/null
9+
../deploy_test_undeploy_single.sh
10+
/snap/bin/microk8s.kubectl wait --for=delete pod -l group=org.springframework.cloud --timeout=60s > /dev/null
11+
popd > /dev/null
12+
done
13+

‎spring-cloud-kubernetes-integration-tests/deploy_test_undeploy_single.sh

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
# The script assumes that there is only one application deployed by FMP running at a time
6+
7+
SCRIPT_ABSOLUTE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
8+
mvnCmd="${SCRIPT_ABSOLUTE_DIR}"/../mvnw
9+
kubectlCmd=/snap/bin/microk8s.kubectl
10+
11+
export KUBECONFIG=/tmp/kubeconfig
12+
13+
echo "Starting application deployment to cluster"
14+
$mvnCmd clean package fabric8:build fabric8:push fabric8:deploy -Pfmp
15+
echo "Finished deployment"
16+
17+
echo "Waiting for pod to become Ready"
18+
# We are leveraging the fact the fact that FMP adds the group label and the fact that only one
19+
# application can be running at a time
20+
$kubectlCmd wait --for=condition=Ready pod -l group=org.springframework.cloud --timeout=60s > /dev/null
21+
22+
echo "Starting the integration tests"
23+
$mvnCmd verify -Pfmp,it -Dfabric8.skip
24+
25+
echo "Successfully executed integration tests"
26+
echo "Undeploying application"
27+
$mvnCmd fabric8:undeploy -Pfmp

‎spring-cloud-kubernetes-integration-tests/pom.xml

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
<parent>
7+
<groupId>org.springframework.cloud</groupId>
8+
<artifactId>spring-cloud-kubernetes</artifactId>
9+
<version>1.0.0.BUILD-SNAPSHOT</version>
10+
</parent>
11+
12+
<artifactId>spring-cloud-kubernetes-integration-tests</artifactId>
13+
<packaging>pom</packaging>
14+
15+
<name>Spring Cloud Kubernetes :: Integration Tests</name>
16+
<description>Integration tests where SCK applications are run inside a Kubernetes cluster
17+
</description>
18+
19+
<properties>
20+
<java.version>1.8</java.version>
21+
<fmp.version>3.5.42</fmp.version>
22+
<arquillian-cube.version>1.18.2</arquillian-cube.version>
23+
<arquillian.version>1.4.0.Final</arquillian.version>
24+
<rest-assured.version>3.2.0</rest-assured.version>
25+
26+
<!-- We use the unix socket where microk8s' docker daemon listens -->
27+
<docker.host>unix:///var/snap/microk8s/current/docker.sock</docker.host>
28+
29+
<!-- This is the default location of the container registry that comes with microk8s -->
30+
<microk8s.registry>localhost:32000</microk8s.registry>
31+
32+
<!--
33+
The port on localhost where the application will listen to from outside the cluster
34+
Will be used to construct a NodePort
35+
-->
36+
<nodeport.value>32222</nodeport.value>
37+
</properties>
38+
39+
<build>
40+
<plugins>
41+
<plugin>
42+
<groupId>org.springframework.boot</groupId>
43+
<artifactId>spring-boot-maven-plugin</artifactId>
44+
</plugin>
45+
46+
<plugin>
47+
<groupId>org.apache.maven.plugins</groupId>
48+
<artifactId>maven-deploy-plugin</artifactId>
49+
<version>${maven-deploy-plugin.version}</version>
50+
<configuration>
51+
<skip>true</skip>
52+
</configuration>
53+
</plugin>
54+
55+
</plugins>
56+
</build>
57+
58+
<profiles>
59+
<profile>
60+
<id>fmp</id>
61+
<build>
62+
<plugins>
63+
<plugin>
64+
<groupId>io.fabric8</groupId>
65+
<artifactId>fabric8-maven-plugin</artifactId>
66+
<version>${fmp.version}</version>
67+
<configuration>
68+
<!--
69+
FMP can work with Openshift as well, so here we explicitly force it
70+
to work with vanilla Kubernetes and Docker builds
71+
-->
72+
<buildStrategy>docker</buildStrategy>
73+
<mode>kubernetes</mode>
74+
<pushRegistry>${microk8s.registry}</pushRegistry>
75+
</configuration>
76+
<executions>
77+
<execution>
78+
<id>fabric8</id>
79+
<goals>
80+
<goal>resource</goal>
81+
<goal>build</goal>
82+
</goals>
83+
</execution>
84+
</executions>
85+
</plugin>
86+
</plugins>
87+
</build>
88+
</profile>
89+
<profile>
90+
<id>it</id>
91+
<build>
92+
<plugins>
93+
<plugin>
94+
<groupId>org.apache.maven.plugins</groupId>
95+
<artifactId>maven-failsafe-plugin</artifactId>
96+
<configuration>
97+
<systemPropertyVariables>
98+
<service.host>localhost</service.host>
99+
<service.port>${nodeport.value}</service.port>
100+
</systemPropertyVariables>
101+
<classesDirectory>${project.build.outputDirectory}</classesDirectory>
102+
</configuration>
103+
<executions>
104+
<execution>
105+
<goals>
106+
<goal>integration-test</goal>
107+
<goal>verify</goal>
108+
</goals>
109+
</execution>
110+
</executions>
111+
</plugin>
112+
</plugins>
113+
</build>
114+
</profile>
115+
</profiles>
116+
117+
<dependencyManagement>
118+
<dependencies>
119+
<dependency>
120+
<groupId>org.jboss.arquillian</groupId>
121+
<artifactId>arquillian-bom</artifactId>
122+
<version>${arquillian.version}</version>
123+
<type>pom</type>
124+
<scope>import</scope>
125+
</dependency>
126+
<dependency>
127+
<groupId>io.rest-assured</groupId>
128+
<artifactId>rest-assured</artifactId>
129+
<version>${rest-assured.version}</version>
130+
</dependency>
131+
</dependencies>
132+
</dependencyManagement>
133+
134+
<modules>
135+
<module>simple-core</module>
136+
<module>simple-configmap</module>
137+
</modules>
138+
139+
140+
</project>

0 commit comments

Comments
 (0)