Skip to content

Commit 202f707

Browse files
andmariosandmarios
committed
Merge branch 'master' into 2.3.0
2 parents 0e1d961 + fcbab20 commit 202f707

File tree

7 files changed

+63
-14
lines changed

7 files changed

+63
-14
lines changed

‎filesystem/usr/local/share/landoop/sample-data/put.sh

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost}
6+
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
77

88
# Create Topics
99
for key in 0 1 2 3 4; do
1010
# Create topic with x partitions and a retention time of 10 years.
1111
kafka-topics \
1212
--zookeeper localhost:${ZK_PORT} \
13+
${GENERATOR_PRODUCER_PROPERTIES} \
1314
--topic "${TOPICS[key]}" \
1415
--partitions "${PARTITIONS[key]}" \
1516
--replication-factor "${REPLICATION[key]}" \
@@ -26,7 +27,8 @@ for key in 0 3 4; do
2627
unset SCHEMA_REGISTRY_LOG4J_OPTS
2728
/usr/local/bin/normcat -r 5000 "${DATA[key]}" | \
2829
kafka-avro-console-producer \
29-
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
30+
--broker-list ${GENERATOR_BROKER} \
31+
${GENERATOR_PRODUCER_PROPERTIES} \
3032
--topic "${TOPICS[key]}" \
3133
--property parse.key=true \
3234
--property key.schema="$(cat "${KEYS[key]}")" \
@@ -42,7 +44,8 @@ for key in 1; do
4244
unset SCHEMA_REGISTRY_LOG4J_OPTS
4345
/usr/local/bin/normcat -r 5000 "${DATA[key]}" | \
4446
kafka-avro-console-producer \
45-
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
47+
--broker-list ${GENERATOR_BROKER} \
48+
${GENERATOR_PRODUCER_PROPERTIES} \
4649
--topic "${TOPICS[key]}" \
4750
--property value.schema="$(cat "${VALUES[key]}")" \
4851
--property schema.registry.url=http://localhost:${REGISTRY_PORT}
@@ -57,7 +60,8 @@ for key in 2; do
5760
/usr/local/bin/normcat -r 5000 "${DATA[key]}" | \
5861
sed -r -e 's/([A-Z0-9-]*):/{"serial_number":"\1"}#/' | \
5962
kafka-console-producer \
60-
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
63+
--broker-list ${GENERATOR_BROKER} \
64+
${GENERATOR_PRODUCER_PROPERTIES} \
6165
--topic "${TOPICS[key]}" \
6266
--property parse.key=true \
6367
--property "key.separator=#"

‎filesystem/usr/local/share/landoop/sample-data/running-ais.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost}
6+
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
77

88
# Create Topics
99
# shellcheck disable=SC2043
@@ -29,7 +29,8 @@ for key in 0; do
2929
unset SCHEMA_REGISTRY_LOG4J_OPTS
3030
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
3131
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
32-
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
32+
--broker-list ${GENERATOR_BROKER} \
33+
${GENERATOR_PRODUCER_PROPERTIES} \
3334
--topic "${TOPICS[key]}" \
3435
--property parse.key=true \
3536
--property key.schema="$(cat "${KEYS[key]}")" \

‎filesystem/usr/local/share/landoop/sample-data/running-reddit.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost}
6+
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
77

88
# Create Topics
99
# shellcheck disable=SC2043
@@ -29,7 +29,8 @@ for key in 1; do
2929
unset SCHEMA_REGISTRY_LOG4J_OPTS
3030
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
3131
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
32-
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
32+
--broker-list ${GENERATOR_BROKER} \
33+
${GENERATOR_PRODUCER_PROPERTIES} \
3334
--topic "${TOPICS[key]}" \
3435
--property parse.key=true \
3536
--property key.schema="$(cat "${KEYS[key]}")" \

‎filesystem/usr/local/share/landoop/sample-data/running-smart.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost}
6+
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
77

88
# Create Topics
99
# shellcheck disable=SC2043
@@ -29,7 +29,8 @@ for key in 2; do
2929
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
3030
sed -r -e 's/([A-Z0-9-]*):/{"serial_number":"\1"}#/' | \
3131
KAFKA_HEAP_OPTS="-Xmx50m" kafka-console-producer \
32-
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
32+
--broker-list ${GENERATOR_BROKER} \
33+
${GENERATOR_PRODUCER_PROPERTIES} \
3334
--topic "${TOPICS[key]}" \
3435
--property parse.key=true \
3536
--property "key.separator=#"

‎filesystem/usr/local/share/landoop/sample-data/running-taxis.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost}
6+
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
77

88
# Create Topics
99
# shellcheck disable=SC2043
@@ -29,7 +29,8 @@ for key in 1; do
2929
unset SCHEMA_REGISTRY_LOG4J_OPTS
3030
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
3131
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
32-
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
32+
--broker-list ${GENERATOR_BROKER} \
33+
${GENERATOR_PRODUCER_PROPERTIES} \
3334
--topic "${TOPICS[key]}" \
3435
--property value.schema="$(cat "${VALUES[key]}")" \
3536
--property schema.registry.url=http://localhost:${REGISTRY_PORT}

‎filesystem/usr/local/share/landoop/sample-data/running-telecom-italia.sh

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost}
6+
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
77

88
# Create Topics
99
# shellcheck disable=SC2043
@@ -31,6 +31,7 @@ for key in 4; do
3131
/usr/local/bin/normcat -v "${DATA[key]}" | \
3232
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
3333
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
34+
${GENERATOR_PRODUCER_PROPERTIES} \
3435
--topic "${TOPICS[key]}" \
3536
--property parse.key=true \
3637
--property key.schema="$(cat "${KEYS[key]}")" \
@@ -46,7 +47,8 @@ for key in 3; do
4647
unset SCHEMA_REGISTRY_LOG4J_OPTS
4748
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
4849
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
49-
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
50+
--broker-list ${GENERATOR_BROKER} \
51+
${GENERATOR_PRODUCER_PROPERTIES} \
5052
--topic "${TOPICS[key]}" \
5153
--property parse.key=true \
5254
--property key.schema="$(cat "${KEYS[key]}")" \

‎hooks/post_push

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
3+
set -o nounset
4+
5+
# Load build information
6+
eval $(docker run --rm $IMAGE_NAME cat build.info)
7+
8+
# MODE=fdd
9+
# if [[ -n $FDD_LENSES_VERSION ]]; then
10+
# MODE=box
11+
# fi
12+
13+
# Master should publish fast-data-dev:latest
14+
if [[ $BUILD_BRANCH == master ]]; then
15+
# 'landooop/fast-data-dev:latest' is pushed automatically
16+
17+
# Also push to lensesio/fast-data-dev
18+
docker tag $IMAGE_NAME lensesio/fast-data-dev:latest
19+
docker push lensesio/fast-data-dev:latest
20+
fi
21+
22+
# FDD branches should publish minor and major version tags
23+
if [[ $BUILD_BRANCH =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ -n $FDD_LKD_VERSION ]]; then
24+
# BUILD_BRANCH is the same as FDD_LKD_VERSION, e.g 2.2.1
25+
# 'landoop/fast-data-dev:$BUILD_BRANCH is pushed automatically
26+
27+
# Also push 'lensesio/fast-data-dev:$BUILD_BRANCH'
28+
docker tag $IMAGE_NAME lensesio/fast-data-dev:$BUILD_BRANCH
29+
docker push lensesio/fast-data-dev:$BUILD_BRANCH
30+
31+
MAJOR_VERSION="$(sed -r -e 's/^([0-9]+\.[0-9]+)\.[0-9]+$/\1/' <<<"$FDD_LKD_VERSION")"
32+
# Also push 'landoop/fast-data-dev:$MAJOR_VERSION'
33+
docker tag $IMAGE_NAME landoop/fast-data-dev:$MAJOR_VERSION
34+
docker push landoop/fast-data-dev:$MAJOR_VERSION
35+
36+
# Also push 'lensesio/fast-data-dev:$MAJOR_VERSION
37+
docker tag $IMAGE_NAME lensesio/fast-data-dev:$MAJOR_VERSION
38+
docker push lensesio/fast-data-dev:$MAJOR_VERSION
39+
fi

0 commit comments

Comments
 (0)