Skip to content

Commit ed64707

Browse files
andmariosandmarios
committed
Merge branch 'master' into 2.3.0
2 parents 45c6de1 + 6988616 commit ed64707

File tree

9 files changed

+44
-51
lines changed

9 files changed

+44
-51
lines changed

‎filesystem/usr/local/share/landoop/etc/supervisord.d/99-supervisord-running-sample-data.conf

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[program:running-sample-data-ais]
22
user=nobody
33
directory=/usr/local/share/landoop/sample-data
4-
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "http://localhost:$REGISTRY_PORT" | grep "{}" && { sleep 5; break; }; done; /usr/local/share/landoop/sample-data/running-ais.sh'
4+
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "$GENERATOR_SCHEMA_REGISTRY_URL" | grep "{}" && { sleep 5; break; }; done; /usr/local/share/landoop/sample-data/running-ais.sh'
55
redirect_stderr=true
66
stdout_logfile=/var/log/running-ais.log
77
autorestart=false
@@ -10,7 +10,7 @@ startretries=1
1010
[program:running-sample-data-taxis]
1111
user=nobody
1212
directory=/usr/local/share/landoop/sample-data
13-
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "http://localhost:$REGISTRY_PORT" | grep "{}" && { sleep 6; break; }; done; /usr/local/share/landoop/sample-data/running-taxis.sh'
13+
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "$GENERATOR_SCHEMA_REGISTRY_URL" | grep "{}" && { sleep 6; break; }; done; /usr/local/share/landoop/sample-data/running-taxis.sh'
1414
redirect_stderr=true
1515
stdout_logfile=/var/log/running-taxis.log
1616
autorestart=false
@@ -19,7 +19,7 @@ startretries=1
1919
# [program:running-sample-data-reddit]
2020
# user=nobody
2121
# directory=/usr/local/share/landoop/sample-data
22-
# command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "http://localhost:$REGISTRY_PORT" | grep "{}" && { sleep 7; break; }; done; /usr/local/share/landoop/sample-data/running-reddit.sh'
22+
# command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "$GENERATOR_SCHEMA_REGISTRY_URL" | grep "{}" && { sleep 7; break; }; done; /usr/local/share/landoop/sample-data/running-reddit.sh'
2323
# redirect_stderr=true
2424
# stdout_logfile=/var/log/running-reddit.log
2525
# autorestart=false
@@ -28,7 +28,7 @@ startretries=1
2828
[program:running-sample-data-backblaze-smart]
2929
user=nobody
3030
directory=/usr/local/share/landoop/sample-data
31-
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "http://localhost:$REGISTRY_PORT" | grep "{}" && { sleep 8; break; }; done; /usr/local/share/landoop/sample-data/running-smart.sh'
31+
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "$GENERATOR_SCHEMA_REGISTRY_URL" | grep "{}" && { sleep 8; break; }; done; /usr/local/share/landoop/sample-data/running-smart.sh'
3232
redirect_stderr=true
3333
stdout_logfile=/var/log/running-smart.log
3434
autorestart=false
@@ -37,7 +37,7 @@ startretries=1
3737
[program:running-sample-data-telecom-italia]
3838
user=nobody
3939
directory=/usr/local/share/landoop/sample-data
40-
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "http://localhost:$REGISTRY_PORT" | grep "{}" && { sleep 9; break; }; done; /usr/local/share/landoop/sample-data/running-telecom-italia.sh'
40+
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "$GENERATOR_SCHEMA_REGISTRY_URL" | grep "{}" && { sleep 9; break; }; done; /usr/local/share/landoop/sample-data/running-telecom-italia.sh'
4141
redirect_stderr=true
4242
stdout_logfile=/var/log/running-telecom-italia.log
4343
autorestart=false

‎filesystem/usr/local/share/landoop/etc/supervisord.d/99-supervisord-sample-data.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[program:sample-data]
22
user=nobody
33
directory=/usr/local/share/landoop/sample-data
4-
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "http://localhost:$REGISTRY_PORT" | grep "{}" && { sleep 5; break; }; done; /usr/local/share/landoop/sample-data/put.sh'
4+
command=bash -c 'for ((i=0;i<90;i++)); do sleep 2; curl "$GENERATOR_SCHEMA_REGISTRY_URL" | grep "{}" && { sleep 5; break; }; done; /usr/local/share/landoop/sample-data/put.sh'
55
redirect_stderr=true
66
stdout_logfile=/var/log/sampledata.log
77
autorestart=false

‎filesystem/usr/local/share/landoop/sample-data/put.sh

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,12 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
7-
86
# Create Topics
97
for key in 0 1 2 3 4; do
108
# Create topic with x partitions and a retention time of 10 years.
119
kafka-topics \
12-
--zookeeper localhost:${ZK_PORT} \
13-
${GENERATOR_PRODUCER_PROPERTIES} \
10+
--zookeeper "${GENERATOR_ZK_HOST}:${ZK_PORT}" \
11+
"${GENERATOR_PRODUCER_PROPERTIES}" \
1412
--topic "${TOPICS[key]}" \
1513
--partitions "${PARTITIONS[key]}" \
1614
--replication-factor "${REPLICATION[key]}" \
@@ -27,13 +25,13 @@ for key in 0 3 4; do
2725
unset SCHEMA_REGISTRY_LOG4J_OPTS
2826
/usr/local/bin/normcat -r 5000 "${DATA[key]}" | \
2927
kafka-avro-console-producer \
30-
--broker-list ${GENERATOR_BROKER} \
31-
${GENERATOR_PRODUCER_PROPERTIES} \
28+
--broker-list "${GENERATOR_BROKER}" \
29+
"${GENERATOR_PRODUCER_PROPERTIES}" \
3230
--topic "${TOPICS[key]}" \
3331
--property parse.key=true \
3432
--property key.schema="$(cat "${KEYS[key]}")" \
3533
--property value.schema="$(cat "${VALUES[key]}")" \
36-
--property schema.registry.url=http://localhost:${REGISTRY_PORT}
34+
--property schema.registry.url="${GENERATOR_SCHEMA_REGISTRY_URL}"
3735
done
3836

3937
# Insert data without keys
@@ -44,11 +42,11 @@ for key in 1; do
4442
unset SCHEMA_REGISTRY_LOG4J_OPTS
4543
/usr/local/bin/normcat -r 5000 "${DATA[key]}" | \
4644
kafka-avro-console-producer \
47-
--broker-list ${GENERATOR_BROKER} \
48-
${GENERATOR_PRODUCER_PROPERTIES} \
45+
--broker-list "${GENERATOR_BROKER}" \
46+
"${GENERATOR_PRODUCER_PROPERTIES}" \
4947
--topic "${TOPICS[key]}" \
5048
--property value.schema="$(cat "${VALUES[key]}")" \
51-
--property schema.registry.url=http://localhost:${REGISTRY_PORT}
49+
--property schema.registry.url="${GENERATOR_SCHEMA_REGISTRY_URL}"
5250
done
5351

5452
# Insert json data with text keys converted to json keys
@@ -60,8 +58,8 @@ for key in 2; do
6058
/usr/local/bin/normcat -r 5000 "${DATA[key]}" | \
6159
sed -r -e 's/([A-Z0-9-]*):/{"serial_number":"\1"}#/' | \
6260
kafka-console-producer \
63-
--broker-list ${GENERATOR_BROKER} \
64-
${GENERATOR_PRODUCER_PROPERTIES} \
61+
--broker-list "${GENERATOR_BROKER}" \
62+
"${GENERATOR_PRODUCER_PROPERTIES}" \
6563
--topic "${TOPICS[key]}" \
6664
--property parse.key=true \
6765
--property "key.separator=#"

‎filesystem/usr/local/share/landoop/sample-data/running-ais.sh

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
7-
86
# Create Topics
97
# shellcheck disable=SC2043
108
for key in 0; do
119
# Create topic with x partitions and a retention size of 50MB, log segment
1210
# size of 20MB and compression type y.
1311
kafka-topics \
14-
--zookeeper localhost:${ZK_PORT} \
12+
--zookeeper "${GENERATOR_ZK_HOST}:${ZK_PORT}" \
1513
--topic "${TOPICS[key]}" \
1614
--partitions "${PARTITIONS[key]}" \
1715
--replication-factor "${REPLICATION[key]}" \
@@ -29,11 +27,11 @@ for key in 0; do
2927
unset SCHEMA_REGISTRY_LOG4J_OPTS
3028
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
3129
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
32-
--broker-list ${GENERATOR_BROKER} \
33-
${GENERATOR_PRODUCER_PROPERTIES} \
30+
--broker-list "${GENERATOR_BROKER}" \
31+
"${GENERATOR_PRODUCER_PROPERTIES}" \
3432
--topic "${TOPICS[key]}" \
3533
--property parse.key=true \
3634
--property key.schema="$(cat "${KEYS[key]}")" \
3735
--property value.schema="$(cat "${VALUES[key]}")" \
38-
--property schema.registry.url=http://localhost:${REGISTRY_PORT}
36+
--property schema.registry.url="${GENERATOR_SCHEMA_REGISTRY_URL}"
3937
done

‎filesystem/usr/local/share/landoop/sample-data/running-reddit.sh

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
7-
86
# Create Topics
97
# shellcheck disable=SC2043
108
for key in 1; do
119
# Create topic with x partitions and a retention size of 50MB, log segment
1210
# size of 20MB and compression type y.
1311
kafka-topics \
14-
--zookeeper localhost:${ZK_PORT} \
12+
--zookeeper "${GENERATOR_ZK_HOST}:${ZK_PORT}" \
1513
--topic "${TOPICS[key]}" \
1614
--partitions "${PARTITIONS[key]}" \
1715
--replication-factor "${REPLICATION[key]}" \
@@ -29,11 +27,11 @@ for key in 1; do
2927
unset SCHEMA_REGISTRY_LOG4J_OPTS
3028
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
3129
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
32-
--broker-list ${GENERATOR_BROKER} \
33-
${GENERATOR_PRODUCER_PROPERTIES} \
30+
--broker-list "${GENERATOR_BROKER}" \
31+
"${GENERATOR_PRODUCER_PROPERTIES}" \
3432
--topic "${TOPICS[key]}" \
3533
--property parse.key=true \
3634
--property key.schema="$(cat "${KEYS[key]}")" \
3735
--property value.schema="$(cat "${VALUES[key]}")" \
38-
--property schema.registry.url=http://localhost:${REGISTRY_PORT}
36+
--property schema.registry.url="${GENERATOR_SCHEMA_REGISTRY_URL}"
3937
done

‎filesystem/usr/local/share/landoop/sample-data/running-smart.sh

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
7-
86
# Create Topics
97
# shellcheck disable=SC2043
108
for key in 2; do
119
# Create topic with x partitions and a retention size of 50MB, log segment
1210
# size of 20MB and compression type y.
1311
kafka-topics \
14-
--zookeeper localhost:${ZK_PORT} \
12+
--zookeeper "${GENERATOR_ZK_HOST}:${ZK_PORT}" \
1513
--topic "${TOPICS[key]}" \
1614
--partitions "${PARTITIONS[key]}" \
1715
--replication-factor "${REPLICATION[key]}" \
@@ -29,8 +27,8 @@ for key in 2; do
2927
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
3028
sed -r -e 's/([A-Z0-9-]*):/{"serial_number":"\1"}#/' | \
3129
KAFKA_HEAP_OPTS="-Xmx50m" kafka-console-producer \
32-
--broker-list ${GENERATOR_BROKER} \
33-
${GENERATOR_PRODUCER_PROPERTIES} \
30+
--broker-list "${GENERATOR_BROKER}" \
31+
"${GENERATOR_PRODUCER_PROPERTIES}" \
3432
--topic "${TOPICS[key]}" \
3533
--property parse.key=true \
3634
--property "key.separator=#"

‎filesystem/usr/local/share/landoop/sample-data/running-taxis.sh

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
7-
86
# Create Topics
97
# shellcheck disable=SC2043
108
for key in 1; do
119
# Create topic with x partitions and a retention size of 50MB, log segment
1210
# size of 20MB and compression type y.
1311
kafka-topics \
14-
--zookeeper localhost:${ZK_PORT} \
12+
--zookeeper "${GENERATOR_ZK_HOST}:${ZK_PORT}" \
1513
--topic "${TOPICS[key]}" \
1614
--partitions "${PARTITIONS[key]}" \
1715
--replication-factor "${REPLICATION[key]}" \
@@ -29,9 +27,9 @@ for key in 1; do
2927
unset SCHEMA_REGISTRY_LOG4J_OPTS
3028
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
3129
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
32-
--broker-list ${GENERATOR_BROKER} \
33-
${GENERATOR_PRODUCER_PROPERTIES} \
30+
--broker-list "${GENERATOR_BROKER}" \
31+
"${GENERATOR_PRODUCER_PROPERTIES}" \
3432
--topic "${TOPICS[key]}" \
3533
--property value.schema="$(cat "${VALUES[key]}")" \
36-
--property schema.registry.url=http://localhost:${REGISTRY_PORT}
34+
--property schema.registry.url="${GENERATOR_SCHEMA_REGISTRY_URL}"
3735
done

‎filesystem/usr/local/share/landoop/sample-data/running-telecom-italia.sh

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@
33
# shellcheck source=variables.env
44
source variables.env
55

6-
GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
7-
86
# Create Topics
97
# shellcheck disable=SC2043
108
for key in 3 4; do
119
# Create topic with x partitions and a retention size of 50MB, log segment
1210
# size of 20MB and compression type y.
1311
kafka-topics \
14-
--zookeeper localhost:${ZK_PORT} \
12+
--zookeeper "${GENERATOR_ZK_HOST}:${ZK_PORT}" \
1513
--topic "${TOPICS[key]}" \
1614
--partitions "${PARTITIONS[key]}" \
1715
--replication-factor "${REPLICATION[key]}" \
@@ -30,13 +28,13 @@ for key in 4; do
3028
unset SCHEMA_REGISTRY_LOG4J_OPTS
3129
/usr/local/bin/normcat -v "${DATA[key]}" | \
3230
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
33-
--broker-list ${GENERATOR_BROKER}:${BROKER_PORT} \
34-
${GENERATOR_PRODUCER_PROPERTIES} \
31+
--broker-list "${GENERATOR_BROKER}" \
32+
"${GENERATOR_PRODUCER_PROPERTIES}" \
3533
--topic "${TOPICS[key]}" \
3634
--property parse.key=true \
3735
--property key.schema="$(cat "${KEYS[key]}")" \
3836
--property value.schema="$(cat "${VALUES[key]}")" \
39-
--property schema.registry.url=http://localhost:${REGISTRY_PORT}
37+
--property schema.registry.url="${GENERATOR_SCHEMA_REGISTRY_URL}"
4038
done
4139

4240
# Insert data with key
@@ -47,11 +45,11 @@ for key in 3; do
4745
unset SCHEMA_REGISTRY_LOG4J_OPTS
4846
/usr/local/bin/normcat -r "${RATES[key]}" -j "${JITTER[key]}" -p "${PERIOD[key]}" -c -v "${DATA[key]}" | \
4947
SCHEMA_REGISTRY_HEAP_OPTS="-Xmx50m" kafka-avro-console-producer \
50-
--broker-list ${GENERATOR_BROKER} \
51-
${GENERATOR_PRODUCER_PROPERTIES} \
48+
--broker-list "${GENERATOR_BROKER}" \
49+
"${GENERATOR_PRODUCER_PROPERTIES}" \
5250
--topic "${TOPICS[key]}" \
5351
--property parse.key=true \
5452
--property key.schema="$(cat "${KEYS[key]}")" \
5553
--property value.schema="$(cat "${VALUES[key]}")" \
56-
--property schema.registry.url=http://localhost:${REGISTRY_PORT}
54+
--property schema.registry.url="${GENERATOR_SCHEMA_REGISTRY_URL}"
5755
done

‎setup-and-run.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,11 @@ export ZOOKEEPER_maxClientCnxns=${ZOOKEEPER_maxClientCnxnxs:-0}
132132
export ZOOKEEPER_LOG4J_OPTS=${ZOOKEEPER_LOG4J_OPTS:--Dlog4j.configuration=file:/var/run/zookeeper/log4j.properties}
133133
export ZOOKEEPER_JMX_OPTS=${ZOOKEEPER_JMX_OPTS:--Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=$ADV_HOST_JMX -Dcom.sun.management.jmxremote.rmi.port=$ZK_JMX_PORT}
134134

135+
# Set env vars for generator
136+
export GENERATOR_BROKER=${GENERATOR_BROKER:-localhost:$BROKER_PORT}
137+
export GENERATOR_ZK_HOST=${GENERATOR_ZK_HOST:-localhost}
138+
export GENERATOR_SCHEMA_REGISTRY_URL=${GENERATOR_SCHEMA_REGISTRY_URL:-http://localhost:$REGISTRY_PORT}
139+
135140
# Set memory limits
136141
# Set connect heap size if needed
137142
if [[ -n $CONNECT_HEAP ]]; then CONNECT_HEAP="-Xmx$CONNECT_HEAP"; fi
@@ -223,7 +228,7 @@ for service in /usr/local/share/landoop/etc/supervisord.templates.d/*.conf; do
223228
envsubst < "$service" > /etc/supervisord.d/"$(basename "$service")"
224229
done
225230
# Disable services if asked
226-
if [[ $ZK_PORT == 0 ]]; then rm /etc/supervisord.d/*zookeeper.conf; fi
231+
if [[ $ZK_PORT == 0 ]] || [[ $GENERATOR_ZK_HOST != "localhost" ]]; then rm /etc/supervisord.d/*zookeeper.conf; fi
227232
if [[ $BROKER_PORT == 0 ]]; then rm /etc/supervisord.d/*broker.conf; fi
228233
if [[ $REGISTRY_PORT == 0 ]]; then rm /etc/supervisord.d/*schema-registry.conf; fi
229234
if [[ $CONNECT_PORT == 0 ]]; then rm /etc/supervisord.d/*connect-distributed.conf; fi

0 commit comments

Comments
 (0)