From 737b823cc577412ddab206d2e3493b855d17bd12 Mon Sep 17 00:00:00 2001 From: su622b Date: Mon, 9 Mar 2020 14:37:18 -0400 Subject: [PATCH] confluent based image Issue-ID: DMAAP-1337 Change-Id: Iebec5410769271183ec5417e98f839a355b9c85f Signed-off-by: su622b --- .../docker/include/etc/confluent/docker/configure | 123 +++++++++++++++++++++ .../docker/include/etc/confluent/docker/ensure | 24 ++++ .../etc/confluent/docker/kafka.properties.template | 20 ++++ .../docker/include/etc/confluent/docker/launch | 37 +++++++ .../etc/confluent/docker/log4j.properties.template | 27 +++++ src/main/docker/include/etc/confluent/docker/run | 41 +++++++ .../docker/tools-log4j.properties.template | 7 ++ 7 files changed, 279 insertions(+) create mode 100644 src/main/docker/include/etc/confluent/docker/configure create mode 100644 src/main/docker/include/etc/confluent/docker/ensure create mode 100644 src/main/docker/include/etc/confluent/docker/kafka.properties.template create mode 100644 src/main/docker/include/etc/confluent/docker/launch create mode 100644 src/main/docker/include/etc/confluent/docker/log4j.properties.template create mode 100644 src/main/docker/include/etc/confluent/docker/run create mode 100644 src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template diff --git a/src/main/docker/include/etc/confluent/docker/configure b/src/main/docker/include/etc/confluent/docker/configure new file mode 100644 index 0000000..f0d8701 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/configure @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. /etc/confluent/docker/bash-config + +dub ensure KAFKA_ZOOKEEPER_CONNECT +dub ensure KAFKA_ADVERTISED_LISTENERS + +# By default, LISTENERS is derived from ADVERTISED_LISTENERS by replacing +# hosts with 0.0.0.0. This is good default as it ensures that the broker +# process listens on all ports. +if [[ -z "${KAFKA_LISTENERS-}" ]] +then + export KAFKA_LISTENERS + KAFKA_LISTENERS=$(cub listeners "$KAFKA_ADVERTISED_LISTENERS") +fi + +dub path /etc/kafka/ writable + +if [[ -z "${KAFKA_LOG_DIRS-}" ]] +then + export KAFKA_LOG_DIRS + KAFKA_LOG_DIRS="/var/lib/kafka/data" +fi + +# advertised.host, advertised.port, host and port are deprecated. Exit if these properties are set. +if [[ -n "${KAFKA_ADVERTISED_PORT-}" ]] +then + echo "advertised.port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +if [[ -n "${KAFKA_ADVERTISED_HOST-}" ]] +then + echo "advertised.host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +if [[ -n "${KAFKA_HOST-}" ]] +then + echo "host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +if [[ -n "${KAFKA_PORT-}" ]] +then + echo "port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +# Set if ADVERTISED_LISTENERS has SSL:// or SASL_SSL:// endpoints. +if [[ $KAFKA_ADVERTISED_LISTENERS == *"SSL://"* ]] +then + echo "SSL is enabled." + + dub ensure KAFKA_SSL_KEYSTORE_FILENAME + export KAFKA_SSL_KEYSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_FILENAME" + dub path "$KAFKA_SSL_KEYSTORE_LOCATION" exists + + dub ensure KAFKA_SSL_KEY_CREDENTIALS + KAFKA_SSL_KEY_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEY_CREDENTIALS" + dub path "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION" exists + export KAFKA_SSL_KEY_PASSWORD + KAFKA_SSL_KEY_PASSWORD=$(cat "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION") + + dub ensure KAFKA_SSL_KEYSTORE_CREDENTIALS + KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_CREDENTIALS" + dub path "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION" exists + export KAFKA_SSL_KEYSTORE_PASSWORD + KAFKA_SSL_KEYSTORE_PASSWORD=$(cat "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION") + + if [[ -n "${KAFKA_SSL_CLIENT_AUTH-}" ]] && ( [[ $KAFKA_SSL_CLIENT_AUTH == *"required"* ]] || [[ $KAFKA_SSL_CLIENT_AUTH == *"requested"* ]] ) + then + dub ensure KAFKA_SSL_TRUSTSTORE_FILENAME + export KAFKA_SSL_TRUSTSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_FILENAME" + dub path "$KAFKA_SSL_TRUSTSTORE_LOCATION" exists + + dub ensure KAFKA_SSL_TRUSTSTORE_CREDENTIALS + KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_CREDENTIALS" + dub path "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION" exists + export KAFKA_SSL_TRUSTSTORE_PASSWORD + KAFKA_SSL_TRUSTSTORE_PASSWORD=$(cat "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION") + fi + +fi + +# Set if KAFKA_ADVERTISED_LISTENERS has SASL_PLAINTEXT:// or SASL_SSL:// endpoints. +if [[ $KAFKA_ADVERTISED_LISTENERS =~ .*SASL_.*://.* ]] +then + echo "SASL" is enabled. + + dub ensure KAFKA_OPTS + + if [[ ! $KAFKA_OPTS == *"java.security.auth.login.config"* ]] + then + echo "KAFKA_OPTS should contain 'java.security.auth.login.config' property." + fi +fi + +if [[ -n "${KAFKA_JMX_OPTS-}" ]] +then + if [[ ! $KAFKA_JMX_OPTS == *"com.sun.management.jmxremote.rmi.port"* ]] + then + echo "KAFKA_OPTS should contain 'com.sun.management.jmxremote.rmi.port' property. It is required for accessing the JMX metrics externally." + fi +fi + +dub template "/etc/confluent/docker/${COMPONENT}.properties.template" "/etc/${COMPONENT}/${COMPONENT}.properties" +dub template "/etc/confluent/docker/log4j.properties.template" "/etc/${COMPONENT}/log4j.properties" +dub template "/etc/confluent/docker/tools-log4j.properties.template" "/etc/${COMPONENT}/tools-log4j.properties" diff --git a/src/main/docker/include/etc/confluent/docker/ensure b/src/main/docker/include/etc/confluent/docker/ensure new file mode 100644 index 0000000..4bc99f3 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/ensure @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. /etc/confluent/docker/bash-config + +export KAFKA_DATA_DIRS=${KAFKA_DATA_DIRS:-"/var/lib/kafka/data"} +echo "===> Check if $KAFKA_DATA_DIRS is writable ..." +dub path "$KAFKA_DATA_DIRS" writable + +echo "===> Check if Zookeeper is healthy ..." +cub zk-ready "$KAFKA_ZOOKEEPER_CONNECT" "${KAFKA_CUB_ZK_TIMEOUT:-40}" diff --git a/src/main/docker/include/etc/confluent/docker/kafka.properties.template b/src/main/docker/include/etc/confluent/docker/kafka.properties.template new file mode 100644 index 0000000..242e393 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/kafka.properties.template @@ -0,0 +1,20 @@ +{% set excluded_props = ['KAFKA_VERSION', + 'KAFKA_HEAP_OPTS' + 'KAFKA_LOG4J_OPTS', + 'KAFKA_OPTS', + 'KAFKA_JMX_OPTS', + 'KAFKA_JVM_PERFORMANCE_OPTS', + 'KAFKA_GC_LOG_OPTS', + 'KAFKA_LOG4J_ROOT_LOGLEVEL', + 'KAFKA_LOG4J_LOGGERS', + 'KAFKA_TOOLS_LOG4J_LOGLEVEL'] +-%} +{% set kafka_props = env_to_props('KAFKA_', '', exclude=excluded_props) -%} +{% for name, value in kafka_props.iteritems() -%} +{{name}}={{value}} +{% endfor -%} + +{% set confluent_support_props = env_to_props('CONFLUENT_SUPPORT_', 'confluent.support.') -%} +{% for name, value in confluent_support_props.iteritems() -%} +{{name}}={{value}} +{% endfor -%} diff --git a/src/main/docker/include/etc/confluent/docker/launch b/src/main/docker/include/etc/confluent/docker/launch new file mode 100644 index 0000000..d1eaf56 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/launch @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. +if [ -z "$KAFKA_JMX_OPTS" ]; then + export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " +fi + +# The JMX client needs to be able to connect to java.rmi.server.hostname. +# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. +# For host n/w, this is the IP that the hostname on the host resolves to. + +# If you have more that one n/w configured, hostname -i gives you all the IPs, +# the default is to pick the first IP (or network). +export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} + +if [ "$KAFKA_JMX_PORT" ]; then + # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. + export JMX_PORT=$KAFKA_JMX_PORT + export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" +fi + +echo "===> Launching ${COMPONENT} ... " +exec "${COMPONENT}"-server-start /etc/"${COMPONENT}"/"${COMPONENT}".properties diff --git a/src/main/docker/include/etc/confluent/docker/log4j.properties.template b/src/main/docker/include/etc/confluent/docker/log4j.properties.template new file mode 100644 index 0000000..bdd6e5b --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/log4j.properties.template @@ -0,0 +1,27 @@ +:x +log4j.rootLogger={{ env["KAFKA_LOG4J_ROOT_LOGLEVEL"] | default('INFO') }}, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +{% set loggers = { + 'kafka': 'INFO', + 'kafka.network.RequestChannel$': 'WARN', + 'kafka.producer.async.DefaultEventHandler': 'DEBUG', + 'kafka.request.logger': 'WARN', + 'kafka.controller': 'TRACE', + 'kafka.log.LogCleaner': 'INFO', + 'state.change.logger': 'TRACE', + 'kafka.authorizer.logger': 'WARN', + 'org.onap': 'INFO' + } -%} + + +{% if env['KAFKA_LOG4J_LOGGERS'] %} +{% set loggers = parse_log4j_loggers(env['KAFKA_LOG4J_LOGGERS'], loggers) %} +{% endif %} + +{% for logger,loglevel in loggers.iteritems() %} +log4j.logger.{{logger}}={{loglevel}} +{% endfor %} diff --git a/src/main/docker/include/etc/confluent/docker/run b/src/main/docker/include/etc/confluent/docker/run new file mode 100644 index 0000000..4501e22 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/run @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. /etc/confluent/docker/bash-config + +# Set environment values if they exist as arguments +if [ $# -ne 0 ]; then + echo "===> Overriding env params with args ..." + for var in "$@" + do + export "$var" + done +fi + +echo "===> ENV Variables ..." +show_env + +echo "===> User" +id + +echo "===> Configuring ..." +/etc/confluent/docker/configure + +echo "===> Running preflight checks ... " +/etc/confluent/docker/ensure + +echo "===> Launching ... " +exec /etc/confluent/docker/launch diff --git a/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template b/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template new file mode 100644 index 0000000..da06f13 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template @@ -0,0 +1,7 @@ + +log4j.rootLogger={{ env["KAFKA_TOOLS_LOG4J_LOGLEVEL"] | default('WARN') }}, stderr + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n +log4j.appender.stderr.Target=System.err -- 2.16.6