SSL example

README.md

@@ -180,7 +180,8 @@ To read more please follow to [chart documentation](charts/kafka-ui/README.md)
 
 UI for Apache Kafka supports TLS (SSL) and SASL connections for [encryption and authentication](http://kafka.apache.org/090/documentation.html#security). This can be configured by providing a combination of the following files (placed into the Kafka root directory):
 
-To be continued
+
+In the ``docker`` directory we have  kafka-ui-ssl.yaml as an example to configure SSL connections
 
 ## <a name="env_variables"></a> Environment Variables
 

docker/kafka-ssl/creds

@@ -0,0 +1 @@
+12345678

docker/kafka-ui-ssl.yaml

@@ -0,0 +1,60 @@
+---
+version: '3.4'
+services:
+
+  kafka-ui:
+    container_name: kafka-ui
+    image: provectuslabs/kafka-ui:latest
+    ports:
+      - 8080:8080
+    depends_on:
+      - zookeeper0
+      - kafka0
+    environment:
+      KAFKA_CLUSTERS_0_NAME: local
+      KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: SSL
+      KAFKA_CLUSTERS_0_PROPERTIES_SSL_MECHANISM: SSL
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:9092
+      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper0:2181
+      KAFKA_CLUSTER_0_SSL_KEYSTORE_LOCATION: /serverkeystore
+      KAFKA_CLUSTER_0_SSL_KEYSTORE_PASSWORD:  12345678
+      KAFKA_CLUSTER_0_SSL_TRUSTSTORE_LOCATION: /servertruststore
+      KAFKA_CLUSTER_0_SSL_TRUSTSTORE_PASSWORD: 12345678
+    volumes:
+      - ./jmx/serverkeystore:/serverkeystore
+      - ./jmx/servertruststore:/servertruststore
+
+  zookeeper0:
+     image: confluentinc/cp-zookeeper:6.0.1
+     environment:
+       ZOOKEEPER_CLIENT_PORT: 2181
+       ZOOKEEPER_TICK_TIME: 2000
+     ports:
+       - 2181:2181
+
+  kafka0:
+    image: confluentinc/cp-kafka:6.0.1
+    hostname: kafka0
+    depends_on:
+      - zookeeper0
+    ports:
+      - '9092:9092'
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper0:2181
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_ADVERTISED_LISTENERS: SSL://kafka0:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SSL:SSL,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: SSL
+      KAFKA_SECURITY_PROTOCOL: SSL
+      KAFKA_SSL_ENABLED_MECHANISMS: PLAIN,SSL
+      KAFKA_SSL_KEYSTORE_FILENAME: kafka.keystore.jks
+      KAFKA_SSL_KEYSTORE_CREDENTIALS: creds
+      KAFKA_SSL_KEY_CREDENTIALS: creds
+      KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.truststore.jks
+      KAFKA_SSL_TRUSTSTORE_CREDENTIALS: creds
+#      KAFKA_SSL_CLIENT_AUTH: 'required'
+      KAFKA_SSL_CLIENT_AUTH: "requested"
+      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ''
+    volumes:
+      - ./truststore:/etc/kafka/secrets

docker/kafka_gen_ssl_auto.sh

@@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+
+set -eu
+
+KEYSTORE_FILENAME="kafka.keystore.jks"
+VALIDITY_IN_DAYS=3650
+DEFAULT_TRUSTSTORE_FILENAME="kafka.truststore.jks"
+TRUSTSTORE_WORKING_DIRECTORY="truststore"
+KEYSTORE_WORKING_DIRECTORY="keystore"
+CA_CERT_FILE="ca-cert"
+KEYSTORE_SIGN_REQUEST="cert-file"
+KEYSTORE_SIGN_REQUEST_SRL="ca-cert.srl"
+KEYSTORE_SIGNED_CERT="cert-signed"
+
+export COUNTRY=US
+export STATE=IL
+export ORGANIZATION_UNIT=SE
+export CITY=Chicago
+export PASSWORD=secret
+
+COUNTRY=$COUNTRY
+STATE=$STATE
+OU=$ORGANIZATION_UNIT
+CN=`hostname -f`
+LOCATION=$CITY
+PASS=$PASSWORD
+
+function file_exists_and_exit() {
+  echo "'$1' cannot exist. Move or delete it before"
+  echo "re-running this script."
+  exit 1
+}
+
+if [ -e "$KEYSTORE_WORKING_DIRECTORY" ]; then
+  file_exists_and_exit $KEYSTORE_WORKING_DIRECTORY
+fi
+
+if [ -e "$CA_CERT_FILE" ]; then
+  file_exists_and_exit $CA_CERT_FILE
+fi
+
+if [ -e "$KEYSTORE_SIGN_REQUEST" ]; then
+  file_exists_and_exit $KEYSTORE_SIGN_REQUEST
+fi
+
+if [ -e "$KEYSTORE_SIGN_REQUEST_SRL" ]; then
+  file_exists_and_exit $KEYSTORE_SIGN_REQUEST_SRL
+fi
+
+if [ -e "$KEYSTORE_SIGNED_CERT" ]; then
+  file_exists_and_exit $KEYSTORE_SIGNED_CERT
+fi
+
+echo "Welcome to the Kafka SSL keystore and trust store generator script."
+
+trust_store_file=""
+trust_store_private_key_file=""
+
+  if [ -e "$TRUSTSTORE_WORKING_DIRECTORY" ]; then
+    file_exists_and_exit $TRUSTSTORE_WORKING_DIRECTORY
+  fi
+
+  mkdir $TRUSTSTORE_WORKING_DIRECTORY
+  echo
+  echo "OK, we'll generate a trust store and associated private key."
+  echo
+  echo "First, the private key."
+  echo
+
+  openssl req -new -x509 -keyout $TRUSTSTORE_WORKING_DIRECTORY/ca-key \
+    -out $TRUSTSTORE_WORKING_DIRECTORY/ca-cert -days $VALIDITY_IN_DAYS -nodes \
+    -subj "/C=$COUNTRY/ST=$STATE/L=$LOCATION/O=$OU/CN=$CN"
+
+  trust_store_private_key_file="$TRUSTSTORE_WORKING_DIRECTORY/ca-key"
+
+  echo
+  echo "Two files were created:"
+  echo " - $TRUSTSTORE_WORKING_DIRECTORY/ca-key -- the private key used later to"
+  echo "   sign certificates"
+  echo " - $TRUSTSTORE_WORKING_DIRECTORY/ca-cert -- the certificate that will be"
+  echo "   stored in the trust store in a moment and serve as the certificate"
+  echo "   authority (CA). Once this certificate has been stored in the trust"
+  echo "   store, it will be deleted. It can be retrieved from the trust store via:"
+  echo "   $ keytool -keystore <trust-store-file> -export -alias CARoot -rfc"
+
+  echo
+  echo "Now the trust store will be generated from the certificate."
+  echo
+
+  keytool -keystore $TRUSTSTORE_WORKING_DIRECTORY/$DEFAULT_TRUSTSTORE_FILENAME \
+    -alias CARoot -import -file $TRUSTSTORE_WORKING_DIRECTORY/ca-cert \
+    -noprompt -dname "C=$COUNTRY, ST=$STATE, L=$LOCATION, O=$OU, CN=$CN" -keypass $PASS -storepass $PASS -storetype JKS
+
+  trust_store_file="$TRUSTSTORE_WORKING_DIRECTORY/$DEFAULT_TRUSTSTORE_FILENAME"
+
+  echo
+  echo "$TRUSTSTORE_WORKING_DIRECTORY/$DEFAULT_TRUSTSTORE_FILENAME was created."
+
+  # don't need the cert because it's in the trust store.
+  rm $TRUSTSTORE_WORKING_DIRECTORY/$CA_CERT_FILE
+
+echo
+echo "Continuing with:"
+echo " - trust store file:        $trust_store_file"
+echo " - trust store private key: $trust_store_private_key_file"
+
+mkdir $KEYSTORE_WORKING_DIRECTORY
+
+echo
+echo "Now, a keystore will be generated. Each broker and logical client needs its own"
+echo "keystore. This script will create only one keystore. Run this script multiple"
+echo "times for multiple keystores."
+echo
+echo "     NOTE: currently in Kafka, the Common Name (CN) does not need to be the FQDN of"
+echo "           this host. However, at some point, this may change. As such, make the CN"
+echo "           the FQDN. Some operating systems call the CN prompt 'first / last name'"
+
+# To learn more about CNs and FQDNs, read:
+# https://docs.oracle.com/javase/7/docs/api/javax/net/ssl/X509ExtendedTrustManager.html
+
+keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME \
+  -alias localhost -validity $VALIDITY_IN_DAYS -genkey -keyalg RSA \
+   -noprompt -dname "C=$COUNTRY, ST=$STATE, L=$LOCATION, O=$OU, CN=$CN" -keypass $PASS -storepass $PASS -storetype JKS
+
+echo
+echo "'$KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME' now contains a key pair and a"
+echo "self-signed certificate. Again, this keystore can only be used for one broker or"
+echo "one logical client. Other brokers or clients need to generate their own keystores."
+
+echo
+echo "Fetching the certificate from the trust store and storing in $CA_CERT_FILE."
+echo
+
+keytool -keystore $trust_store_file -export -alias CARoot -rfc -file $CA_CERT_FILE -keypass $PASS -storepass $PASS
+
+echo
+echo "Now a certificate signing request will be made to the keystore."
+echo
+keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME -alias localhost \
+  -certreq -file $KEYSTORE_SIGN_REQUEST -keypass $PASS -storepass $PASS
+
+echo
+echo "Now the trust store's private key (CA) will sign the keystore's certificate."
+echo
+openssl x509 -req -CA $CA_CERT_FILE -CAkey $trust_store_private_key_file \
+  -in $KEYSTORE_SIGN_REQUEST -out $KEYSTORE_SIGNED_CERT \
+  -days $VALIDITY_IN_DAYS -CAcreateserial
+# creates $KEYSTORE_SIGN_REQUEST_SRL which is never used or needed.
+
+echo
+echo "Now the CA will be imported into the keystore."
+echo
+keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME -alias CARoot \
+  -import -file $CA_CERT_FILE -keypass $PASS -storepass $PASS -noprompt
+rm $CA_CERT_FILE # delete the trust store cert because it's stored in the trust store.
+
+echo
+echo "Now the keystore's signed certificate will be imported back into the keystore."
+echo
+keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME -alias localhost -import \
+  -file $KEYSTORE_SIGNED_CERT -keypass $PASS -storepass $PASS
+
+echo
+echo "All done!"
+echo
+echo "Deleting intermediate files. They are:"
+echo " - '$KEYSTORE_SIGN_REQUEST_SRL': CA serial number"
+echo " - '$KEYSTORE_SIGN_REQUEST': the keystore's certificate signing request"
+echo "   (that was fulfilled)"
+echo " - '$KEYSTORE_SIGNED_CERT': the keystore's certificate, signed by the CA, and stored back"
+echo "    into the keystore"
+
+  rm $KEYSTORE_SIGN_REQUEST_SRL
+  rm $KEYSTORE_SIGN_REQUEST
+  rm $KEYSTORE_SIGNED_CERT

docker/truststore/ca-key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDiGNzmx3KqKPoo
+1GObzOxXFK/H6DmMAksKmXIb/8BBlFCW78kPkC1fJVqzaH+7CR3M07bJZbug4XYH
+DeaQbdO+EAlnbQwNIzeEQ4E+jgvZgXHSWlX9VjcIVnQl7hdiFYc3G7F3PYvHU28T
+/z0NtGTghlPKq9cRZSazh/gJauCIESWH1ytCzUs4fVrFF+9G92ik8bP7jK+4CZJd
+dL57Dp3E8jumBxrguKrUPwyZoRTWlwVPwRLsl8Cb4tHhkwBKMz7HGJAHHxgn98jF
+KFoHdNK/Wii+aUHYlvTtrjSqiL5pXlkhHKH2jb4z+Gj5umwdWkRgloia818AGsSo
+zmPEu65PAgMBAAECggEABRawKhWFFCVV9ANLtp73FsKlbidg3DDVBYkMk2q/Jfqz
+kaIV8V66VrDdiLoqXchHvmtpasD6ZxWL9X6vmMKQCHAN9iDzK3yGbFikmQbTZ3LN
+YAGee9Qr6ukfNhdUuWVopKnf5aIh+jt7fYS6GAveVVtQmXgioAaZx2KhB307TnSh
+gjPLfrx1nrrmZwaQp80KkQxkKFvOadBT/swaD8rKnLNcD4vHWr2E8bZSg19flHaN
+eWziB0cu20bptmwKNZasPPgrXe19UJFNRISDP0NffKTub50T3SNZ8Jk4RI9ISiB5
+Jte6fnwdLp4M932//9uwObe3bCo1tihQAkRiypOKcQKBgQDyAciHlCRl08pjqAmB
+NfygIi9wMV+IJNU28w6OJaVZiagMkvxzxuU24fidBCg0cXL6lUPFlgaXq1ug5JuU
+i8jPJw0B0Xa2kot/XaN9Fp5jDFW0OQ2kU5Inadj/2HrimMecxy2i1yHDjDK4tbz1
+eySnkploh3QIbMfi9tWoPUX7eQKBgQDvK5SBko0Qk6S13jvIxjpqbRI9aVYnd6dJ
+V/wrJS/r+mItLn3p1MVzCdU8b9RZvc587zCCl8nj9Og2e+GdyxsmRYzMgCO9YvvZ
+UP6Rp6lashO25knUU7DJW7aCcJeEcYT7/iQ4iKqxHOJl413gkqStDHrudXLdBSyr
+ca+nZx6+BwKBgBLX/rRH3bmsDxNJb0wHVx9k6gqzXXvtWkjIDjcVSVW6eUJZY5zk
+PM7UM5aOT08eElTesIM/Fk5TAnR5uH+NjtpKcSHWHIdEy+Xj9an0y+pseHPrCVvl
+Qfkdrc7mHVNtkAVuIOiQFcnzLbPiOs9mgkkmChnwiiFpzvqmO6a49q0JAoGBAMUI
+yxyABnTcnXoy7l+2ZxOcbcVoDVEQcyGqOZW12if3K4B7Z8IGxu/B7wkOPAv+oakt
+SHKd5/ZDojaYkkNzuxcpfxhOjKMOc9sRrdeDH7MXN8SbLcGdfOw2AUKuzTSQbPVI
+5sAoXy7wU9eKm6FSThY+TNw8p/dLtwQ68XHhM2NrAoGBAOMHrXsa1s0XFUFJhnrH
+/fKCz/onuFaCXpoYSDuY2bLXTXhVMToBll7YRxnrXEo8jdcgOnagXb3/altMBSAE
+TYhX1q3GN/+AxNMCyRRB+nta+xH7S0rmJYGlyzPDUnoDRzhOKH+9r+kiRVo2ys9Q
+h31eHC+HxfTI1WXotTft/U0n
+-----END PRIVATE KEY-----

docker/truststore/creds

@@ -0,0 +1 @@
+secret