diff --git a/compose/development-compose.yml b/compose/development-compose.yml index 6e6fed8..b0d9327 100644 --- a/compose/development-compose.yml +++ b/compose/development-compose.yml @@ -11,6 +11,7 @@ volumes: influx-data: postgres: redis: + kafka: services: @@ -73,3 +74,71 @@ services: - 6379:6379 volumes: - redis:/data + + kafka: + image: bitnami/kafka:latest + container_name: Kafka + restart: unless-stopped + ports: + - 9091:9091 + environment: + # ---- KRaft single-node (broker + controller) ---- + - KAFKA_ENABLE_KRAFT=yes + - KAFKA_CFG_PROCESS_ROLES=broker,controller + - KAFKA_CFG_NODE_ID=1 + - KAFKA_KRAFT_CLUSTER_ID=Q1BGMDAwMDAwMDAwMDAwMA + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093 + + # ---- Listeners ---- + # Bind all, three names: INTERNAL (cluster+in-docker clients), EXTERNAL (host clients), CONTROLLER (KRaft) + - KAFKA_CFG_LISTENERS=INTERNAL://:9092,EXTERNAL://:9091,CONTROLLER://:9093 + # How clients discover us: + - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka:9092,EXTERNAL://localhost:9091 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,INTERNAL:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL + + # ---- SASL/PLAIN (no TLS) ---- + # Enable PLAIN globally and for both listeners + - KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN + - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN + - KAFKA_CFG_LISTENER_NAME_INTERNAL_SASL_ENABLED_MECHANISMS=PLAIN + - KAFKA_CFG_LISTENER_NAME_EXTERNAL_SASL_ENABLED_MECHANISMS=PLAIN + + # Users (Bitnami helper vars wire JAAS for you) + - KAFKA_CLIENT_USERS=kafka + - KAFKA_CLIENT_PASSWORDS=password + - KAFKA_INTER_BROKER_USER=broker + - KAFKA_INTER_BROKER_PASSWORD=password + + # Dev-friendly defaults + - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true + - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 + - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=1 + - KAFKA_HEAP_OPTS=-Xms512m -Xmx512m + volumes: + - kafka:/bitnami/kafka + networks: + - develop + + kafka-ui: + image: provectuslabs/kafka-ui:latest + container_name: KafkaUi + restart: unless-stopped + depends_on: + - kafka + ports: + - "9090:8080" + environment: + # Optional: allow adding clusters dynamically in UI + - DYNAMIC_CONFIG_ENABLED=true + + # First cluster: local Docker Kafka + - KAFKA_CLUSTERS_0_NAME=local + - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9092 + - KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL=SASL_PLAINTEXT + - KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM=PLAIN + - KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password"; + networks: + - develop diff --git a/compose/hlaeja-compose.yml b/compose/hlaeja-compose.yml index 0c89e3e..e660cd0 100644 --- a/compose/hlaeja-compose.yml +++ b/compose/hlaeja-compose.yml @@ -8,7 +8,7 @@ networks: services: device-registry: - image: ghcr.io/swordsteel/hlaeja-device-registry:0.6.0 + image: ghcr.io/swordsteel/hlaeja-device-registry:0.7.0 container_name: DeviceRegistry restart: unless-stopped environment: @@ -61,7 +61,7 @@ services: - ./cert/device_api_keystore.p12:/app/resources/cert/keystore.p12 registry-api: - image: ghcr.io/swordsteel/hlaeja-registry-api:0.3.0 + image: ghcr.io/swordsteel/hlaeja-registry-api:0.4.0 container_name: RegistryApi restart: unless-stopped environment: @@ -77,11 +77,14 @@ services: - ./cert/account_public_key.pem:/app/resources/cert/public_key.pem account-registry: - image: ghcr.io/swordsteel/hlaeja-account-registry:0.3.0 + image: ghcr.io/swordsteel/hlaeja-account-registry:0.4.0 container_name: AccountRegistry restart: unless-stopped environment: SPRING_PROFILES_ACTIVE: docker + SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT + SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN + SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password"; networks: - develop ports: @@ -91,15 +94,19 @@ services: - ./cert/account_private_key.pem:/app/resources/cert/private_key.pem management: - image: ghcr.io/swordsteel/hlaeja-management:0.3.0 + image: ghcr.io/swordsteel/hlaeja-management:0.4.0 container_name: Management restart: unless-stopped environment: SPRING_PROFILES_ACTIVE: docker + SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT + SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN + SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password"; networks: - develop ports: - - 9060:8080 + - 9060:8443 # mount a local `account_public_key.pem` into the container. volumes: - ./cert/account_public_key.pem:/app/resources/cert/public_key.pem + - ./cert/management_keystore.p12:/app/resources/cert/keystore.p12 diff --git a/doc/kafka.md b/doc/kafka.md new file mode 100644 index 0000000..bb79b2b --- /dev/null +++ b/doc/kafka.md @@ -0,0 +1,49 @@ +# Kafka + +## Creat topic + +### Create Client Properties + +Run this inside the container. + +```shell +cat < /tmp/client.properties +security.protocol=SASL_PLAINTEXT +sasl.mechanism=PLAIN +sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \ +username="" \ +password=""; +EOF +``` + +### Create Topic + +Create custom topic, set `topic` name to be used and `retention.ms` time to live in millisecond + +```shell +kafka-topics.sh \ +--create \ +--bootstrap-server localhost:9092 \ +--topic \ +--partitions 1 \ +--replication-factor 1 \ +--config retention.ms= \ +--command-config /tmp/client.properties +``` + +### List Topic + +Get a list of all topics + +```shell +kafka-topics.sh \ +--bootstrap-server localhost:9092 \ +--command-config /tmp/client.properties \ +--list +``` + +### Access Kafka in K8s + +```shell +kubectl -n hlaeja-testing exec -it dependency-kafka-controller-0 -- /bin/bash +```