add kafka for dev

compose/development-compose.yml

@@ -11,6 +11,7 @@ volumes:
   influx-data:
   postgres:
   redis:
+  kafka:
 
 services:
 
@@ -73,3 +74,71 @@ services:
       - 6379:6379
     volumes:
       - redis:/data
+
+  kafka:
+    image: bitnami/kafka:latest
+    container_name: Kafka
+    restart: unless-stopped
+    ports:
+      - 9091:9091
+    environment:
+      # ---- KRaft single-node (broker + controller) ----
+      - KAFKA_ENABLE_KRAFT=yes
+      - KAFKA_CFG_PROCESS_ROLES=broker,controller
+      - KAFKA_CFG_NODE_ID=1
+      - KAFKA_KRAFT_CLUSTER_ID=Q1BGMDAwMDAwMDAwMDAwMA
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093
+
+      # ---- Listeners ----
+      # Bind all, three names: INTERNAL (cluster+in-docker clients), EXTERNAL (host clients), CONTROLLER (KRaft)
+      - KAFKA_CFG_LISTENERS=INTERNAL://:9092,EXTERNAL://:9091,CONTROLLER://:9093
+      # How clients discover us:
+      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka:9092,EXTERNAL://localhost:9091
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,INTERNAL:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
+
+      # ---- SASL/PLAIN (no TLS) ----
+      # Enable PLAIN globally and for both listeners
+      - KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN
+      - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN
+      - KAFKA_CFG_LISTENER_NAME_INTERNAL_SASL_ENABLED_MECHANISMS=PLAIN
+      - KAFKA_CFG_LISTENER_NAME_EXTERNAL_SASL_ENABLED_MECHANISMS=PLAIN
+
+      # Users (Bitnami helper vars wire JAAS for you)
+      - KAFKA_CLIENT_USERS=kafka
+      - KAFKA_CLIENT_PASSWORDS=password
+      - KAFKA_INTER_BROKER_USER=broker
+      - KAFKA_INTER_BROKER_PASSWORD=password
+
+      # Dev-friendly defaults
+      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
+      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=1
+      - KAFKA_HEAP_OPTS=-Xms512m -Xmx512m
+    volumes:
+      - kafka:/bitnami/kafka
+    networks:
+      - develop
+
+  kafka-ui:
+    image: provectuslabs/kafka-ui:latest
+    container_name: KafkaUi
+    restart: unless-stopped
+    depends_on:
+      - kafka
+    ports:
+      - "9090:8080"
+    environment:
+      # Optional: allow adding clusters dynamically in UI
+      - DYNAMIC_CONFIG_ENABLED=true
+
+      # First cluster: local Docker Kafka
+      - KAFKA_CLUSTERS_0_NAME=local
+      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9092
+      - KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL=SASL_PLAINTEXT
+      - KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM=PLAIN
+      - KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password";
+    networks:
+      - develop

compose/hlaeja-compose.yml

@@ -8,7 +8,7 @@ networks:
 services:
 
   device-registry:
-    image: ghcr.io/swordsteel/hlaeja-device-registry:0.6.0
+    image: ghcr.io/swordsteel/hlaeja-device-registry:0.7.0
     container_name: DeviceRegistry
     restart: unless-stopped
     environment:
@@ -61,7 +61,7 @@ services:
       - ./cert/device_api_keystore.p12:/app/resources/cert/keystore.p12
 
   registry-api:
-    image: ghcr.io/swordsteel/hlaeja-registry-api:0.3.0
+    image: ghcr.io/swordsteel/hlaeja-registry-api:0.4.0
     container_name: RegistryApi
     restart: unless-stopped
     environment:
@@ -77,11 +77,14 @@ services:
       - ./cert/account_public_key.pem:/app/resources/cert/public_key.pem
 
   account-registry:
-    image: ghcr.io/swordsteel/hlaeja-account-registry:0.3.0
+    image: ghcr.io/swordsteel/hlaeja-account-registry:0.4.0
     container_name: AccountRegistry
     restart: unless-stopped
     environment:
       SPRING_PROFILES_ACTIVE: docker
+      SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT
+      SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN
+      SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password";
     networks:
       - develop
     ports:
@@ -91,15 +94,19 @@ services:
       - ./cert/account_private_key.pem:/app/resources/cert/private_key.pem
 
   management:
-    image: ghcr.io/swordsteel/hlaeja-management:0.3.0
+    image: ghcr.io/swordsteel/hlaeja-management:0.4.0
     container_name: Management
     restart: unless-stopped
     environment:
       SPRING_PROFILES_ACTIVE: docker
+      SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT
+      SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN
+      SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password";
     networks:
       - develop
     ports:
-      - 9060:8080
+      - 9060:8443
     # mount a local `account_public_key.pem` into the container.
     volumes:
       - ./cert/account_public_key.pem:/app/resources/cert/public_key.pem
+      - ./cert/management_keystore.p12:/app/resources/cert/keystore.p12

doc/kafka.md

@@ -0,0 +1,49 @@
+# Kafka
+
+## Creat topic 
+
+### Create Client Properties
+
+Run this inside the container.
+
+```shell
+cat <<EOF > /tmp/client.properties
+security.protocol=SASL_PLAINTEXT
+sasl.mechanism=PLAIN
+sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
+username="<CLIENT_USERNAME>" \
+password="<CLIENT_PASSWORD>";
+EOF
+```
+
+### Create Topic
+
+Create custom topic, set `topic` name to be used and `retention.ms` time to live in millisecond
+
+```shell
+kafka-topics.sh \
+--create \
+--bootstrap-server localhost:9092 \
+--topic <TOPIC> \
+--partitions 1 \
+--replication-factor 1 \
+--config retention.ms=<TTL MS> \
+--command-config /tmp/client.properties
+```
+
+### List Topic
+
+Get a list of all topics
+
+```shell
+kafka-topics.sh \
+--bootstrap-server localhost:9092 \
+--command-config /tmp/client.properties \
+--list
+```
+
+### Access Kafka in K8s
+
+```shell
+kubectl -n hlaeja-testing exec -it dependency-kafka-controller-0 -- /bin/bash
+```