Hlæja concept v0.8.0

.gitignore

@@ -1,5 +1,5 @@
 .idea/
-cert/
+compose/cert/
 http/*/http-client.env.json
 helm/values/*/
 helm/charts/02-dependency/Chart.lock

compose/development-compose.yml

@@ -11,6 +11,7 @@ volumes:
   influx-data:
   postgres:
   redis:
+  kafka:
 
 services:
 
@@ -73,3 +74,71 @@ services:
       - 6379:6379
     volumes:
       - redis:/data
+
+  kafka:
+    image: bitnami/kafka:latest
+    container_name: Kafka
+    restart: unless-stopped
+    ports:
+      - 9091:9091
+    environment:
+      # ---- KRaft single-node (broker + controller) ----
+      - KAFKA_ENABLE_KRAFT=yes
+      - KAFKA_CFG_PROCESS_ROLES=broker,controller
+      - KAFKA_CFG_NODE_ID=1
+      - KAFKA_KRAFT_CLUSTER_ID=Q1BGMDAwMDAwMDAwMDAwMA
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093
+
+      # ---- Listeners ----
+      # Bind all, three names: INTERNAL (cluster+in-docker clients), EXTERNAL (host clients), CONTROLLER (KRaft)
+      - KAFKA_CFG_LISTENERS=INTERNAL://:9092,EXTERNAL://:9091,CONTROLLER://:9093
+      # How clients discover us:
+      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka:9092,EXTERNAL://localhost:9091
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,INTERNAL:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
+
+      # ---- SASL/PLAIN (no TLS) ----
+      # Enable PLAIN globally and for both listeners
+      - KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN
+      - KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN
+      - KAFKA_CFG_LISTENER_NAME_INTERNAL_SASL_ENABLED_MECHANISMS=PLAIN
+      - KAFKA_CFG_LISTENER_NAME_EXTERNAL_SASL_ENABLED_MECHANISMS=PLAIN
+
+      # Users (Bitnami helper vars wire JAAS for you)
+      - KAFKA_CLIENT_USERS=kafka
+      - KAFKA_CLIENT_PASSWORDS=password
+      - KAFKA_INTER_BROKER_USER=broker
+      - KAFKA_INTER_BROKER_PASSWORD=password
+
+      # Dev-friendly defaults
+      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
+      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=1
+      - KAFKA_HEAP_OPTS=-Xms512m -Xmx512m
+    volumes:
+      - kafka:/bitnami/kafka
+    networks:
+      - develop
+
+  kafka-ui:
+    image: provectuslabs/kafka-ui:latest
+    container_name: KafkaUi
+    restart: unless-stopped
+    depends_on:
+      - kafka
+    ports:
+      - "9090:8080"
+    environment:
+      # Optional: allow adding clusters dynamically in UI
+      - DYNAMIC_CONFIG_ENABLED=true
+
+      # First cluster: local Docker Kafka
+      - KAFKA_CLUSTERS_0_NAME=local
+      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9092
+      - KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL=SASL_PLAINTEXT
+      - KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM=PLAIN
+      - KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password";
+    networks:
+      - develop

compose/hlaeja-compose.yml

@@ -8,7 +8,7 @@ networks:
 services:
 
   device-registry:
-    image: ghcr.io/swordsteel/hlaeja-device-registry:0.6.0
+    image: ghcr.io/swordsteel/hlaeja-device-registry:0.7.0
     container_name: DeviceRegistry
     restart: unless-stopped
     environment:
@@ -61,7 +61,7 @@ services:
       - ./cert/device_api_keystore.p12:/app/resources/cert/keystore.p12
 
   registry-api:
-    image: ghcr.io/swordsteel/hlaeja-registry-api:0.3.0
+    image: ghcr.io/swordsteel/hlaeja-registry-api:0.4.0
     container_name: RegistryApi
     restart: unless-stopped
     environment:
@@ -77,11 +77,14 @@ services:
       - ./cert/account_public_key.pem:/app/resources/cert/public_key.pem
 
   account-registry:
-    image: ghcr.io/swordsteel/hlaeja-account-registry:0.3.0
+    image: ghcr.io/swordsteel/hlaeja-account-registry:0.4.0
     container_name: AccountRegistry
     restart: unless-stopped
     environment:
       SPRING_PROFILES_ACTIVE: docker
+      SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT
+      SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN
+      SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password";
     networks:
       - develop
     ports:
@@ -91,15 +94,19 @@ services:
       - ./cert/account_private_key.pem:/app/resources/cert/private_key.pem
 
   management:
-    image: ghcr.io/swordsteel/hlaeja-management:0.3.0
+    image: ghcr.io/swordsteel/hlaeja-management:0.4.0
     container_name: Management
     restart: unless-stopped
     environment:
       SPRING_PROFILES_ACTIVE: docker
+      SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT
+      SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN
+      SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password";
     networks:
       - develop
     ports:
-      - 9060:8080
+      - 9060:8443
     # mount a local `account_public_key.pem` into the container.
     volumes:
       - ./cert/account_public_key.pem:/app/resources/cert/public_key.pem
+      - ./cert/management_keystore.p12:/app/resources/cert/keystore.p12

doc/dependency.md

@@ -49,7 +49,9 @@ graph TD
       HDCS[Service] --> HDCD[(Cassandra)]
     end
     subgraph HAR[Hlæja Account Registry]
+      direction LR
       HARS[Service] --> HARD[(Postgres)]
+      HARS[Service] --> HDRK[/KAFKA\]
     end
   end
   subgraph HDA[Hlæja Device API]
@@ -59,10 +61,11 @@ graph TD
     HRAS[Service]
   end
   subgraph HM[Hlæja Management]
-    HMS[Service] -.-> HMR[(Redis)]
+    direction LR
+    HMS[Service] --> HMR[(Redis)]
+    HMK[/KAFKA\] --> HMS[Service]
   end
 
-
   HM --> HAR
   HM --> HDR
   HM -.-> HDC
@@ -75,7 +78,7 @@ graph TD
   HDA --> HDD
 ```
 
-## Library and Gradle plugin dependency
+## Gradle Plugin Dependency
 
 ```mermaid
 graph RL
@@ -94,8 +97,10 @@ graph RL
     PSC[Plugin Service Container]
     PSIT[Plugin Service Integration Test]
     PSPR[Plugin Service Process Resource]
+    PUS[Plugin UI Service]
+    PUSTM[Plugin UI Service Thymeleaf Minify]
   end
-  
+
   PLM --> PL
   PLP --> PL
   PCo ---> PL
@@ -107,54 +112,72 @@ graph RL
   PSIT --> PS
   PSPR --> PS
 
+  PUSTM --> PUS
+  PS --> PUS
+
+  CL[Common Library]
+  PL ---> CL
+
+  CS[Common Service]
+  PCe -.-> CS
+  PS --> CS
+
+  CUS[Common UI Service]
+  PUS --> CUS
+  PCe -.-> CUS
+```
+
+## Library And Gradle Plugin Dependency
+
+```mermaid
+graph RL
+;
+  
+  HGP[Hlaeja Gradle Plugin]
+
   CML[Common Messages Library]
-  PL --> CML
+  HGP --> CML
-
+  
   JL[JWT Library]
-  PL --> JL
+  HGP --> JL
-
+  
   TL[Test Library]
-  PL --> TL
+  HGP --> TL
 
   DRS[Device Registry Service]
-  PS --> DRS
+  HGP --> DRS
-  PCe --> DRS
+  TL --> DRS
-  TL -.-> DRS
   CML --> DRS
   JL --> DRS
-
+    
   DDS[Device Data Service]
-  PS --> DDS
+  HGP --> DDS
   TL -.-> DDS
   CML --> DDS
-
+  
   DCS[Device Configuration Service]
   TL -.-> DCS
-  PS --> DCS
+  HGP --> DCS
   CML --> DCS
-
+  
-  AS[Account Service]
+  AS[Account Registry Service]
   TL --> AS
+  HGP --> AS
   CML --> AS
-  PS --> AS
-  PCe --> AS
   JL --> AS
-
+  
   DAS[Device API Service]
-  PS --> DAS
   CML --> DAS
   JL --> DAS
-  PCe --> DAS
+  HGP --> DAS
-
+  
   RAS[Registry API Service]
   CML --> RAS
   JL --> RAS
-  PS --> RAS
+  HGP --> RAS
-  PCe --> RAS
+  
-
   MUS[Management UI Service]
   CML --> MUS
   JL --> MUS
-  PS --> MUS
+  HGP --> MUS
-  PCe -.-> MUS
 ```

doc/kafka.md

@@ -0,0 +1,49 @@
+# Kafka
+
+## Creat topic 
+
+### Create Client Properties
+
+Run this inside the container.
+
+```shell
+cat <<EOF > /tmp/client.properties
+security.protocol=SASL_PLAINTEXT
+sasl.mechanism=PLAIN
+sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
+username="<CLIENT_USERNAME>" \
+password="<CLIENT_PASSWORD>";
+EOF
+```
+
+### Create Topic
+
+Create custom topic, set `topic` name to be used and `retention.ms` time to live in millisecond
+
+```shell
+kafka-topics.sh \
+--create \
+--bootstrap-server localhost:9092 \
+--topic <TOPIC> \
+--partitions 1 \
+--replication-factor 1 \
+--config retention.ms=<TTL MS> \
+--command-config /tmp/client.properties
+```
+
+### List Topic
+
+Get a list of all topics
+
+```shell
+kafka-topics.sh \
+--bootstrap-server localhost:9092 \
+--command-config /tmp/client.properties \
+--list
+```
+
+### Access Kafka in K8s
+
+```shell
+kubectl -n hlaeja-testing exec -it dependency-kafka-controller-0 -- /bin/bash
+```

helm/charts/01-secrets/values.yaml

@@ -1,4 +1,3 @@
-namespace: hlaeja-testing
 environment: testing
 
 secrets:
@@ -43,4 +42,9 @@ secrets:
 #      tier: frontend
 #      keystorePassword: ServiceKeystorePassword
 #      keystoreFile: ServiceKeystoreFileBase64==
+#    - name: management-keystore
+#      app: management
+#      tier: frontend
+#      keystorePassword: ServiceKeystorePassword
+#      keystoreFile: ServiceKeystoreFileBase64==
 

helm/charts/02-dependency/Chart.yaml

@@ -24,3 +24,8 @@ dependencies:
   - name: redis
     version: 21.2.13
     repository: "oci://registry-1.docker.io/bitnamicharts"
+
+  # https://artifacthub.io/packages/helm/bitnami/kafka
+  - name: kafka
+    version: 32.4.2
+    repository: "oci://registry-1.docker.io/bitnamicharts"

helm/charts/02-dependency/values.yaml

@@ -74,3 +74,12 @@ redis:
 #      type: LoadBalancer
 #      loadBalancerIP: 10.0.3.34
 
+kafka:
+#  clusterId: 00000000-0000-0000-0000-000000000000
+  sasl:
+    enabledMechanisms: PLAIN
+    client:
+#      users:
+#        - username
+#      passwords:
+#        - password

helm/charts/03-account-registry/templates/01-secret.yaml

@@ -9,3 +9,4 @@ metadata:
 type: Opaque
 data:
   SPRING_R2DBC_PASSWORD: {{ .Values.secrets.r2dbcPassword | b64enc | quote }}
+  SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: {{ .Values.secrets.kafkaPlainLoginModule | b64enc | quote }}

helm/charts/03-account-registry/templates/02-configmap.yaml

@@ -11,3 +11,6 @@ data:
   SPRING_R2DBC_URL: {{ .Values.config.r2dbcUrl | quote }}
   SPRING_R2DBC_USERNAME: {{ .Values.config.r2dbcUsername | quote }}
   JWT_PRIVATE_KEY: "cert/{{ .Values.jwtPrivetKey.filename }}"
+  SPRING_KAFKA_BOOTSTRAP_SERVERS: "dependency-kafka:9092"
+  SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: "SASL_PLAINTEXT"
+  SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: "PLAIN"

helm/charts/03-account-registry/values.yaml

@@ -8,6 +8,7 @@ docker:
 
 secrets:
   r2dbcPassword: "password"
+  kafkaPlainLoginModule: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"clientUsername\" password=\"clientPassword\";"
 
 config:
   profiles: testing

helm/charts/07-device-api/templates/02-configmap.yaml

@@ -19,10 +19,3 @@ data:
   SERVER_SSL_ENABLED: "true"
   SERVER_SSL_KEY_STORE: "/app/resources/cert/keystore.p12"
   SERVER_SSL_KEY_STORE_TYPE: "PKCS12"
-
-  # This was experimental and should be removed in later versions
-  MANAGEMENT_METRICS_TAGS_APPLICATION: "device-api"
-  MANAGEMENT_INFLUX_METRICS_EXPORT_ENABLED: "false"
-  MANAGEMENT_INFLUX_METRICS_EXPORT_URL: "http://influxdb"
-  # adding this here as it's going to be deleted and is not sued internally
-  MANAGEMENT_INFLUX_METRICS_EXPORT_TOKEN: "invalidInfluxDbToken=="

helm/charts/08-registry-api/templates/01-configmap.yaml

@@ -16,10 +16,3 @@ data:
   SERVER_SSL_ENABLED: "true"
   SERVER_SSL_KEY_STORE: "/app/resources/cert/keystore.p12"
   SERVER_SSL_KEY_STORE_TYPE: "PKCS12"
-
-  # This was experimental and should be removed in later versions
-  MANAGEMENT_METRICS_TAGS_APPLICATION: "register-api"
-  MANAGEMENT_INFLUX_METRICS_EXPORT_ENABLED: "false"
-  MANAGEMENT_INFLUX_METRICS_EXPORT_URL: "http://influxdb"
-  # adding this here as it's going to be deleted and is not sued internally
-  MANAGEMENT_INFLUX_METRICS_EXPORT_TOKEN: "invalidInfluxDbToken=="

helm/charts/09-management/templates/01-secret.yaml

@@ -9,3 +9,4 @@ metadata:
 type: Opaque
 data:
   SPRING_DATA_REDIS_PASSWORD: {{ .Values.secrets.redisPassword | b64enc | quote }}
+  SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: {{ .Values.secrets.kafkaPlainLoginModule | b64enc | quote }}

helm/charts/09-management/templates/02-configmap.yaml

@@ -8,7 +8,37 @@ metadata:
     tier: frontend
 data:
   SPRING_PROFILES_ACTIVE: {{ .Values.config.profiles | quote }}
+  {{- if and .Values.config.timeout (ne .Values.config.timeout "") }}
+  SPRING_SESSION_TIMEOUT: {{ .Values.config.timeout | quote }}
+  {{- end }}
+
+  {{- if and .Values.config.redis }}
+  {{- if and .Values.config.redis.database (ne .Values.config.redis.database "") }}
   SPRING_DATA_REDIS_DATABASE: {{ .Values.config.redis.database | quote }}
+  {{- end }}
+  {{- if and .Values.config.redis.host (ne .Values.config.redis.host "") }}
   SPRING_DATA_REDIS_HOST: {{ .Values.config.redis.host | quote }}
-  ACCOUNT_REGISTRY_URL: {{ .Values.config.accountRegistryUrl | quote }}
+  {{- end }}
-  DEVICE_REGISTRY_URL: {{ .Values.config.deviceRegistryUrl | quote }}
+  {{- end }}
+
+  {{- if and .Values.config.error }}
+  {{- if and .Values.config.error.exception (ne .Values.config.error.exception "") }}
+  SERVER_ERROR_INCLUDE_EXCEPTION: {{ .Values.config.error.exception | quote }}
+  {{- end }}
+  {{- if and .Values.config.error.message (ne .Values.config.error.message "") }}
+  SERVER_ERROR_INCLUDE_MESSAGE: {{ .Values.config.error.message | quote }}
+  {{- end }}
+  {{- if and .Values.config.error.stacktrace (ne .Values.config.error.stacktrace "") }}
+  SERVER_ERROR_INCLUDE_STACKTRACE: {{ .Values.config.error.stacktrace | quote }}
+  {{- end }}
+  {{- end }}
+
+  {{- if and .Values.config.accountRegistryUrl (ne .Values.config.accountRegistryUrl "") }}
+  ACCOUNT_REGISTRY_URL: {{ .Values.config.accountRegistryUrl }}
+  {{- end }}
+  {{- if and .Values.config.deviceRegistryUrl (ne .Values.config.deviceRegistryUrl "") }}
+  DEVICE_REGISTRY_URL: {{ .Values.config.deviceRegistryUrl }}
+  {{- end }}
+  SPRING_KAFKA_BOOTSTRAP_SERVERS: "dependency-kafka:9092"
+  SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: "SASL_PLAINTEXT"
+  SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: "PLAIN"

helm/charts/09-management/templates/03-deployment.yaml

@@ -27,17 +27,33 @@ spec:
                 name: management-environment
             - secretRef:
                 name: management-environment
+          env:
+            - name: SERVER_SSL_KEY_STORE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.keystoreRef }}
+                  key: keystore-password
           ports:
             - containerPort: 8080
           volumeMounts:
+            - name: keystore-volume
+              mountPath: /app/resources/cert/keystore.p12
+              subPath: keystore.p12
+              readOnly: true
             - name: jwt-volume
-              mountPath: /app/resources/cert/{{ .Values.jwtPublicKey.filename }}
+              mountPath: /app/resources/cert/public_key.pem
-              subPath: {{ .Values.jwtPublicKey.filename }}
+              subPath: public_key.pem
               readOnly: true
       volumes:
+        - name: keystore-volume
+          secret:
+            secretName: {{ .Values.keystoreRef }}
+            items:
+              - key: keystore.p12
+                path: keystore.p12
         - name: jwt-volume
           secret:
             secretName: {{ .Values.jwtPublicKey.name }}
             items:
-              - key: {{ .Values.jwtPublicKey.filename }}
+              - key: {{ .Values.jwtPublicKey.key }}
-                path: {{ .Values.jwtPublicKey.filename }}
+                path: public_key.pem

helm/charts/09-management/templates/03-service.yaml

@@ -15,5 +15,5 @@ spec:
     app: management
   ports:
     - protocol: TCP
-      port: 80
+      port: 443
-      targetPort: 8080
+      targetPort: 8443

helm/charts/09-management/values.yaml

@@ -8,15 +8,24 @@ docker:
 
 secrets:
   redisPassword: redisPassword
+  kafkaPlainLoginModule: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"clientUsername\" password=\"clientPassword\";"
 
 config:
-  profiles: testing
+  profiles: kubernetes
-  redis:
+#  error:
-    database: 2
+#    exception: true
-    host: dependency-redis-master
+#    message: always
-  accountRegistryUrl: http://account-register
+#    stacktrace: always
-  deviceRegistryUrl: http://device-register
+#  timeout: 60m
+#  redis:
+#    database:
+#    port: 6379
+#    host: dependency-redis-master
+#  accountRegistryUrl: http://account-register
+#  deviceRegistryUrl: http://device-register
 
 jwtPublicKey:
   name: account-jwt-public-key
-  filename: public_key.pem
+  key: public_key.pem
+
+keystoreRef: management-keystore

version.md

@@ -1,6 +1,16 @@
 # Hlæja Version History
 
-## Hlæja V0.7.0
+## Hlæja v0.8.0
+
+- hlaeja-account-registry:0.4.0
+- hlaeja-device-api:0.5.0
+- hlaeja-device-configuration:0.2.0
+- hlaeja-device-data:0.2.0
+- hlaeja-device-registry:0.7.0
+- hlaeja-management:0.4.0
+- hlaeja-registry-api:0.4.0
+
+## Hlæja v0.7.0
 
 - hlaeja-account-registry:0.3.0
 - hlaeja-device-api:0.5.0
@@ -10,7 +20,7 @@
 - hlaeja-management:0.3.0
 - hlaeja-registry-api:0.3.0
 
-## Hlæja V0.6.0
+## Hlæja v0.6.0
 
 - hlaeja-account-registry:0.2.0
 - hlaeja-device-api:0.4.0
@@ -20,7 +30,7 @@
 - hlaeja-management:0.2.0
 - hlaeja-registry-api:0.2.0
 
-## Hlæja V0.5.0
+## Hlæja v0.5.0
 
 - hlaeja-account-registry:0.2.0
 - hlaeja-device-api:0.4.0
@@ -30,7 +40,7 @@
 - hlaeja-management:0.1.0
 - hlaeja-registry-api:0.2.0
 
-## Hlæja V0.4.0
+## Hlæja v0.4.0
 
 - hlaeja-account-registry:0.1.0
 - hlaeja-device-api:0.4.0
@@ -39,7 +49,7 @@
 - hlaeja-device-registry:0.4.0
 - hlaeja-registry-api:0.2.0
 
-## Hlæja V0.3.0
+## Hlæja v0.3.0
 
 - hlaeja-device-api:0.3.0
 - hlaeja-device-configuration:0.1.0
@@ -47,21 +57,21 @@
 - hlaeja-device-registry:0.3.0
 - hlaeja-registry-api:0.1.0
 
-## Hlæja V0.2.1
+## Hlæja v0.2.1
 
 - hlaeja-device-api:0.2.0
 - hlaeja-device-configuration:0.1.0
 - hlaeja-device-data:0.1.1
 - hlaeja-device-registry:0.2.0
 
-## Hlæja V0.2.0
+## Hlæja v0.2.0
 
 - hlaeja-device-api:0.2.0
 - hlaeja-device-configuration:0.1.0
 - hlaeja-device-data:0.1.0
 - hlaeja-device-registry:0.2.0
 
-## Hlæja V0.1.0
+## Hlæja v0.1.0
 
 - hlaeja-device-api:0.1.0
 - hlaeja-device-configuration:0.1.0