4 Commits

Author SHA1 Message Date
e14b37fe4c Hlæja concept v0.8.0 2025-08-21 15:33:43 +02:00
4f9400c2f9 update helm 2025-08-20 22:59:28 +02:00
18d0ddd159 add kafka for dev 2025-08-20 22:57:23 +02:00
0e8e48dbdb cleanup 2025-08-20 22:52:36 +02:00
19 changed files with 296 additions and 73 deletions

2
.gitignore vendored
View File

@@ -1,5 +1,5 @@
.idea/
cert/
compose/cert/
http/*/http-client.env.json
helm/values/*/
helm/charts/02-dependency/Chart.lock

View File

@@ -11,6 +11,7 @@ volumes:
influx-data:
postgres:
redis:
kafka:
services:
@@ -73,3 +74,71 @@ services:
- 6379:6379
volumes:
- redis:/data
kafka:
image: bitnami/kafka:latest
container_name: Kafka
restart: unless-stopped
ports:
- 9091:9091
environment:
# ---- KRaft single-node (broker + controller) ----
- KAFKA_ENABLE_KRAFT=yes
- KAFKA_CFG_PROCESS_ROLES=broker,controller
- KAFKA_CFG_NODE_ID=1
- KAFKA_KRAFT_CLUSTER_ID=Q1BGMDAwMDAwMDAwMDAwMA
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093
# ---- Listeners ----
# Bind all, three names: INTERNAL (cluster+in-docker clients), EXTERNAL (host clients), CONTROLLER (KRaft)
- KAFKA_CFG_LISTENERS=INTERNAL://:9092,EXTERNAL://:9091,CONTROLLER://:9093
# How clients discover us:
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka:9092,EXTERNAL://localhost:9091
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,INTERNAL:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
# ---- SASL/PLAIN (no TLS) ----
# Enable PLAIN globally and for both listeners
- KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN
- KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN
- KAFKA_CFG_LISTENER_NAME_INTERNAL_SASL_ENABLED_MECHANISMS=PLAIN
- KAFKA_CFG_LISTENER_NAME_EXTERNAL_SASL_ENABLED_MECHANISMS=PLAIN
# Users (Bitnami helper vars wire JAAS for you)
- KAFKA_CLIENT_USERS=kafka
- KAFKA_CLIENT_PASSWORDS=password
- KAFKA_INTER_BROKER_USER=broker
- KAFKA_INTER_BROKER_PASSWORD=password
# Dev-friendly defaults
- KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
- KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=1
- KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
- KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=1
- KAFKA_HEAP_OPTS=-Xms512m -Xmx512m
volumes:
- kafka:/bitnami/kafka
networks:
- develop
kafka-ui:
image: provectuslabs/kafka-ui:latest
container_name: KafkaUi
restart: unless-stopped
depends_on:
- kafka
ports:
- "9090:8080"
environment:
# Optional: allow adding clusters dynamically in UI
- DYNAMIC_CONFIG_ENABLED=true
# First cluster: local Docker Kafka
- KAFKA_CLUSTERS_0_NAME=local
- KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9092
- KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL=SASL_PLAINTEXT
- KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM=PLAIN
- KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password";
networks:
- develop

View File

@@ -8,7 +8,7 @@ networks:
services:
device-registry:
image: ghcr.io/swordsteel/hlaeja-device-registry:0.6.0
image: ghcr.io/swordsteel/hlaeja-device-registry:0.7.0
container_name: DeviceRegistry
restart: unless-stopped
environment:
@@ -61,7 +61,7 @@ services:
- ./cert/device_api_keystore.p12:/app/resources/cert/keystore.p12
registry-api:
image: ghcr.io/swordsteel/hlaeja-registry-api:0.3.0
image: ghcr.io/swordsteel/hlaeja-registry-api:0.4.0
container_name: RegistryApi
restart: unless-stopped
environment:
@@ -77,11 +77,14 @@ services:
- ./cert/account_public_key.pem:/app/resources/cert/public_key.pem
account-registry:
image: ghcr.io/swordsteel/hlaeja-account-registry:0.3.0
image: ghcr.io/swordsteel/hlaeja-account-registry:0.4.0
container_name: AccountRegistry
restart: unless-stopped
environment:
SPRING_PROFILES_ACTIVE: docker
SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT
SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN
SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password";
networks:
- develop
ports:
@@ -91,15 +94,19 @@ services:
- ./cert/account_private_key.pem:/app/resources/cert/private_key.pem
management:
image: ghcr.io/swordsteel/hlaeja-management:0.3.0
image: ghcr.io/swordsteel/hlaeja-management:0.4.0
container_name: Management
restart: unless-stopped
environment:
SPRING_PROFILES_ACTIVE: docker
SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT
SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN
SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: org.apache.kafka.common.security.plain.PlainLoginModule required username="kafka" password="password";
networks:
- develop
ports:
- 9060:8080
- 9060:8443
# mount a local `account_public_key.pem` into the container.
volumes:
- ./cert/account_public_key.pem:/app/resources/cert/public_key.pem
- ./cert/management_keystore.p12:/app/resources/cert/keystore.p12

View File

@@ -49,7 +49,9 @@ graph TD
HDCS[Service] --> HDCD[(Cassandra)]
end
subgraph HAR[Hlæja Account Registry]
direction LR
HARS[Service] --> HARD[(Postgres)]
HARS[Service] --> HDRK[/KAFKA\]
end
end
subgraph HDA[Hlæja Device API]
@@ -59,10 +61,11 @@ graph TD
HRAS[Service]
end
subgraph HM[Hlæja Management]
HMS[Service] -.-> HMR[(Redis)]
direction LR
HMS[Service] --> HMR[(Redis)]
HMK[/KAFKA\] --> HMS[Service]
end
HM --> HAR
HM --> HDR
HM -.-> HDC
@@ -75,7 +78,7 @@ graph TD
HDA --> HDD
```
## Library and Gradle plugin dependency
## Gradle Plugin Dependency
```mermaid
graph RL
@@ -94,8 +97,10 @@ graph RL
PSC[Plugin Service Container]
PSIT[Plugin Service Integration Test]
PSPR[Plugin Service Process Resource]
PUS[Plugin UI Service]
PUSTM[Plugin UI Service Thymeleaf Minify]
end
PLM --> PL
PLP --> PL
PCo ---> PL
@@ -107,54 +112,72 @@ graph RL
PSIT --> PS
PSPR --> PS
PUSTM --> PUS
PS --> PUS
CL[Common Library]
PL ---> CL
CS[Common Service]
PCe -.-> CS
PS --> CS
CUS[Common UI Service]
PUS --> CUS
PCe -.-> CUS
```
## Library And Gradle Plugin Dependency
```mermaid
graph RL
;
HGP[Hlaeja Gradle Plugin]
CML[Common Messages Library]
PL --> CML
HGP --> CML
JL[JWT Library]
PL --> JL
HGP --> JL
TL[Test Library]
PL --> TL
HGP --> TL
DRS[Device Registry Service]
PS --> DRS
PCe --> DRS
TL -.-> DRS
HGP --> DRS
TL --> DRS
CML --> DRS
JL --> DRS
DDS[Device Data Service]
PS --> DDS
HGP --> DDS
TL -.-> DDS
CML --> DDS
DCS[Device Configuration Service]
TL -.-> DCS
PS --> DCS
HGP --> DCS
CML --> DCS
AS[Account Service]
AS[Account Registry Service]
TL --> AS
HGP --> AS
CML --> AS
PS --> AS
PCe --> AS
JL --> AS
DAS[Device API Service]
PS --> DAS
CML --> DAS
JL --> DAS
PCe --> DAS
HGP --> DAS
RAS[Registry API Service]
CML --> RAS
JL --> RAS
PS --> RAS
PCe --> RAS
HGP --> RAS
MUS[Management UI Service]
CML --> MUS
JL --> MUS
PS --> MUS
PCe -.-> MUS
HGP --> MUS
```

49
doc/kafka.md Normal file
View File

@@ -0,0 +1,49 @@
# Kafka
## Creat topic
### Create Client Properties
Run this inside the container.
```shell
cat <<EOF > /tmp/client.properties
security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
username="<CLIENT_USERNAME>" \
password="<CLIENT_PASSWORD>";
EOF
```
### Create Topic
Create custom topic, set `topic` name to be used and `retention.ms` time to live in millisecond
```shell
kafka-topics.sh \
--create \
--bootstrap-server localhost:9092 \
--topic <TOPIC> \
--partitions 1 \
--replication-factor 1 \
--config retention.ms=<TTL MS> \
--command-config /tmp/client.properties
```
### List Topic
Get a list of all topics
```shell
kafka-topics.sh \
--bootstrap-server localhost:9092 \
--command-config /tmp/client.properties \
--list
```
### Access Kafka in K8s
```shell
kubectl -n hlaeja-testing exec -it dependency-kafka-controller-0 -- /bin/bash
```

View File

@@ -1,4 +1,3 @@
namespace: hlaeja-testing
environment: testing
secrets:
@@ -43,4 +42,9 @@ secrets:
# tier: frontend
# keystorePassword: ServiceKeystorePassword
# keystoreFile: ServiceKeystoreFileBase64==
# - name: management-keystore
# app: management
# tier: frontend
# keystorePassword: ServiceKeystorePassword
# keystoreFile: ServiceKeystoreFileBase64==

View File

@@ -24,3 +24,8 @@ dependencies:
- name: redis
version: 21.2.13
repository: "oci://registry-1.docker.io/bitnamicharts"
# https://artifacthub.io/packages/helm/bitnami/kafka
- name: kafka
version: 32.4.2
repository: "oci://registry-1.docker.io/bitnamicharts"

View File

@@ -74,3 +74,12 @@ redis:
# type: LoadBalancer
# loadBalancerIP: 10.0.3.34
kafka:
# clusterId: 00000000-0000-0000-0000-000000000000
sasl:
enabledMechanisms: PLAIN
client:
# users:
# - username
# passwords:
# - password

View File

@@ -9,3 +9,4 @@ metadata:
type: Opaque
data:
SPRING_R2DBC_PASSWORD: {{ .Values.secrets.r2dbcPassword | b64enc | quote }}
SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: {{ .Values.secrets.kafkaPlainLoginModule | b64enc | quote }}

View File

@@ -11,3 +11,6 @@ data:
SPRING_R2DBC_URL: {{ .Values.config.r2dbcUrl | quote }}
SPRING_R2DBC_USERNAME: {{ .Values.config.r2dbcUsername | quote }}
JWT_PRIVATE_KEY: "cert/{{ .Values.jwtPrivetKey.filename }}"
SPRING_KAFKA_BOOTSTRAP_SERVERS: "dependency-kafka:9092"
SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: "SASL_PLAINTEXT"
SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: "PLAIN"

View File

@@ -8,6 +8,7 @@ docker:
secrets:
r2dbcPassword: "password"
kafkaPlainLoginModule: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"clientUsername\" password=\"clientPassword\";"
config:
profiles: testing

View File

@@ -19,10 +19,3 @@ data:
SERVER_SSL_ENABLED: "true"
SERVER_SSL_KEY_STORE: "/app/resources/cert/keystore.p12"
SERVER_SSL_KEY_STORE_TYPE: "PKCS12"
# This was experimental and should be removed in later versions
MANAGEMENT_METRICS_TAGS_APPLICATION: "device-api"
MANAGEMENT_INFLUX_METRICS_EXPORT_ENABLED: "false"
MANAGEMENT_INFLUX_METRICS_EXPORT_URL: "http://influxdb"
# adding this here as it's going to be deleted and is not sued internally
MANAGEMENT_INFLUX_METRICS_EXPORT_TOKEN: "invalidInfluxDbToken=="

View File

@@ -16,10 +16,3 @@ data:
SERVER_SSL_ENABLED: "true"
SERVER_SSL_KEY_STORE: "/app/resources/cert/keystore.p12"
SERVER_SSL_KEY_STORE_TYPE: "PKCS12"
# This was experimental and should be removed in later versions
MANAGEMENT_METRICS_TAGS_APPLICATION: "register-api"
MANAGEMENT_INFLUX_METRICS_EXPORT_ENABLED: "false"
MANAGEMENT_INFLUX_METRICS_EXPORT_URL: "http://influxdb"
# adding this here as it's going to be deleted and is not sued internally
MANAGEMENT_INFLUX_METRICS_EXPORT_TOKEN: "invalidInfluxDbToken=="

View File

@@ -9,3 +9,4 @@ metadata:
type: Opaque
data:
SPRING_DATA_REDIS_PASSWORD: {{ .Values.secrets.redisPassword | b64enc | quote }}
SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG: {{ .Values.secrets.kafkaPlainLoginModule | b64enc | quote }}

View File

@@ -8,7 +8,37 @@ metadata:
tier: frontend
data:
SPRING_PROFILES_ACTIVE: {{ .Values.config.profiles | quote }}
{{- if and .Values.config.timeout (ne .Values.config.timeout "") }}
SPRING_SESSION_TIMEOUT: {{ .Values.config.timeout | quote }}
{{- end }}
{{- if and .Values.config.redis }}
{{- if and .Values.config.redis.database (ne .Values.config.redis.database "") }}
SPRING_DATA_REDIS_DATABASE: {{ .Values.config.redis.database | quote }}
{{- end }}
{{- if and .Values.config.redis.host (ne .Values.config.redis.host "") }}
SPRING_DATA_REDIS_HOST: {{ .Values.config.redis.host | quote }}
ACCOUNT_REGISTRY_URL: {{ .Values.config.accountRegistryUrl | quote }}
DEVICE_REGISTRY_URL: {{ .Values.config.deviceRegistryUrl | quote }}
{{- end }}
{{- end }}
{{- if and .Values.config.error }}
{{- if and .Values.config.error.exception (ne .Values.config.error.exception "") }}
SERVER_ERROR_INCLUDE_EXCEPTION: {{ .Values.config.error.exception | quote }}
{{- end }}
{{- if and .Values.config.error.message (ne .Values.config.error.message "") }}
SERVER_ERROR_INCLUDE_MESSAGE: {{ .Values.config.error.message | quote }}
{{- end }}
{{- if and .Values.config.error.stacktrace (ne .Values.config.error.stacktrace "") }}
SERVER_ERROR_INCLUDE_STACKTRACE: {{ .Values.config.error.stacktrace | quote }}
{{- end }}
{{- end }}
{{- if and .Values.config.accountRegistryUrl (ne .Values.config.accountRegistryUrl "") }}
ACCOUNT_REGISTRY_URL: {{ .Values.config.accountRegistryUrl }}
{{- end }}
{{- if and .Values.config.deviceRegistryUrl (ne .Values.config.deviceRegistryUrl "") }}
DEVICE_REGISTRY_URL: {{ .Values.config.deviceRegistryUrl }}
{{- end }}
SPRING_KAFKA_BOOTSTRAP_SERVERS: "dependency-kafka:9092"
SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: "SASL_PLAINTEXT"
SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: "PLAIN"

View File

@@ -27,17 +27,33 @@ spec:
name: management-environment
- secretRef:
name: management-environment
env:
- name: SERVER_SSL_KEY_STORE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.keystoreRef }}
key: keystore-password
ports:
- containerPort: 8080
volumeMounts:
- name: keystore-volume
mountPath: /app/resources/cert/keystore.p12
subPath: keystore.p12
readOnly: true
- name: jwt-volume
mountPath: /app/resources/cert/{{ .Values.jwtPublicKey.filename }}
subPath: {{ .Values.jwtPublicKey.filename }}
mountPath: /app/resources/cert/public_key.pem
subPath: public_key.pem
readOnly: true
volumes:
- name: keystore-volume
secret:
secretName: {{ .Values.keystoreRef }}
items:
- key: keystore.p12
path: keystore.p12
- name: jwt-volume
secret:
secretName: {{ .Values.jwtPublicKey.name }}
items:
- key: {{ .Values.jwtPublicKey.filename }}
path: {{ .Values.jwtPublicKey.filename }}
- key: {{ .Values.jwtPublicKey.key }}
path: public_key.pem

View File

@@ -15,5 +15,5 @@ spec:
app: management
ports:
- protocol: TCP
port: 80
targetPort: 8080
port: 443
targetPort: 8443

View File

@@ -8,15 +8,24 @@ docker:
secrets:
redisPassword: redisPassword
kafkaPlainLoginModule: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"clientUsername\" password=\"clientPassword\";"
config:
profiles: testing
redis:
database: 2
host: dependency-redis-master
accountRegistryUrl: http://account-register
deviceRegistryUrl: http://device-register
profiles: kubernetes
# error:
# exception: true
# message: always
# stacktrace: always
# timeout: 60m
# redis:
# database:
# port: 6379
# host: dependency-redis-master
# accountRegistryUrl: http://account-register
# deviceRegistryUrl: http://device-register
jwtPublicKey:
name: account-jwt-public-key
filename: public_key.pem
key: public_key.pem
keystoreRef: management-keystore

View File

@@ -1,6 +1,16 @@
# Hlæja Version History
## Hlæja V0.7.0
## Hlæja v0.8.0
- hlaeja-account-registry:0.4.0
- hlaeja-device-api:0.5.0
- hlaeja-device-configuration:0.2.0
- hlaeja-device-data:0.2.0
- hlaeja-device-registry:0.7.0
- hlaeja-management:0.4.0
- hlaeja-registry-api:0.4.0
## Hlæja v0.7.0
- hlaeja-account-registry:0.3.0
- hlaeja-device-api:0.5.0
@@ -10,7 +20,7 @@
- hlaeja-management:0.3.0
- hlaeja-registry-api:0.3.0
## Hlæja V0.6.0
## Hlæja v0.6.0
- hlaeja-account-registry:0.2.0
- hlaeja-device-api:0.4.0
@@ -20,7 +30,7 @@
- hlaeja-management:0.2.0
- hlaeja-registry-api:0.2.0
## Hlæja V0.5.0
## Hlæja v0.5.0
- hlaeja-account-registry:0.2.0
- hlaeja-device-api:0.4.0
@@ -30,7 +40,7 @@
- hlaeja-management:0.1.0
- hlaeja-registry-api:0.2.0
## Hlæja V0.4.0
## Hlæja v0.4.0
- hlaeja-account-registry:0.1.0
- hlaeja-device-api:0.4.0
@@ -39,7 +49,7 @@
- hlaeja-device-registry:0.4.0
- hlaeja-registry-api:0.2.0
## Hlæja V0.3.0
## Hlæja v0.3.0
- hlaeja-device-api:0.3.0
- hlaeja-device-configuration:0.1.0
@@ -47,21 +57,21 @@
- hlaeja-device-registry:0.3.0
- hlaeja-registry-api:0.1.0
## Hlæja V0.2.1
## Hlæja v0.2.1
- hlaeja-device-api:0.2.0
- hlaeja-device-configuration:0.1.0
- hlaeja-device-data:0.1.1
- hlaeja-device-registry:0.2.0
## Hlæja V0.2.0
## Hlæja v0.2.0
- hlaeja-device-api:0.2.0
- hlaeja-device-configuration:0.1.0
- hlaeja-device-data:0.1.0
- hlaeja-device-registry:0.2.0
## Hlæja V0.1.0
## Hlæja v0.1.0
- hlaeja-device-api:0.1.0
- hlaeja-device-configuration:0.1.0