refactor!: ensure compatibility with the latest Langfuse version (#93)

This pull request introduces significant updates to the RAG
infrastructure and evaluation codebase, primarily focused on improving
Langfuse integration, updating dependencies, and refactoring
configuration for external service management. The changes enhance
traceability, simplify environment setup, and ensure compatibility with
the latest Langfuse version.

**Langfuse Integration and Evaluation Refactor:**

* Refactored the Langfuse evaluation workflow in
`langfuse_ragas_evaluator.py` to use context managers
(`start_as_current_generation` and `run`) for better trace and
generation handling, and replaced manual scoring and linking with direct
trace scoring.
[[1]](diffhunk://#diff-fe148c64db46f85d0ef8df1dfbebe80d8a816c2025bef7c15f7ae9282457ac81R165-R198)
[[2]](diffhunk://#diff-fe148c64db46f85d0ef8df1dfbebe80d8a816c2025bef7c15f7ae9282457ac81L213-R219)
* Updated the dataset creation method to use the new Langfuse API
signature (`create_dataset(name=...)`).

**Dependency Updates:**

* Upgraded the `langfuse` Python package from `3.0.0` to `3.3.4` across
all relevant `pyproject.toml` files for improved compatibility and
features.
[[1]](diffhunk://#diff-9c5aeb0db77c2eec077d07ddc3b3810ae1a4a1e50ee7061fba37a46706c513fbL107-R107)
[[2]](diffhunk://#diff-9c4162cc1c16dd4c7ec5e95e79df285e8c0882a1db7ff2892c746a0537d26c36L34-R34)
[[3]](diffhunk://#diff-b19ab043535569caf9345971969d115d6515ae951a21b00a278145a28230fba1L21-R21)

**Infrastructure Configuration Improvements:**

* Updated the Langfuse Helm chart dependency version from `0.12.1` to
`1.5.1` in `Chart.yaml`.
* Overhauled `values.yaml` to externalize configuration for PostgreSQL,
Redis (KeyDB), ClickHouse, and S3/MinIO, removing embedded environment
variable definitions and introducing dedicated configuration sections
for each service.
[[1]](diffhunk://#diff-673dd2d3d4e66a8fd4e45f9c1c9900711313f946bf8b6a89e96c954988fc14f3L448-R480)
[[2]](diffhunk://#diff-673dd2d3d4e66a8fd4e45f9c1c9900711313f946bf8b6a89e96c954988fc14f3L482-R497)
[[3]](diffhunk://#diff-673dd2d3d4e66a8fd4e45f9c1c9900711313f946bf8b6a89e96c954988fc14f3R512-R582)
* Removed internal deployment of MinIO and Valkey, and clarified that
KeyDB is used for Redis functionality.
[[1]](diffhunk://#diff-673dd2d3d4e66a8fd4e45f9c1c9900711313f946bf8b6a89e96c954988fc14f3L448-R480)
[[2]](diffhunk://#diff-673dd2d3d4e66a8fd4e45f9c1c9900711313f946bf8b6a89e96c954988fc14f3L540-L543)
[[3]](diffhunk://#diff-673dd2d3d4e66a8fd4e45f9c1c9900711313f946bf8b6a89e96c954988fc14f3R615-R619)

**Tracing Enhancements:**

* Improved tracing in `traced_runnable.py` by updating span names to
reflect the actual chain class and including input/output in the trace,
enabling more detailed observability.

Author: a-klos

infrastructure/rag/Chart.lock

@@ -1,7 +1,7 @@
 dependencies:
 - name: langfuse
   repository: https://langfuse.github.io/langfuse-k8s
-  version: 0.12.1
+  version: 1.5.1
 - name: qdrant
   repository: https://qdrant.github.io/qdrant-helm
   version: 1.15.4
@@ -14,5 +14,5 @@ dependencies:
 - name: ollama
   repository: https://otwld.github.io/ollama-helm/
   version: 1.27.0
-digest: sha256:27cfa430e217d545d1d85479029dc83bf946e32ae1744d35eb94050f1c4714c4
-generated: "2025-09-24T12:23:52.977733+02:00"
+digest: sha256:86652599791c7f3ae4e66e6ed4404630d733cf35503c4b94b90da9c3c950c7fb
+generated: "2025-10-07T10:23:59.709679+02:00"

infrastructure/rag/Chart.yaml

@@ -10,7 +10,7 @@ appVersion: "1.0.0"
 dependencies:
 - name: langfuse
   repository: https://langfuse.github.io/langfuse-k8s
-  version: "0.12.1"
+  version: "1.5.1"
   condition: features.langfuse.enabled
 - name: qdrant
   version: 1.15.4

infrastructure/rag/values.yaml

@@ -450,39 +450,39 @@ shared:
 
 
 langfuse:
-  minio:
-    image:
-      repository: bitnamilegacy/minio
-    deploy: false
-  valkey:
-    image:
-      repository: bitnamilegacy/valkey
-    deploy: false #<--- keydb is used instead of valkey
-  image:
-    repository: ghcr.io/langfuse/langfuse
-    pullPolicy: Always
-    tag: "3.27.2"
-  postgresql:
-    image:
-      repository: bitnamilegacy/postgresql
-    deploy: true
-    auth:
-      username: postgres
-      password: postgres
-      database: langfuse
-  clickhouse:
-    image:
-      repository: bitnamilegacy/clickhouse
-    zookeeper:
+  # Core Langfuse Configuration
+  langfuse:
+    # Used to hash API keys
+    salt:
+      value: "changeme"
+
+    # Authentication settings
+    features:
+      telemetryEnabled: true
+      signUpDisabled: false
+
+    # Web deployment configuration
+    web:
       image:
-        repository: bitnamilegacy/zookeeper
+        repository: langfuse/langfuse
+        tag: "3.106.3"
+        pullPolicy: Always
 
+    # Worker deployment configuration
+    worker:
+      image:
+        repository: langfuse/langfuse-worker
+        tag: "3.106.3"
+        pullPolicy: Always
+      port: 3030
 
-  langfuse:
+    # NextAuth configuration
     nextauth:
       url: http://localhost:3000
-      secret: changeme
-    salt: changeme
+      secret:
+        value: "changeme"
+
+    # Additional environment variables (only for init values)
     additionalEnv:
     - name: LANGFUSE_INIT_ORG_ID
       value: ""
@@ -498,33 +498,8 @@ langfuse:
       value: ""
     - name: LANGFUSE_INIT_USER_PASSWORD
       value: ""
-    # REDIS
-    - name: "REDIS_CONNECTION_STRING"
-      value: "redis://rag-keydb:6379"
-    # CLICKHOUSE
-    - name: "CLICKHOUSE_MIGRATION_URL"
-      value: "clickhouse://rag-clickhouse:9000"
-    - name: "CLICKHOUSE_URL"
-      value: "http://rag-clickhouse:8123"
-    - name: "CLICKHOUSE_USER"
-      value: "default"
-    - name: "CLICKHOUSE_PASSWORD"
-      value: "changeme"
-    # S3 / MinIO
-    - name: "LANGFUSE_S3_EVENT_UPLOAD_ENABLED"
-      value: "true"
-    - name: "LANGFUSE_S3_EVENT_UPLOAD_BUCKET"
-      value: "langfuse"
-    - name: "LANGFUSE_S3_EVENT_UPLOAD_REGION"
-      value: "auto"
-    - name: "LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID"
-      value: "admin"
-    - name: "LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY"
-      value: "adminpassword"
-    - name: "LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT"
-      value: "http://rag-minio:9000"
-    - name: "LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE"
-      value: "true"
+
+    # Additional init containers
     extraInitContainers:
     - name: wait-for-postgres
       image: busybox
@@ -539,6 +514,77 @@ langfuse:
       # Define a reasonable timeout in case PostgreSQL fails to come up
       timeoutSeconds: 300
 
+  # PostgreSQL Configuration (use external PostgreSQL)
+  postgresql:
+    deploy: true
+    host: "rag-postgresql"
+    port: 5432
+    auth:
+      username: postgres
+      password: postgres
+      database: langfuse
+
+  # Redis Configuration (external KeyDB)
+  redis:
+    deploy: false
+    host: "rag-keydb"
+    port: 6379
+    auth:
+      username: "default"
+      password: ""
+
+  # ClickHouse Configuration (external ClickHouse)
+  clickhouse:
+    deploy: true
+    host: "rag-clickhouse"
+    httpPort: 8123
+    nativePort: 9000
+    auth:
+      username: "default"
+      password: "changeme"
+    migration:
+      url: "clickhouse://rag-clickhouse:9000"
+      ssl: false
+      autoMigrate: true
+    resources:
+      limits:
+        cpu: "2"
+        memory: "8Gi"
+      requests:
+        cpu: "2"
+        memory: "4Gi"
+
+    zookeeper:
+      resources:
+        limits:
+          cpu: "2"
+          memory: "2Gi"
+        requests:
+          cpu: "1"
+          memory: "1Gi"
+
+  # S3/MinIO Configuration (external MinIO)
+  s3:
+    deploy: false
+    bucket: "langfuse"
+    region: "auto"
+    endpoint: "http://rag-minio:9000"
+    forcePathStyle: true
+    accessKeyId:
+      value: "admin"
+    secretAccessKey:
+      value: "adminpassword"
+    eventUpload:
+      enabled: true
+      bucket: "langfuse"
+      region: "auto"
+      endpoint: "http://rag-minio:9000"
+      forcePathStyle: true
+      accessKeyId:
+        value: "admin"
+      secretAccessKey:
+        value: "adminpassword"
+
 minio:
   image:
     repository: bitnamilegacy/minio
@@ -558,13 +604,9 @@ minio:
     enabled: false
   mode: standalone
 
-
-
-
-
 ollama:
   image:
-    tag: 0.5.1
+    tag: 0.12.3
   ollama:
     models:
       pull:
@@ -577,3 +619,8 @@ ollama:
 qdrant:
   image:
     tag: v1.15.4
+
+keydb:
+  multiMaster: "no"
+  activeReplicas: "no"
+  nodes: 1