Improve configuration of transform and partitioner

Author: josephcummings

.gitmodules

@@ -45,4 +45,69 @@ Return the proper Replicator image name
 {{- $repositoryName := .Values.image.repository -}}
 {{- $tag := .Values.image.tag | toString -}}
 {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
+{{- end -}}
+
+{{/*
+Checks if replicator is configured to use javascript transform
+*/}}
+{{- define "replicator.shouldUseJavascriptTransform" -}}
+{{- if and 
+    .Values.replicator.transform 
+    (eq .Values.replicator.transform.type "js") 
+    .Values.replicator.transform.config 
+    .Values.transformJs 
+-}}
+{{- print "true" }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Checks if replicator is configured to use a custom partitioner
+*/}}
+{{- define "replicator.shouldUseCustomPartitioner" -}}
+{{- if and 
+    .Values.replicator.sink.partitioner 
+    .Values.partitionerJs 
+-}}
+{{- print "true" }}
+{{- end -}}
+{{- end -}}
+
+{{- define "replicator.transform.filename" -}}
+{{- if eq (include "replicator.shouldUseJavascriptTransform" .) "true" -}}
+{{ printf "%s" (include "replicator.helpers.filename" .Values.replicator.transform.config) }}
+{{- end -}}
+{{- end -}}
+
+{{- define "replicator.transform.filepath" -}}
+{{- if eq (include "replicator.shouldUseJavascriptTransform" .) "true" -}}
+{{ printf "%s" (include "replicator.helpers.cleansePath" .Values.replicator.transform.config) }}
+{{- end -}}
+{{- end -}}
+
+{{- define "replicator.sink.partitioner.filename" -}}
+{{- if eq (include "replicator.shouldUseCustomPartitioner" .) "true" -}}
+{{ printf "%s" (include "replicator.helpers.filename" .Values.replicator.sink.partitioner) }}
+{{- end -}}
+{{- end -}}
+
+{{- define "replicator.sink.partitioner.filepath" -}}
+{{- if eq (include "replicator.shouldUseCustomPartitioner" .) "true" -}}
+{{ printf "%s" (include "replicator.helpers.cleansePath" .Values.replicator.sink.partitioner) }}
+{{- end -}}
+{{- end -}}
+
+{{- define "replicator.helpers.filename" -}}
+{{- $file := . -}}
+{{- $filename := last (splitList "/" $file) -}}
+{{- $filename -}}
+{{- end -}}
+
+{{- define "replicator.helpers.cleansePath" -}}
+{{- $path := . -}}
+{{- $path = replace "./" "/" $path -}}
+{{- if not (hasPrefix "/" $path) -}}
+{{- $path = printf "/%s" $path -}}
+{{- end -}}
+{{- $path -}}
 {{- end -}}

EventStore.Replicator.sln.DotSettings

@@ -0,0 +1,15 @@
+{{- if eq (include "replicator.shouldUseCustomPartitioner" .) "true" -}}
+apiVersion: v1
+data:
+{{ include "replicator.sink.partitioner.filename" . | indent 2 }}: |-
+{{ .Values.partitionerJs | indent 4 }}
+
+kind: ConfigMap
+metadata:
+  name: {{ template "replicator.fullname" . }}-partitioner-js
+  labels:
+    app: {{ template "replicator.name" . }}
+    chart: {{ template "replicator.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+{{- end }}

charts/replicator/templates/_helpers.tpl

@@ -1,6 +1,7 @@
+{{- if eq (include "replicator.shouldUseJavascriptTransform" .) "true" -}}
 apiVersion: v1
 data:
-  transform.js: |-
+{{ include "replicator.transform.filename" . | indent 2 }}: |-
 {{ .Values.transformJs | indent 4 }}
 
 kind: ConfigMap
@@ -11,3 +12,4 @@ metadata:
     chart: {{ template "replicator.chart" . }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
+{{- end }}

charts/replicator/templates/configmap-partitioner.yaml

@@ -50,9 +50,16 @@ spec:
           mountPath: /app/js/{{ .fileName }}
           subPath: {{ .fileName }}
 {{- end }}
+{{- if eq (include "replicator.shouldUseJavascriptTransform" .) "true" }}
         - name: transform-js
-          mountPath: /app/transform.js
-          subPath: transform.js
+          mountPath: /app{{ include "replicator.transform.filepath" . }}
+          subPath: {{ include "replicator.transform.filename" . }}
+{{- end }}
+{{- if eq (include "replicator.shouldUseCustomPartitioner" .) "true" }}
+        - name: partitioner-js
+          mountPath: /app{{ include "replicator.sink.partitioner.filepath" . }}
+          subPath: {{ include "replicator.sink.partitioner.filename" . }}
+{{- end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
       imagePullSecrets: []
@@ -66,9 +73,16 @@ spec:
         configMap:
           name: {{ .configMapName }}
 {{- end }}
+{{- if eq (include "replicator.shouldUseJavascriptTransform" .) "true" }}
       - name: transform-js
         configMap:
           name: {{ template "replicator.fullname" . }}-transform-js
+{{- end }}
+{{- if eq (include "replicator.shouldUseCustomPartitioner" .) "true" }}
+      - name: partitioner-js
+        configMap:
+          name: {{ template "replicator.fullname" . }}-partitioner-js
+{{- end }}
       - name: {{ template "replicator.name" . }}
         persistentVolumeClaim:
           claimName: {{ template "replicator.fullname" . }}

charts/replicator/templates/configmap-transform.yaml

@@ -30,7 +30,5 @@ replicator:
   scavenge: true
   filters: []
   transform: null
-transformJs: |-
-
 #  restartOnFailure: false
 #  runContinuously: true

charts/replicator/templates/statefulset.yaml

@@ -1,9 +1,7 @@
-
-
 // ReSharper disable UnusedAutoPropertyAccessor.Global
 
 #nullable disable
-namespace es_replicator.Settings; 
+namespace es_replicator.Settings;
 
 public record EsdbSettings {
     public string ConnectionString { get; init; }
@@ -57,10 +55,10 @@ public static class ConfigExtensions {
     }
 
     public static void AddOptions<T>(
-        this IServiceCollection services, IConfiguration configuration
+        this IServiceCollection services,
+        IConfiguration          configuration
     )
         where T : class {
         services.Configure<T>(configuration.GetSection(typeof(T).Name));
     }
-}
-#nullable enable
+}

charts/replicator/values.yaml

@@ -11,11 +11,17 @@ public static TransformEvent GetTransformer(Replicator settings) {
 
         return type switch {
             "default" => Transforms.DefaultWithExtraMeta,
-            "http"    => new HttpTransform(settings.Transform?.Config).Transform,
+            "http"    => GetHttpTransform().Transform,
             "js"      => GetJsTransform().Transform,
             _         => Transforms.DefaultWithExtraMeta,
         };
 
+        HttpTransform GetHttpTransform() {
+            Ensure.NotEmpty(settings.Transform?.Config, "Transform config");
+
+            return new HttpTransform(settings.Transform!.Config);
+        }
+
         JsTransform GetJsTransform() {
             Ensure.NotEmpty(settings.Transform?.Config, "Transform config");
 

src/es-replicator/Settings/ReplicatorSettings.cs

@@ -1,4 +1,3 @@
-using System.Collections.Immutable;
 using es_replicator.HttpApi;
 using es_replicator.Settings;
 using EventStore.Replicator;