add support for recovery section in event streams (zalando#2421)

Author: FxKu

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -247,16 +247,18 @@ type ConnectionPooler struct {
 
 // Stream defines properties for creating FabricEventStream resources
 type Stream struct {
-	ApplicationId string                 `json:"applicationId"`
-	Database      string                 `json:"database"`
-	Tables        map[string]StreamTable `json:"tables"`
-	Filter        map[string]*string     `json:"filter,omitempty"`
-	BatchSize     *uint32                `json:"batchSize,omitempty"`
+	ApplicationId  string                 `json:"applicationId"`
+	Database       string                 `json:"database"`
+	Tables         map[string]StreamTable `json:"tables"`
+	Filter         map[string]*string     `json:"filter,omitempty"`
+	BatchSize      *uint32                `json:"batchSize,omitempty"`
+	EnableRecovery *bool                  `json:"enableRecovery,omitempty"`
 }
 
 // StreamTable defines properties of outbox tables for FabricEventStreams
 type StreamTable struct {
-	EventType     string  `json:"eventType"`
-	IdColumn      *string `json:"idColumn,omitempty"`
-	PayloadColumn *string `json:"payloadColumn,omitempty"`
+	EventType         string  `json:"eventType"`
+	RecoveryEventType string  `json:"recoveryEventType"`
+	IdColumn          *string `json:"idColumn,omitempty"`
+	PayloadColumn     *string `json:"payloadColumn,omitempty"`
 }

pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go

@@ -33,9 +33,10 @@ type FabricEventStreamList struct {
 
 // EventStream defines the source, flow and sink of the event stream
 type EventStream struct {
-	EventStreamFlow   EventStreamFlow   `json:"flow"`
-	EventStreamSink   EventStreamSink   `json:"sink"`
-	EventStreamSource EventStreamSource `json:"source"`
+	EventStreamFlow     EventStreamFlow     `json:"flow"`
+	EventStreamSink     EventStreamSink     `json:"sink"`
+	EventStreamSource   EventStreamSource   `json:"source"`
+	EventStreamRecovery EventStreamRecovery `json:"recovery"`
 }
 
 // EventStreamFlow defines the flow characteristics of the event stream
@@ -51,6 +52,12 @@ type EventStreamSink struct {
 	MaxBatchSize *uint32 `json:"maxBatchSize,omitempty"`
 }
 
+// EventStreamRecovery defines the target of dead letter queue
+type EventStreamRecovery struct {
+	Type string           `json:"type"`
+	Sink *EventStreamSink `json:"sink"`
+}
+
 // EventStreamSource defines the source of the event stream and connection for FES operator
 type EventStreamSource struct {
 	Type             string           `json:"type"`

pkg/apis/zalando.org/v1/fabriceventstream.go

@@ -145,11 +145,13 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEvent
 			streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn)
 			streamFlow := getEventStreamFlow(stream, table.PayloadColumn)
 			streamSink := getEventStreamSink(stream, table.EventType)
+			streamRecovery := getEventStreamRecovery(stream, table.RecoveryEventType, table.EventType)
 
 			eventStreams = append(eventStreams, zalandov1.EventStream{
-				EventStreamFlow:   streamFlow,
-				EventStreamSink:   streamSink,
-				EventStreamSource: streamSource})
+				EventStreamFlow:     streamFlow,
+				EventStreamRecovery: streamRecovery,
+				EventStreamSink:     streamSink,
+				EventStreamSource:   streamSource})
 		}
 	}
 
@@ -204,6 +206,28 @@ func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1.EventS
 	}
 }
 
+func getEventStreamRecovery(stream acidv1.Stream, recoveryEventType, eventType string) zalandov1.EventStreamRecovery {
+	if (stream.EnableRecovery != nil && !*stream.EnableRecovery) ||
+		(stream.EnableRecovery == nil && recoveryEventType == "") {
+		return zalandov1.EventStreamRecovery{
+			Type: constants.EventStreamRecoveryNoneType,
+		}
+	}
+
+	if stream.EnableRecovery != nil && *stream.EnableRecovery && recoveryEventType == "" {
+		recoveryEventType = fmt.Sprintf("%s-%s", eventType, constants.EventStreamRecoverySuffix)
+	}
+
+	return zalandov1.EventStreamRecovery{
+		Type: constants.EventStreamRecoveryDLQType,
+		Sink: &zalandov1.EventStreamSink{
+			Type:         constants.EventStreamSinkNakadiType,
+			EventType:    recoveryEventType,
+			MaxBatchSize: stream.BatchSize,
+		},
+	}
+}
+
 func getTableSchema(fullTableName string) (tableName, schemaName string) {
 	schemaName = "public"
 	tableName = fullTableName
@@ -381,7 +405,8 @@ func sameStreams(curEventStreams, newEventStreams []zalandov1.EventStream) (matc
 		for _, curStream := range curEventStreams {
 			if reflect.DeepEqual(newStream.EventStreamSource, curStream.EventStreamSource) &&
 				reflect.DeepEqual(newStream.EventStreamFlow, curStream.EventStreamFlow) &&
-				reflect.DeepEqual(newStream.EventStreamSink, curStream.EventStreamSink) {
+				reflect.DeepEqual(newStream.EventStreamSink, curStream.EventStreamSink) &&
+				reflect.DeepEqual(newStream.EventStreamRecovery, curStream.EventStreamRecovery) {
 				match = true
 				break
 			}

pkg/apis/zalando.org/v1/zz_generated.deepcopy.go

@@ -65,9 +65,11 @@ var (
 							PayloadColumn: k8sutil.StringToPointer("b_payload"),
 						},
 						"data.foobar": acidv1.StreamTable{
-							EventType: "stream-type-b",
+							EventType:         "stream-type-b",
+							RecoveryEventType: "stream-type-b-dlq",
 						},
 					},
+					EnableRecovery: util.True(),
 					Filter: map[string]*string{
 						"data.bar": k8sutil.StringToPointer("[?(@.source.txId > 500 && @.source.lsn > 123456)]"),
 					},
@@ -106,6 +108,14 @@ var (
 						PayloadColumn: k8sutil.StringToPointer("b_payload"),
 						Type:          constants.EventStreamFlowPgGenericType,
 					},
+					EventStreamRecovery: zalandov1.EventStreamRecovery{
+						Type: constants.EventStreamRecoveryDLQType,
+						Sink: &zalandov1.EventStreamSink{
+							EventType:    fmt.Sprintf("%s-%s", "stream-type-a", constants.EventStreamRecoverySuffix),
+							MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
+							Type:         constants.EventStreamSinkNakadiType,
+						},
+					},
 					EventStreamSink: zalandov1.EventStreamSink{
 						EventType:    "stream-type-a",
 						MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
@@ -136,6 +146,14 @@ var (
 					EventStreamFlow: zalandov1.EventStreamFlow{
 						Type: constants.EventStreamFlowPgGenericType,
 					},
+					EventStreamRecovery: zalandov1.EventStreamRecovery{
+						Type: constants.EventStreamRecoveryDLQType,
+						Sink: &zalandov1.EventStreamSink{
+							EventType:    "stream-type-b-dlq",
+							MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
+							Type:         constants.EventStreamSinkNakadiType,
+						},
+					},
 					EventStreamSink: zalandov1.EventStreamSink{
 						EventType:    "stream-type-b",
 						MaxBatchSize: k8sutil.UInt32ToPointer(uint32(100)),
@@ -251,7 +269,8 @@ func TestSameStreams(t *testing.T) {
 	testName := "TestSameStreams"
 
 	stream1 := zalandov1.EventStream{
-		EventStreamFlow: zalandov1.EventStreamFlow{},
+		EventStreamFlow:     zalandov1.EventStreamFlow{},
+		EventStreamRecovery: zalandov1.EventStreamRecovery{},
 		EventStreamSink: zalandov1.EventStreamSink{
 			EventType: "stream-type-a",
 		},
@@ -263,7 +282,8 @@ func TestSameStreams(t *testing.T) {
 	}
 
 	stream2 := zalandov1.EventStream{
-		EventStreamFlow: zalandov1.EventStreamFlow{},
+		EventStreamFlow:     zalandov1.EventStreamFlow{},
+		EventStreamRecovery: zalandov1.EventStreamRecovery{},
 		EventStreamSink: zalandov1.EventStreamSink{
 			EventType: "stream-type-b",
 		},

pkg/cluster/streams.go

@@ -11,4 +11,7 @@ const (
 	EventStreamSourceAuthType    = "DatabaseAuthenticationSecret"
 	EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent"
 	EventStreamSinkNakadiType    = "Nakadi"
+	EventStreamRecoveryNoneType  = "None"
+	EventStreamRecoveryDLQType   = "DeadLetter"
+	EventStreamRecoverySuffix    = "dead-letter-queue"
 )