loki.process: enable structured_metadata labels conversion (loki/#10752) (grafana#5481)

Author: salvacorts

CHANGELOG.md

@@ -42,6 +42,9 @@ Main (unreleased)
 
 - Update Prometheus dependency to v2.47.2. (@tpaschalis)
 
+- Allow converting labels to structured metadata with Loki's structured_metadata stage. (@gonzalesraul)
+
+
 v0.37.1 (2023-10-10)
 -----------------
 

component/loki/process/stages/structured_metadata.go

@@ -32,6 +32,26 @@ func (s *structuredMetadataStage) Run(in chan Entry) chan Entry {
 		processLabelsConfigs(s.logger, e.Extracted, s.cfgs, func(labelName model.LabelName, labelValue model.LabelValue) {
 			e.StructuredMetadata = append(e.StructuredMetadata, logproto.LabelAdapter{Name: string(labelName), Value: string(labelValue)})
 		})
-		return e
+		return s.extractFromLabels(e)
 	})
 }
+
+func (s *structuredMetadataStage) extractFromLabels(e Entry) Entry {
+	labels := e.Labels
+	foundLabels := []model.LabelName{}
+
+	for lName, lSrc := range s.cfgs.Values {
+		labelKey := model.LabelName(*lSrc)
+		if lValue, ok := labels[labelKey]; ok {
+			e.StructuredMetadata = append(e.StructuredMetadata, logproto.LabelAdapter{Name: lName, Value: string(lValue)})
+			foundLabels = append(foundLabels, labelKey)
+		}
+	}
+
+	// Remove found labels, do this after append to structure metadata
+	for _, fl := range foundLabels {
+		delete(labels, fl)
+	}
+	e.Labels = labels
+	return e
+}

component/loki/process/stages/structured_metadata_test.go

@@ -71,6 +71,26 @@ stage.labels {
 }
 `
 
+var pipelineStagesStructuredMetadataFromStaticLabels = `
+stage.static_labels {
+	values = {"component" = "querier", "pod" = "loki-querier-664f97db8d-qhnwg"}
+}
+
+stage.structured_metadata {
+	values = {"pod" = ""}
+}
+`
+
+var pipelineStagesStructuredMetadataFromStaticLabelsDifferentKey = `
+stage.static_labels {
+	values = {"component" = "querier", "pod" = "loki-querier-664f97db8d-qhnwg"}
+}
+
+stage.structured_metadata {
+	values = {"pod_name" = "pod"}
+}
+`
+
 func Test_StructuredMetadataStage(t *testing.T) {
 	tests := map[string]struct {
 		pipelineStagesYaml         string
@@ -104,6 +124,18 @@ func Test_StructuredMetadataStage(t *testing.T) {
 			expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}},
 			expectedLabels:             model.LabelSet{model.LabelName("component"): model.LabelValue("ingester")},
 		},
+		"expected structured metadata and regular labels to be extracted with static labels stage and to be added to entry": {
+			pipelineStagesYaml:         pipelineStagesStructuredMetadataFromStaticLabels,
+			logLine:                    `sample log line`,
+			expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "pod", Value: "loki-querier-664f97db8d-qhnwg"}},
+			expectedLabels:             model.LabelSet{model.LabelName("component"): model.LabelValue("querier")},
+		},
+		"expected structured metadata and regular labels to be extracted with static labels stage using different structured key": {
+			pipelineStagesYaml:         pipelineStagesStructuredMetadataFromStaticLabelsDifferentKey,
+			logLine:                    `sample log line`,
+			expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "pod_name", Value: "loki-querier-664f97db8d-qhnwg"}},
+			expectedLabels:             model.LabelSet{model.LabelName("component"): model.LabelValue("querier")},
+		},
 	}
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {