negative structured metadata bytes received

[vnwnv]

Describe the bug
Same issue as #17371

To Reproduce
Steps to reproduce the behavior:

1. install helm chart loki, version 6.30.1

Expected behavior
A clear and concise description of what you expected to happen.

Environment:

• Infrastructure: kubernetes v1.33.1+rke2r1
• Deployment tool: helm by argocd

Screenshots, Promtail config, or terminal output
log by distributor

level=error ts=2025-06-04T10:55:58.830498042Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:55:59.017079169Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:55:59.881793829Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:55:59.893130729Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:00.829058549Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:01.881635199Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:02.017588143Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:02.82911715Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:02.893320948Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:04.830134015Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:04.880711326Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:04.932059906Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:05.017259277Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:05.830399348Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0
level=error ts=2025-06-04T10:56:06.017364923Z caller=push.go:202 org_id=self-monitoring msg="negative structured metadata bytes received" userID=self-monitoring retentionHours=672 isAggregatedMetric=false policyName= size=0

log by gateway

10.42.5.168 - self-monitoring [04/Jun/2025:10:59:38 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.3.124 - self-monitoring [04/Jun/2025:10:59:39 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.0.205 - self-monitoring [04/Jun/2025:10:59:39 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.2.162 - self-monitoring [04/Jun/2025:10:59:39 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.4.176 - self-monitoring [04/Jun/2025:10:59:39 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.1.191 - self-monitoring [04/Jun/2025:10:59:39 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.5.168 - self-monitoring [04/Jun/2025:10:59:39 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.3.124 - self-monitoring [04/Jun/2025:10:59:40 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.0.205 - self-monitoring [04/Jun/2025:10:59:40 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.2.162 - self-monitoring [04/Jun/2025:10:59:40 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.4.176 - self-monitoring [04/Jun/2025:10:59:40 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.1.191 - self-monitoring [04/Jun/2025:10:59:40 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.5.168 - self-monitoring [04/Jun/2025:10:59:40 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.3.124 - self-monitoring [04/Jun/2025:10:59:41 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.0.205 - self-monitoring [04/Jun/2025:10:59:41 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.2.162 - self-monitoring [04/Jun/2025:10:59:41 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.4.176 - self-monitoring [04/Jun/2025:10:59:41 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.1.191 - self-monitoring [04/Jun/2025:10:59:41 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.5.168 - self-monitoring [04/Jun/2025:10:59:41 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.3.124 - self-monitoring [04/Jun/2025:10:59:42 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.0.205 - self-monitoring [04/Jun/2025:10:59:42 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.2.162 - self-monitoring [04/Jun/2025:10:59:42 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.4.176 - self-monitoring [04/Jun/2025:10:59:42 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.1.191 - self-monitoring [04/Jun/2025:10:59:42 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"
10.42.5.168 - self-monitoring [04/Jun/2025:10:59:42 +0000]  204 "POST /loki/api/v1/push HTTP/1.1" 0 "-" "canary-push/3.5.0" "-"

[vnwnv]

After back to 6.29.0 the gateway still post lots of requests, but the distributor not show the error log

[s-luchina]

did you try to override loki tag in helm to 3.5.1? in previous conversation probably i saw that it is fixed, waiting for helm-chart update

[vnwnv]

I tried 3.5.1, pod's behavior looks like same to 3.4.x the gateway pod print lots of log, it received many of POST requests. About 1 request per pod per second. Is that normal?

did you try to override loki tag in helm to 3.5.1? in previous conversation probably i saw that it is fixed, waiting for helm-chart update

[Hitesh-Agrawal]

Getting this same error on upgrading loki with helm-chart version grafana/loki-6.30.0. The tag for distributor is 3.5.0 and the distributor shutdowns after a while failing health checks and is evicted by kubernetes..
level=error ts=2025-06-09T10:35:26.813205661Z caller=push.go:202 org_id=fake traceID=3277b1f6e6c39ec5 msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:27.450238996Z caller=push.go:202 org_id=fake traceID=52a81525cd6f74b2 msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:28.125064457Z caller=push.go:202 org_id=fake traceID=10aa4c6d4f7aab6f msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:29.024121722Z caller=push.go:202 org_id=fake traceID=4d3c41c38e4dc276 msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:29.113213906Z caller=push.go:202 org_id=fake traceID=08864ea0cd259a26 msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:29.693544823Z caller=push.go:202 org_id=fake traceID=5a683e18ba2f025e msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:29.697687295Z caller=push.go:202 org_id=fake traceID=2a8ac79eecd5f26f msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:29.737491917Z caller=push.go:202 org_id=fake traceID=249547515be5af3f msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:30.681258444Z caller=push.go:202 org_id=fake traceID=6e198b42c150eb35 msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:30.797178034Z caller=push.go:202 org_id=fake traceID=5e88f4d93c8d2a0a msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=error ts=2025-06-09T10:35:31.038257634Z caller=push.go:202 org_id=fake traceID=378b5defa552c310 msg="negative structured metadata bytes received" userID=fake retentionHours=43800 isAggregatedMetric=false policyName= size=0 level=info ts=2025-06-09T10:35:31.343276918Z caller=signals.go:62 msg="=== received SIGINT/SIGTERM ===\n*** exiting" level=info ts=2025-06-09T10:35:31.34357545Z caller=basic_lifecycler.go:242 component=distributor msg="ring lifecycler is shutting down" ring=distributor level=info ts=2025-06-09T10:35:31.343718194Z caller=basic_lifecycler.go:407 component=distributor msg="unregistering instance from ring" ring=distributor level=info ts=2025-06-09T10:35:31.343760883Z caller=basic_lifecycler.go:282 component=distributor msg="instance removed from the ring" ring=distributor level=info ts=2025-06-09T10:35:31.343863488Z caller=module_service.go:120 msg="module stopped" module=distributor level=info ts=2025-06-09T10:35:31.343913686Z caller=module_service.go:120 msg="module stopped" module=ring level=info ts=2025-06-09T10:35:31.34393172Z caller=module_service.go:120 msg="module stopped" module=analytics level=info ts=2025-06-09T10:35:31.343949513Z caller=module_service.go:120 msg="module stopped" module=runtime-config level=info ts=2025-06-09T10:35:31.343955477Z caller=memberlist_client.go:773 msg="leaving memberlist cluster"