diff --git a/pkg/phlaredb/block_querier.go b/pkg/phlaredb/block_querier.go index 283f04af9..3aefefa71 100644 --- a/pkg/phlaredb/block_querier.go +++ b/pkg/phlaredb/block_querier.go @@ -373,7 +373,10 @@ func NewSingleBlockQuerierFromMeta(phlarectx context.Context, bucketReader phlar func (b *singleBlockQuerier) Close() error { b.openLock.Lock() - defer b.openLock.Unlock() + defer func() { + b.openLock.Unlock() + b.metrics.blockOpened.Dec() + }() errs := multierror.New() if b.index != nil { err := b.index.Close() @@ -1076,6 +1079,7 @@ func (q *singleBlockQuerier) Open(ctx context.Context) error { if err := q.openFiles(ctx); err != nil { return err } + q.metrics.blockOpened.Inc() q.opened = true return nil } diff --git a/pkg/phlaredb/block_querier_test.go b/pkg/phlaredb/block_querier_test.go index ac90bced0..0d76f2087 100644 --- a/pkg/phlaredb/block_querier_test.go +++ b/pkg/phlaredb/block_querier_test.go @@ -101,7 +101,10 @@ func TestQuerierBlockEviction(t *testing.T) { for _, tc := range testCases { q := BlockQuerier{queriers: make([]*singleBlockQuerier, len(tc.blocks))} for i, b := range tc.blocks { - q.queriers[i] = &singleBlockQuerier{meta: &block.Meta{ULID: ulid.MustParse(b)}} + q.queriers[i] = &singleBlockQuerier{ + meta: &block.Meta{ULID: ulid.MustParse(b)}, + metrics: newBlocksMetrics(nil), + } } evicted, err := q.evict(ulid.MustParse(blockToEvict)) diff --git a/pkg/phlaredb/metrics.go b/pkg/phlaredb/metrics.go index 01b210813..142eff55e 100644 --- a/pkg/phlaredb/metrics.go +++ b/pkg/phlaredb/metrics.go @@ -29,6 +29,7 @@ type headMetrics struct { sampleValuesIngested *prometheus.CounterVec sampleValuesReceived *prometheus.CounterVec + samples prometheus.Gauge flushedFileSizeBytes *prometheus.HistogramVec flushedBlockSizeBytes prometheus.Histogram @@ -145,6 +146,10 @@ func newHeadMetrics(reg prometheus.Registerer) *headMetrics { // [512KB, 1MB, 2MB, 4MB, 8MB, 16MB, 32MB, 64MB, 128MB, 256MB, 512MB] Buckets: prometheus.ExponentialBuckets(512*1024, 2, 11), }), + samples: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "pyroscope_head_samples", + Help: "Number of samples in the head.", + }), } m.register(reg) @@ -192,6 +197,7 @@ type blocksMetrics struct { query *query.Metrics blockOpeningLatency prometheus.Histogram + blockOpened prometheus.Gauge } func newBlocksMetrics(reg prometheus.Registerer) *blocksMetrics { @@ -201,8 +207,13 @@ func newBlocksMetrics(reg prometheus.Registerer) *blocksMetrics { Name: "pyroscopedb_block_opening_duration", Help: "Latency of opening a block in seconds", }), + blockOpened: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "pyroscopedb_blocks_currently_open", + Help: "Number of blocks opened", + }), } m.blockOpeningLatency = util.RegisterOrGet(reg, m.blockOpeningLatency) + m.blockOpened = util.RegisterOrGet(reg, m.blockOpened) return m } diff --git a/pkg/phlaredb/profile_store.go b/pkg/phlaredb/profile_store.go index 74bb57733..0ebfd237c 100644 --- a/pkg/phlaredb/profile_store.go +++ b/pkg/phlaredb/profile_store.go @@ -279,6 +279,7 @@ func (s *profileStore) cutRowGroup(count int) (err error) { defer s.profilesLock.Unlock() for i := range s.slice[:count] { // don't retain profiles and samples in memory as re-slice. + s.metrics.samples.Sub(float64(len(s.slice[i].Samples))) s.slice[i] = nil } // reset slice and metrics @@ -420,6 +421,7 @@ func (s *profileStore) ingest(_ context.Context, profiles []*schemav1.Profile, l // add to slice s.slice = append(s.slice, p) + s.metrics.samples.Add(float64(len(p.Samples))) }