This repository was archived by the owner on Jul 19, 2023. It is now read-only.
Add support for global size-based retention policy #628
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,225 @@ | ||
| package ingester | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "io/fs" | ||
| "os" | ||
| "path/filepath" | ||
| "sort" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/go-kit/log" | ||
| "github.com/go-kit/log/level" | ||
| "github.com/grafana/dskit/services" | ||
| "github.com/oklog/ulid" | ||
|
|
||
| "github.com/grafana/phlare/pkg/phlaredb" | ||
| "github.com/grafana/phlare/pkg/phlaredb/block" | ||
| diskutil "github.com/grafana/phlare/pkg/util/disk" | ||
| ) | ||
|
|
||
| const ( | ||
| defaultMinFreeDisk = 10 * 1024 * 1024 * 1024 // 10Gi | ||
| defaultMinDiskAvailablePercentage = 0.05 | ||
| defaultRetentionPolicyEnforcementInterval = 5 * time.Minute | ||
|
Comment on lines
+24
to
+26
There was a problem hiding this comment. I think it's worth making these parameters configurable There was a problem hiding this comment. I tend to agree, it will always be a balance between flexibility and making Pyroscope complex to operate, but being able to change that or disable deletion all together is more than a nice to have. That said I wouldn't mind if you do this in here or in a follow up PR. |
||
|
|
||
| // TODO(kolesnikovae): Unify with pkg/phlaredb. | ||
| phlareDBLocalPath = "local" | ||
| ) | ||
|
|
||
| type retentionPolicy struct { | ||
| MinFreeDisk uint64 | ||
| MinDiskAvailablePercentage float64 | ||
| EnforcementInterval time.Duration | ||
| } | ||
|
|
||
| func defaultRetentionPolicy() retentionPolicy { | ||
| return retentionPolicy{ | ||
| MinFreeDisk: defaultMinFreeDisk, | ||
| MinDiskAvailablePercentage: defaultMinDiskAvailablePercentage, | ||
| EnforcementInterval: defaultRetentionPolicyEnforcementInterval, | ||
| } | ||
| } | ||
|
|
||
| type retentionPolicyEnforcer struct { | ||
| services.Service | ||
|
|
||
| logger log.Logger | ||
| retentionPolicy retentionPolicy | ||
| blockEvicter blockEvicter | ||
| dbConfig phlaredb.Config | ||
| fileSystem fileSystem | ||
| volumeChecker diskutil.VolumeChecker | ||
|
|
||
| stopCh chan struct{} | ||
| wg sync.WaitGroup | ||
| } | ||
|
|
||
| type tenantBlock struct { | ||
| ulid ulid.ULID | ||
| tenantID string | ||
| path string | ||
| } | ||
|
|
||
| type fileSystem interface { | ||
| fs.ReadDirFS | ||
| RemoveAll(string) error | ||
| } | ||
|
|
||
| type realFileSystem struct{} | ||
|
|
||
| func (*realFileSystem) Open(name string) (fs.File, error) { return os.Open(name) } | ||
| func (*realFileSystem) ReadDir(name string) ([]fs.DirEntry, error) { return os.ReadDir(name) } | ||
| func (*realFileSystem) RemoveAll(path string) error { return os.RemoveAll(path) } | ||
|
|
||
| // blockEvicter unloads blocks from tenant instance. | ||
| type blockEvicter interface { | ||
| // evictBlock evicts the block by its ID for the given tenant and invokes | ||
| // fn callback, if the tenant is found. The call is thread-safe: tenant | ||
| // can't be added or removed during the execution. | ||
| evictBlock(tenant string, b ulid.ULID, fn func() error) error | ||
| } | ||
|
|
||
| func newRetentionPolicyEnforcer(logger log.Logger, blockEvicter blockEvicter, retentionPolicy retentionPolicy, dbConfig phlaredb.Config) *retentionPolicyEnforcer { | ||
| e := retentionPolicyEnforcer{ | ||
| logger: logger, | ||
| blockEvicter: blockEvicter, | ||
| retentionPolicy: retentionPolicy, | ||
| dbConfig: dbConfig, | ||
| stopCh: make(chan struct{}), | ||
| fileSystem: new(realFileSystem), | ||
| volumeChecker: diskutil.NewVolumeChecker(retentionPolicy.MinFreeDisk, retentionPolicy.MinDiskAvailablePercentage), | ||
| } | ||
| e.Service = services.NewBasicService(nil, e.running, e.stopping) | ||
| return &e | ||
| } | ||
|
|
||
| func (e *retentionPolicyEnforcer) running(ctx context.Context) error { | ||
| e.wg.Add(1) | ||
| retentionPolicyEnforcerTicker := time.NewTicker(e.retentionPolicy.EnforcementInterval) | ||
| defer func() { | ||
| retentionPolicyEnforcerTicker.Stop() | ||
| e.wg.Done() | ||
| }() | ||
| for { | ||
| // Enforce retention policy immediately at start. | ||
| level.Debug(e.logger).Log("msg", "enforcing retention policy") | ||
| if err := e.cleanupBlocksWhenHighDiskUtilization(ctx); err != nil { | ||
| level.Error(e.logger).Log("msg", "failed to enforce retention policy", "err", err) | ||
| } | ||
| select { | ||
| case <-retentionPolicyEnforcerTicker.C: | ||
| case <-ctx.Done(): | ||
| return nil | ||
| case <-e.stopCh: | ||
| return nil | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func (e *retentionPolicyEnforcer) stopping(_ error) error { | ||
| close(e.stopCh) | ||
| e.wg.Wait() | ||
| return nil | ||
| } | ||
|
|
||
| func (e *retentionPolicyEnforcer) localBlocks(dir string) ([]*tenantBlock, error) { | ||
| blocks := make([]*tenantBlock, 0, 32) | ||
| tenants, err := fs.ReadDir(e.fileSystem, dir) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| var blockDirs []fs.DirEntry | ||
| for _, tenantDir := range tenants { | ||
| if !tenantDir.IsDir() { | ||
| continue | ||
| } | ||
| tenantID := tenantDir.Name() | ||
| tenantDirPath := filepath.Join(dir, tenantID, phlareDBLocalPath) | ||
| if blockDirs, err = fs.ReadDir(e.fileSystem, tenantDirPath); err != nil { | ||
| if os.IsNotExist(err) { | ||
| // Must be created by external means, skipping. | ||
| continue | ||
| } | ||
| return nil, err | ||
| } | ||
| for _, blockDir := range blockDirs { | ||
| if !blockDir.IsDir() { | ||
| continue | ||
| } | ||
| blockPath := filepath.Join(tenantDirPath, blockDir.Name()) | ||
| if blockID, ok := block.IsBlockDir(blockPath); ok { | ||
| blocks = append(blocks, &tenantBlock{ | ||
| ulid: blockID, | ||
| path: blockPath, | ||
| tenantID: tenantID, | ||
| }) | ||
| } | ||
| // A malformed/invalid ULID likely means that the | ||
| // directory is not a valid block, ignoring. | ||
| } | ||
| } | ||
|
|
||
| // Sort the blocks by their id, which will be the time they've been created. | ||
| sort.Slice(blocks, func(i, j int) bool { | ||
| return blocks[i].ulid.Compare(blocks[j].ulid) < 0 | ||
| }) | ||
|
|
||
| return blocks, nil | ||
| } | ||
|
|
||
| func (e *retentionPolicyEnforcer) cleanupBlocksWhenHighDiskUtilization(ctx context.Context) error { | ||
| var volumeStatsPrev *diskutil.VolumeStats | ||
| volumeStatsCurrent, err := e.volumeChecker.HasHighDiskUtilization(e.dbConfig.DataPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| // Not in high disk utilization, nothing to do. | ||
| if !volumeStatsCurrent.HighDiskUtilization { | ||
| return nil | ||
| } | ||
| // Get all block across all the tenants. Any block | ||
| // produced or imported during the procedure is ignored. | ||
| blocks, err := e.localBlocks(e.dbConfig.DataPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for volumeStatsCurrent.HighDiskUtilization && len(blocks) > 0 && ctx.Err() == nil { | ||
| // When disk utilization is not lower since the last loop, we end the | ||
| // cleanup there to avoid deleting all blocks when disk usage reporting | ||
| // is delayed. | ||
| if volumeStatsPrev != nil && volumeStatsPrev.BytesAvailable >= volumeStatsCurrent.BytesAvailable { | ||
| level.Warn(e.logger).Log("msg", "disk utilization is not lowered by deletion of a block, pausing until next cycle") | ||
|
There was a problem hiding this comment. Just a note here, for my own Laptop, the filesystem There was a problem hiding this comment. Oh, that's a great point, which worth mentioning in docs. As a small optimization, I'll implement batch removal that relies on the actual block size on disk. |
||
| break | ||
| } | ||
| // Delete the oldest block. | ||
| var b *tenantBlock | ||
| b, blocks = blocks[0], blocks[1:] | ||
| level.Warn(e.logger).Log("msg", "disk utilization is high, deleting the oldest block", "path", b.path) | ||
| if err = e.deleteBlock(b); err != nil { | ||
| return err | ||
| } | ||
| volumeStatsPrev = volumeStatsCurrent | ||
| if volumeStatsCurrent, err = e.volumeChecker.HasHighDiskUtilization(e.dbConfig.DataPath); err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| return ctx.Err() | ||
| } | ||
|
|
||
| func (e *retentionPolicyEnforcer) deleteBlock(b *tenantBlock) error { | ||
| return e.blockEvicter.evictBlock(b.tenantID, b.ulid, func() error { | ||
| switch err := e.fileSystem.RemoveAll(b.path); { | ||
| case err == nil: | ||
| case os.IsNotExist(err): | ||
| level.Warn(e.logger).Log("msg", "block not found on disk", "path", b.path) | ||
| default: | ||
| return fmt.Errorf("failed to delete block %q: %w", b.path, err) | ||
| } | ||
| return nil | ||
| }) | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.