reader/{tailx,dirx}: clean expired submeta after sync #688
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
lgtm |
wonderflow
reviewed
Aug 3, 2018
reader/dirx/dirx.go
Outdated
| @@ -341,6 +343,10 @@ func (r *Reader) SyncMeta() { | |||
| log.Errorf("Runner[%v] write meta[%v] failed: %v", r.meta.RunnerName, string(data), err) | |||
| return | |||
| } | |||
|
|
|||
| if r.expire.Nanoseconds() > 0 { | |||
| r.meta.CleanExpiredSubMetas(r.expire * 3) | |||
There was a problem hiding this comment.
这里不要用expire的3倍时间了，单独做个参数吧，默认30天
wonderflow
reviewed
Aug 3, 2018
reader/meta.go
Outdated
|
|
||
| // CleanExpiredSubMetas 清除超过指定过期时长的 submetas 子目录， | ||
| // 如果 submetas 没有存放在该 meta 的子目录则调用此方法无效 | ||
| func (m *Meta) CleanExpiredSubMetas(expire time.Duration) { |
There was a problem hiding this comment.
这个函数是跟在sync后面调用的，执行时间长会影响读取性能，可以设置个上次调用的时间，一小时调用一次就好了。
wonderflow
suggested changes
Aug 3, 2018
reader/rest_reader_models.go
Outdated
| @@ -473,6 +473,16 @@ var ModeKeyOptions = map[string][]Option{ | |||
| CheckRegex: "\\d+[hms]", | |||
| ToolTip: `当文件夹内所有的日志达到expire时间，则放弃追踪。写法为：数字加单位符号，组成字符串duration写法，支持时h、分m、秒s为单位，类似3h(3小时)，10m(10分钟)，5s(5秒)，默认的expire时间是0s，即永不过期`, | |||
| }, | |||
| { | |||
| KeyName: KeySubmetaExpire, | |||
reader/meta.go
Outdated
| func (m *Meta) CleanExpiredSubMetas(expire time.Duration) { | ||
| // 为避免频繁扫描目录影响性能，多次调用间隔必须大于最小阈值 | ||
| if m.subMetaLastCleaned.Add(defaultSubMetaCleanInterval).After(time.Now()) { | ||
| log.Debugf("Meta %q has cleaned submetas at %s, ignored this time", m.Dir, m.subMetaLastCleaned) |
There was a problem hiding this comment.
这里不要日志感觉比较好，倒是下面开始clean的时候可以加个Debug日志
reader/tailx/tailx.go
Outdated
| @@ -581,6 +581,8 @@ func (r *Reader) SyncMeta() { | |||
| log.Errorf("%v sync meta WriteBuf error %v, buf %v", r.Name(), err, string(buf)) | |||
| return | |||
| } | |||
|
|
|||
| r.meta.CleanExpiredSubMetas(r.expire * 3) | |||
| @@ -128,6 +136,8 @@ func NewReader(meta *reader.Meta, conf conf.MapConf) (reader.Reader, error) { | |||
| dirReaders: newDirReaders(meta, expire, cachedLines), | |||
| logPathPattern: strings.TrimSuffix(logPathPattern, "/"), | |||
| statInterval: statInterval, | |||
| expire: expire, | |||
| submetaExpire: submetaExpire, | |||
There was a problem hiding this comment.
这个值还要跟expire比较一下，如果小于expire，要报错
reader/meta.go
Outdated
| } | ||
|
|
||
| if fi.ModTime().Add(expire).Before(time.Now()) { | ||
| log.Debugf("Expired submeta %q has been removed with error %v", path, os.RemoveAll(path)) |
wonderflow
reviewed
Aug 6, 2018
reader/meta.go
Outdated
| numCleaned := 0 | ||
| for path := range m.subMetaExpired { | ||
| if numCleaned >= maximumSubMetaCleanNumOneTime { | ||
| log.Infof("Cleaned %d expired submeta of %q and there are %d left, but break at this time", maximumSubMetaCleanNumOneTime, path, len(m.subMetaExpired)) |
There was a problem hiding this comment.
加一句: left expired submeta will be cleaned next time
wonderflow
reviewed
Aug 6, 2018
reader/meta.go
Outdated
| // 二次确认 submeta 目录在删除前的一刻仍旧是过期状态才执行删除操作 | ||
| if hasSubMetaExpired(path, expire) { | ||
| err := os.RemoveAll(path) | ||
| log.Infof("Expired submeta %q has been removed with error %v", path, err) |
wonderflow
reviewed
Aug 6, 2018
reader/rest_reader_models.go
Outdated
| { | ||
| KeyName: KeySubmetaExpire, | ||
| ChooseOnly: false, | ||
| Default: "72h", |
wonderflow
approved these changes
Aug 6, 2018
|
文档对应修改 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
JIRA: https://jira.qiniu.io/browse/PDR-7006