Skip to content

Commit ba52d50

Browse files
RockeyCossMeowZhengJunjun2016
authored
[Feature] add log collector (open-mmlab#1175)
* [Feature] add log collector * Update .dev/log_collector/readme.md Co-authored-by: Miao Zheng <[email protected]> * Update .dev/log_collector/example_config.py Co-authored-by: Miao Zheng <[email protected]> * fix typo and so on * modify readme * fix some bugs and revise the readme.md * more elegant * Update .dev/log_collector/readme.md Co-authored-by: Junjun2016 <[email protected]> Co-authored-by: Miao Zheng <[email protected]> Co-authored-by: Junjun2016 <[email protected]>
1 parent 81b51ad commit ba52d50

File tree

4 files changed

+324
-0
lines changed

4 files changed

+324
-0
lines changed

‎.dev/log_collector/example_config.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
work_dir = '../../work_dirs'
2+
metric = 'mIoU'
3+
4+
# specify the log files we would like to collect in `log_items`
5+
log_items = [
6+
'segformer_mit-b5_512x512_160k_ade20k_cnn_lr_with_warmup',
7+
'segformer_mit-b5_512x512_160k_ade20k_cnn_no_warmup_lr',
8+
'segformer_mit-b5_512x512_160k_ade20k_mit_trans_lr',
9+
'segformer_mit-b5_512x512_160k_ade20k_swin_trans_lr'
10+
]
11+
# or specify ignore_keywords, then the folders whose name contain
12+
# `'segformer'` won't be collected
13+
# ignore_keywords = ['segformer']
14+
15+
# should not include metric
16+
other_info_keys = ['mAcc']
17+
markdown_file = 'markdowns/lr_in_trans.json.md'
18+
json_file = 'jsons/trans_in_cnn.json'

‎.dev/log_collector/log_collector.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# Copyright (c) OpenMMLab. All rights reserved.
2+
import argparse
3+
import datetime
4+
import json
5+
import os
6+
import os.path as osp
7+
from collections import OrderedDict
8+
9+
from utils import load_config
10+
11+
# automatically collect all the results
12+
13+
# The structure of the directory:
14+
# ├── work-dir
15+
# │ ├── config_1
16+
# │ │ ├── time1.log.json
17+
# │ │ ├── time2.log.json
18+
# │ │ ├── time3.log.json
19+
# │ │ ├── time4.log.json
20+
# │ ├── config_2
21+
# │ │ ├── time5.log.json
22+
# │ │ ├── time6.log.json
23+
# │ │ ├── time7.log.json
24+
# │ │ ├── time8.log.json
25+
26+
27+
def parse_args():
28+
parser = argparse.ArgumentParser(description='extract info from log.json')
29+
parser.add_argument('config_dir')
30+
args = parser.parse_args()
31+
return args
32+
33+
34+
def has_keyword(name: str, keywords: list):
35+
for a_keyword in keywords:
36+
if a_keyword in name:
37+
return True
38+
return False
39+
40+
41+
def main():
42+
args = parse_args()
43+
cfg = load_config(args.config_dir)
44+
work_dir = cfg['work_dir']
45+
metric = cfg['metric']
46+
log_items = cfg.get('log_items', [])
47+
ignore_keywords = cfg.get('ignore_keywords', [])
48+
other_info_keys = cfg.get('other_info_keys', [])
49+
markdown_file = cfg.get('markdown_file', None)
50+
json_file = cfg.get('json_file', None)
51+
52+
if json_file and osp.split(json_file)[0] != '':
53+
os.makedirs(osp.split(json_file)[0], exist_ok=True)
54+
if markdown_file and osp.split(markdown_file)[0] != '':
55+
os.makedirs(osp.split(markdown_file)[0], exist_ok=True)
56+
57+
assert not (log_items and ignore_keywords), \
58+
'log_items and ignore_keywords cannot be specified at the same time'
59+
assert metric not in other_info_keys, \
60+
'other_info_keys should not contain metric'
61+
62+
if ignore_keywords and isinstance(ignore_keywords, str):
63+
ignore_keywords = [ignore_keywords]
64+
if other_info_keys and isinstance(other_info_keys, str):
65+
other_info_keys = [other_info_keys]
66+
if log_items and isinstance(log_items, str):
67+
log_items = [log_items]
68+
69+
if not log_items:
70+
log_items = [
71+
item for item in sorted(os.listdir(work_dir))
72+
if not has_keyword(item, ignore_keywords)
73+
]
74+
75+
experiment_info_list = []
76+
for config_dir in log_items:
77+
preceding_path = os.path.join(work_dir, config_dir)
78+
log_list = [
79+
item for item in os.listdir(preceding_path)
80+
if item.endswith('.log.json')
81+
]
82+
log_list = sorted(
83+
log_list,
84+
key=lambda time_str: datetime.datetime.strptime(
85+
time_str, '%Y%m%d_%H%M%S.log.json'))
86+
val_list = []
87+
last_iter = 0
88+
for log_name in log_list:
89+
with open(os.path.join(preceding_path, log_name), 'r') as f:
90+
# ignore the info line
91+
f.readline()
92+
all_lines = f.readlines()
93+
val_list.extend([
94+
json.loads(line) for line in all_lines
95+
if json.loads(line)['mode'] == 'val'
96+
])
97+
for index in range(len(all_lines) - 1, -1, -1):
98+
line_dict = json.loads(all_lines[index])
99+
if line_dict['mode'] == 'train':
100+
last_iter = max(last_iter, line_dict['iter'])
101+
break
102+
103+
new_log_dict = dict(
104+
method=config_dir, metric_used=metric, last_iter=last_iter)
105+
for index, log in enumerate(val_list, 1):
106+
new_ordered_dict = OrderedDict()
107+
new_ordered_dict['eval_index'] = index
108+
new_ordered_dict[metric] = log[metric]
109+
for key in other_info_keys:
110+
if key in log:
111+
new_ordered_dict[key] = log[key]
112+
val_list[index - 1] = new_ordered_dict
113+
114+
assert len(val_list) >= 1, \
115+
f"work dir {config_dir} doesn't contain any evaluation."
116+
new_log_dict['last eval'] = val_list[-1]
117+
new_log_dict['best eval'] = max(val_list, key=lambda x: x[metric])
118+
experiment_info_list.append(new_log_dict)
119+
print(f'{config_dir} is processed')
120+
121+
if json_file:
122+
with open(json_file, 'w') as f:
123+
json.dump(experiment_info_list, f, indent=4)
124+
125+
if markdown_file:
126+
lines_to_write = []
127+
for index, log in enumerate(experiment_info_list, 1):
128+
lines_to_write.append(
129+
f"|{index}|{log['method']}|{log['best eval'][metric]}"
130+
f"|{log['best eval']['eval_index']}|"
131+
f"{log['last eval'][metric]}|"
132+
f"{log['last eval']['eval_index']}|{log['last_iter']}|\n")
133+
with open(markdown_file, 'w') as f:
134+
f.write(f'|exp_num|method|{metric} best|best index|'
135+
f'{metric} last|last index|last iter num|\n')
136+
f.write('|:---:|:---:|:---:|:---:|:---:|:---:|:---:|\n')
137+
f.writelines(lines_to_write)
138+
139+
print('processed successfully')
140+
141+
142+
if __name__ == '__main__':
143+
main()

‎.dev/log_collector/readme.md‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# Log Collector
2+
3+
## Function
4+
5+
Automatically collect logs and write the result in a json file or markdown file.
6+
7+
If there are several `.log.json` files in one folder, Log Collector assumes that the `.log.json` files other than the first one are resume from the preceding `.log.json` file. Log Collector returns the result considering all `.log.json` files.
8+
9+
## Usage:
10+
11+
To use log collector, you need to write a config file to configure the log collector first.
12+
13+
For example:
14+
15+
example_config.py:
16+
17+
```python
18+
# The work directory that contains folders that contains .log.json files.
19+
work_dir = '../../work_dirs'
20+
# The metric used to find the best evaluation.
21+
metric = 'mIoU'
22+
23+
# **Don't specify the log_items and ignore_keywords at the same time.**
24+
# Specify the log files we would like to collect in `log_items`.
25+
# The folders specified should be the subdirectories of `work_dir`.
26+
log_items = [
27+
'segformer_mit-b5_512x512_160k_ade20k_cnn_lr_with_warmup',
28+
'segformer_mit-b5_512x512_160k_ade20k_cnn_no_warmup_lr',
29+
'segformer_mit-b5_512x512_160k_ade20k_mit_trans_lr',
30+
'segformer_mit-b5_512x512_160k_ade20k_swin_trans_lr'
31+
]
32+
# Or specify `ignore_keywords`. The folders whose name contain one
33+
# of the keywords in the `ignore_keywords` list(e.g., `'segformer'`)
34+
# won't be collected.
35+
# ignore_keywords = ['segformer']
36+
37+
# Other log items in .log.json that you want to collect.
38+
# should not include metric.
39+
other_info_keys = ["mAcc"]
40+
# The output markdown file's name.
41+
markdown_file ='markdowns/lr_in_trans.json.md'
42+
# The output json file's name. (optional)
43+
json_file = 'jsons/trans_in_cnn.json'
44+
```
45+
46+
The structure of the work-dir directory should be like：
47+
48+
```text
49+
├── work-dir
50+
│ ├── folder1
51+
│ │ ├── time1.log.json
52+
│ │ ├── time2.log.json
53+
│ │ ├── time3.log.json
54+
│ │ ├── time4.log.json
55+
│ ├── folder2
56+
│ │ ├── time5.log.json
57+
│ │ ├── time6.log.json
58+
│ │ ├── time7.log.json
59+
│ │ ├── time8.log.json
60+
```
61+
62+
Then , cd to the log collector folder.
63+
64+
Now you can run log_collector.py by using command:
65+
66+
```bash
67+
python log_collector.py ./example_config.py
68+
```
69+
70+
The output markdown file is like:
71+
72+
|exp_num|method|mIoU best|best index|mIoU last|last index|last iter num|
73+
|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
74+
|1|segformer_mit-b5_512x512_160k_ade20k_cnn_lr_with_warmup|0.2776|10|0.2776|10|160000|
75+
|2|segformer_mit-b5_512x512_160k_ade20k_cnn_no_warmup_lr|0.2802|10|0.2802|10|160000|
76+
|3|segformer_mit-b5_512x512_160k_ade20k_mit_trans_lr|0.4943|11|0.4943|11|160000|
77+
|4|segformer_mit-b5_512x512_160k_ade20k_swin_trans_lr|0.4883|11|0.4883|11|160000|
78+
79+
The output json file is like:
80+
```json
81+
[
82+
{
83+
"method": "segformer_mit-b5_512x512_160k_ade20k_cnn_lr_with_warmup",
84+
"metric_used": "mIoU",
85+
"last_iter": 160000,
86+
"last eval": {
87+
"eval_index": 10,
88+
"mIoU": 0.2776,
89+
"mAcc": 0.3779
90+
},
91+
"best eval": {
92+
"eval_index": 10,
93+
"mIoU": 0.2776,
94+
"mAcc": 0.3779
95+
}
96+
},
97+
{
98+
"method": "segformer_mit-b5_512x512_160k_ade20k_cnn_no_warmup_lr",
99+
"metric_used": "mIoU",
100+
"last_iter": 160000,
101+
"last eval": {
102+
"eval_index": 10,
103+
"mIoU": 0.2802,
104+
"mAcc": 0.3764
105+
},
106+
"best eval": {
107+
"eval_index": 10,
108+
"mIoU": 0.2802,
109+
"mAcc": 0.3764
110+
}
111+
},
112+
{
113+
"method": "segformer_mit-b5_512x512_160k_ade20k_mit_trans_lr",
114+
"metric_used": "mIoU",
115+
"last_iter": 160000,
116+
"last eval": {
117+
"eval_index": 11,
118+
"mIoU": 0.4943,
119+
"mAcc": 0.6097
120+
},
121+
"best eval": {
122+
"eval_index": 11,
123+
"mIoU": 0.4943,
124+
"mAcc": 0.6097
125+
}
126+
},
127+
{
128+
"method": "segformer_mit-b5_512x512_160k_ade20k_swin_trans_lr",
129+
"metric_used": "mIoU",
130+
"last_iter": 160000,
131+
"last eval": {
132+
"eval_index": 11,
133+
"mIoU": 0.4883,
134+
"mAcc": 0.6061
135+
},
136+
"best eval": {
137+
"eval_index": 11,
138+
"mIoU": 0.4883,
139+
"mAcc": 0.6061
140+
}
141+
}
142+
]
143+
```

‎.dev/log_collector/utils.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Copyright (c) OpenMMLab. All rights reserved.
2+
# modified from https://github.dev/open-mmlab/mmcv
3+
import os.path as osp
4+
import sys
5+
from importlib import import_module
6+
7+
8+
def load_config(cfg_dir: str) -> dict:
9+
assert cfg_dir.endswith('.py')
10+
root_path, file_name = osp.split(cfg_dir)
11+
temp_module = osp.splitext(file_name)[0]
12+
sys.path.insert(0, root_path)
13+
mod = import_module(temp_module)
14+
sys.path.pop(0)
15+
cfg_dict = {
16+
k: v
17+
for k, v in mod.__dict__.items() if not k.startswith('__')
18+
}
19+
del sys.modules[temp_module]
20+
return cfg_dict

0 commit comments

Comments
 (0)