Skip to content

Commit e494c00

Browse files
hangzhaomithangzhaomit
authored
Merge pull request CSAILVision#244 from davidbau/tutorial
Add a tutorial notebook.
2 parents 4a4d88a + df30f12 commit e494c00

File tree

11 files changed

+359
-5
lines changed

11 files changed

+359
-5
lines changed

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*.ipynb filter=clean_ipynb

‎.gitignore‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,9 @@ vis/
55
log/
66
pretrained/
77

8-
.idea/
8+
.ipynb_checkpoints
9+
10+
ADE_val*.jpg
11+
ADE_val*.png
12+
13+
.idea/

‎demo_test.sh‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,16 @@
22

33
# Image and model names
44
TEST_IMG=ADE_val_00001519.jpg
5-
MODEL_PATH=ade20k-resnet50dilated-ppm_deepsup
5+
MODEL_NAME=ade20k-resnet50dilated-ppm_deepsup
6+
MODEL_PATH=ckpt/$MODEL_NAME
67
RESULT_PATH=./
78

8-
ENCODER=$MODEL_PATH/encoder_epoch_20.pth
9-
DECODER=$MODEL_PATH/decoder_epoch_20.pth
9+
ENCODER=$MODEL_NAME/encoder_epoch_20.pth
10+
DECODER=$MODEL_NAME/decoder_epoch_20.pth
1011

1112
# Download model weights and image
1213
if [ ! -e $MODEL_PATH ]; then
13-
mkdir $MODEL_PATH
14+
mkdir -p $MODEL_PATH
1415
fi
1516
if [ ! -e $ENCODER ]; then
1617
wget -P $MODEL_PATH http://sceneparsing.csail.mit.edu/model/pytorch/$ENCODER
@@ -22,10 +23,15 @@ if [ ! -e $TEST_IMG ]; then
2223
wget -P $RESULT_PATH http://sceneparsing.csail.mit.edu/data/ADEChallengeData2016/images/validation/$TEST_IMG
2324
fi
2425

26+
if [ -z "$DOWNLOAD_ONLY" ]
27+
then
28+
2529
# Inference
2630
python3 -u test.py \
2731
--imgs $TEST_IMG \
2832
--cfg config/ade20k-resnet50dilated-ppm_deepsup.yaml \
2933
DIR $MODEL_PATH \
3034
TEST.result ./ \
3135
TEST.checkpoint epoch_20.pth
36+
37+
fi

‎notebooks/DemoSegmenter.ipynb‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# Semantic Segmentation Demo\n",
8+
"\n",
9+
"This is a notebook for running the benchmark semantic segmentation network from the the [ADE20K MIT Scene Parsing Benchchmark](http://sceneparsing.csail.mit.edu/).\n",
10+
"\n",
11+
"The code for this notebook is available here\n",
12+
"https://github.com/davidbau/semantic-segmentation-pytorch/tree/tutorial/notebooks\n",
13+
"\n",
14+
"It can be run on Colab at this URL https://colab.research.google.com/github/davidbau/semantic-segmentation-pytorch/blob/tutorial/notebooks/DemoSegmenter.ipynb"
15+
]
16+
},
17+
{
18+
"cell_type": "markdown",
19+
"metadata": {},
20+
"source": [
21+
"### Environment Setup\n",
22+
"\n",
23+
"First, download the code and pretrained models if we are on colab."
24+
]
25+
},
26+
{
27+
"cell_type": "code",
28+
"execution_count": null,
29+
"metadata": {},
30+
"outputs": [],
31+
"source": [
32+
"%%bash\n",
33+
"# Colab-specific setup\n",
34+
"!(stat -t /usr/local/lib/*/dist-packages/google/colab > /dev/null 2>&1) && exit \n",
35+
"pip install yacs 2>&1 >> install.log\n",
36+
"git init 2>&1 >> install.log\n",
37+
"git remote add origin https://github.com/davidbau/semantic-segmentation-pytorch.git 2>> install.log\n",
38+
"git pull origin tutorial 2>&1 >> install.log\n",
39+
"DOWNLOAD_ONLY=1 ./demo_test.sh 2>> install.log"
40+
]
41+
},
42+
{
43+
"cell_type": "markdown",
44+
"metadata": {},
45+
"source": [
46+
"## Imports and utility functions\n",
47+
"\n",
48+
"We need pytorch, numpy, and the code for the segmentation model. And some utilities for visualizing the data."
49+
]
50+
},
51+
{
52+
"cell_type": "code",
53+
"execution_count": null,
54+
"metadata": {},
55+
"outputs": [],
56+
"source": [
57+
"# System libs\n",
58+
"import os, csv, torch, numpy, scipy.io, PIL.Image, torchvision.transforms\n",
59+
"# Our libs\n",
60+
"from mit_semseg.models import ModelBuilder, SegmentationModule\n",
61+
"from mit_semseg.utils import colorEncode\n",
62+
"\n",
63+
"colors = scipy.io.loadmat('data/color150.mat')['colors']\n",
64+
"names = {}\n",
65+
"with open('data/object150_info.csv') as f:\n",
66+
" reader = csv.reader(f)\n",
67+
" next(reader)\n",
68+
" for row in reader:\n",
69+
" names[int(row[0])] = row[5].split(\";\")[0]\n",
70+
"\n",
71+
"def visualize_result(img, pred, index=None):\n",
72+
" # filter prediction class if requested\n",
73+
" if index is not None:\n",
74+
" pred = pred.copy()\n",
75+
" pred[pred != index] = -1\n",
76+
" print(f'{names[index+1]}:')\n",
77+
" \n",
78+
" # colorize prediction\n",
79+
" pred_color = colorEncode(pred, colors).astype(numpy.uint8)\n",
80+
"\n",
81+
" # aggregate images and save\n",
82+
" im_vis = numpy.concatenate((img, pred_color), axis=1)\n",
83+
" display(PIL.Image.fromarray(im_vis))"
84+
]
85+
},
86+
{
87+
"cell_type": "markdown",
88+
"metadata": {},
89+
"source": [
90+
"## Loading the segmentation model\n",
91+
"\n",
92+
"Here we load a pretrained segmentation model. Like any pytorch model, we can call it like a function, or examine the parameters in all the layers.\n",
93+
"\n",
94+
"After loading, we put it on the GPU. And since we are doing inference, not training, we put the model in eval mode."
95+
]
96+
},
97+
{
98+
"cell_type": "code",
99+
"execution_count": null,
100+
"metadata": {},
101+
"outputs": [],
102+
"source": [
103+
"# Network Builders\n",
104+
"net_encoder = ModelBuilder.build_encoder(\n",
105+
" arch='resnet50dilated',\n",
106+
" fc_dim=2048,\n",
107+
" weights='ckpt/ade20k-resnet50dilated-ppm_deepsup/encoder_epoch_20.pth')\n",
108+
"net_decoder = ModelBuilder.build_decoder(\n",
109+
" arch='ppm_deepsup',\n",
110+
" fc_dim=2048,\n",
111+
" num_class=150,\n",
112+
" weights='ckpt/ade20k-resnet50dilated-ppm_deepsup/decoder_epoch_20.pth',\n",
113+
" use_softmax=True)\n",
114+
"\n",
115+
"crit = torch.nn.NLLLoss(ignore_index=-1)\n",
116+
"segmentation_module = SegmentationModule(net_encoder, net_decoder, crit)\n",
117+
"segmentation_module.eval()\n",
118+
"segmentation_module.cuda()"
119+
]
120+
},
121+
{
122+
"cell_type": "markdown",
123+
"metadata": {},
124+
"source": [
125+
"## Load test data\n",
126+
"\n",
127+
"Now we load and normalize a single test image. Here we use the commonplace convention of normalizing the image to a scale for which the RGB values of a large photo dataset would have zero mean and unit standard deviation. (These numbers come from the imagenet dataset.) With this normalization, the limiiting ranges of RGB values are within about (-2.2 to +2.7)."
128+
]
129+
},
130+
{
131+
"cell_type": "code",
132+
"execution_count": null,
133+
"metadata": {},
134+
"outputs": [],
135+
"source": [
136+
"# Load and normalize one image as a singleton tensor batch\n",
137+
"pil_to_tensor = torchvision.transforms.Compose([\n",
138+
" torchvision.transforms.ToTensor(),\n",
139+
" torchvision.transforms.Normalize(\n",
140+
" mean=[0.485, 0.456, 0.406], # These are RGB mean+std values\n",
141+
" std=[0.229, 0.224, 0.225]) # across a large photo dataset.\n",
142+
"])\n",
143+
"pil_image = PIL.Image.open('ADE_val_00001519.jpg').convert('RGB')\n",
144+
"img_original = numpy.array(pil_image)\n",
145+
"img_data = pil_to_tensor(pil_image)\n",
146+
"singleton_batch = {'img_data': img_data[None].cuda()}\n",
147+
"output_size = img_data.shape[1:]"
148+
]
149+
},
150+
{
151+
"cell_type": "markdown",
152+
"metadata": {},
153+
"source": [
154+
"## Run the Model\n",
155+
"\n",
156+
"Finally we just pass the test image to the segmentation model.\n",
157+
"\n",
158+
"The segmentation model is coded as a function that takes a dictionary as input, because it wants to know both the input batch image data as well as the desired output segmentation resolution. We ask for full resolution output.\n",
159+
"\n",
160+
"Then we use the previously-defined visualize_result function to render the semgnatioon map."
161+
]
162+
},
163+
{
164+
"cell_type": "code",
165+
"execution_count": null,
166+
"metadata": {
167+
"scrolled": false
168+
},
169+
"outputs": [],
170+
"source": [
171+
"# Run the segmentation at the highest resolution.\n",
172+
"with torch.no_grad():\n",
173+
" scores = segmentation_module(singleton_batch, segSize=output_size)\n",
174+
" \n",
175+
"# Get the predicted scores for each pixel\n",
176+
"_, pred = torch.max(scores, dim=1)\n",
177+
"pred = pred.cpu()[0].numpy()\n",
178+
"visualize_result(img_original, pred)"
179+
]
180+
},
181+
{
182+
"cell_type": "markdown",
183+
"metadata": {},
184+
"source": [
185+
"## Showing classes individually\n",
186+
"\n",
187+
"To see which colors are which, here we visualize individual classes, one at a time."
188+
]
189+
},
190+
{
191+
"cell_type": "code",
192+
"execution_count": null,
193+
"metadata": {},
194+
"outputs": [],
195+
"source": [
196+
"# Top classes in answer\n",
197+
"predicted_classes = numpy.bincount(pred.flatten()).argsort()[::-1]\n",
198+
"for c in predicted_classes[:15]:\n",
199+
" visualize_result(img_original, pred, c)"
200+
]
201+
}
202+
],
203+
"metadata": {
204+
"accelerator": "GPU",
205+
"kernelspec": {
206+
"display_name": "Python 3",
207+
"language": "python",
208+
"name": "python3"
209+
},
210+
"language_info": {
211+
"codemirror_mode": {
212+
"name": "ipython",
213+
"version": 3
214+
},
215+
"file_extension": ".py",
216+
"mimetype": "text/x-python",
217+
"name": "python",
218+
"nbconvert_exporter": "python",
219+
"pygments_lexer": "ipython3",
220+
"version": "3.6.7"
221+
}
222+
},
223+
"nbformat": 4,
224+
"nbformat_minor": 2
225+
}

‎notebooks/ckpt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../ckpt

‎notebooks/config‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../config

‎notebooks/data‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../data

‎notebooks/ipynb_drop_output.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/usr/bin/env python
2+
3+
"""
4+
Suppress output and prompt numbers in git version control.
5+
6+
This script will tell git to ignore prompt numbers and cell output
7+
when looking at ipynb files UNLESS their metadata contains:
8+
9+
"git" : { "keep_output" : true }
10+
11+
The notebooks themselves are not changed.
12+
13+
See also this blogpost: http://pascalbugnion.net/blog/ipython-notebooks-and-git.html.
14+
15+
Usage instructions
16+
==================
17+
18+
1. Put this script in a directory that is on the system's path.
19+
For future reference, I will assume you saved it in
20+
`~/scripts/ipynb_drop_output`.
21+
2. Make sure it is executable by typing the command
22+
`chmod +x ~/scripts/ipynb_drop_output`.
23+
3. Register a filter for ipython notebooks by
24+
putting the following line in `~/.config/git/attributes`:
25+
`*.ipynb filter=clean_ipynb`
26+
4. Connect this script to the filter by running the following
27+
git commands:
28+
29+
git config --global filter.clean_ipynb.clean ipynb_drop_output
30+
git config --global filter.clean_ipynb.smudge cat
31+
32+
To tell git NOT to ignore the output and prompts for a notebook,
33+
open the notebook's metadata (Edit > Edit Notebook Metadata). A
34+
panel should open containing the lines:
35+
36+
{
37+
"name" : "",
38+
"signature" : "some very long hash"
39+
}
40+
41+
Add an extra line so that the metadata now looks like:
42+
43+
{
44+
"name" : "",
45+
"signature" : "don't change the hash, but add a comma at the end of the line",
46+
"git" : { "keep_outputs" : true }
47+
}
48+
49+
You may need to "touch" the notebooks for git to actually register a change, if
50+
your notebooks are already under version control.
51+
52+
Notes
53+
=====
54+
55+
Changed by David Bau to make stripping output the default.
56+
57+
This script is inspired by http://stackoverflow.com/a/20844506/827862, but
58+
lets the user specify whether the ouptut of a notebook should be kept
59+
in the notebook's metadata, and works for IPython v3.0.
60+
"""
61+
62+
import sys
63+
import json
64+
65+
nb = sys.stdin.read()
66+
67+
json_in = json.loads(nb)
68+
nb_metadata = json_in["metadata"]
69+
keep_output = False
70+
if "git" in nb_metadata:
71+
if "keep_outputs" in nb_metadata["git"] and nb_metadata["git"]["keep_outputs"]:
72+
keep_output = True
73+
if keep_output:
74+
sys.stdout.write(nb)
75+
exit()
76+
77+
78+
ipy_version = int(json_in["nbformat"])-1 # nbformat is 1 more than actual version.
79+
80+
def strip_output_from_cell(cell):
81+
if "outputs" in cell:
82+
cell["outputs"] = []
83+
if "prompt_number" in cell:
84+
del cell["prompt_number"]
85+
if "execution_count" in cell:
86+
cell["execution_count"] = None
87+
88+
89+
if ipy_version == 2:
90+
for sheet in json_in["worksheets"]:
91+
for cell in sheet["cells"]:
92+
strip_output_from_cell(cell)
93+
else:
94+
for cell in json_in["cells"]:
95+
strip_output_from_cell(cell)
96+
97+
json.dump(json_in, sys.stdout, sort_keys=True, indent=1, separators=(",",": "))

‎notebooks/mit_semseg‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../mit_semseg

0 commit comments

Comments
 (0)