Skip to content

Commit eb87207

Browse files
agrechnevagrechnev
committed
Added 2_1, 2_2
1 parent 9836fb1 commit eb87207

File tree

4 files changed

+333
-0
lines changed

4 files changed

+333
-0
lines changed

‎first_steps/2_1/BUILD

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
cc_binary(
2+
name="2_1",
3+
srcs=["main.cpp"],
4+
deps=[
5+
"//mediapipe/framework:calculator_framework",
6+
"//mediapipe/calculators/core:pass_through_calculator",
7+
"//mediapipe/framework/formats:image_frame",
8+
"//mediapipe/framework/formats:image_frame_opencv",
9+
"//mediapipe/framework/port:opencv_highgui",
10+
"//mediapipe/framework/port:opencv_imgproc",
11+
"//mediapipe/framework/port:parse_text_proto",
12+
"//mediapipe/framework/port:status",
13+
],
14+
)

‎first_steps/2_1/main.cpp

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/// Example 2.1 : Video pipeline
2+
/// By Oleksiy Grechnyev, IT-JIM
3+
/// Here I create our very first video pipeline
4+
5+
/// Here we use OpenCV, and MP has its own wrappers for opencv headers (How stupid is this?)
6+
/// MP uses an mutable class ImageFrame for images
7+
/// One must convert cv::Mat <-> ImageFrame back and forth
8+
/// Note that cv::Mat uses BGR by default, while ImageFrame uses RGB by default
9+
/// We stick to this convention and put cv::cvtColor when needed
10+
11+
/// ACHTUNG! This pipeline is NOT real-time!
12+
/// With the default execution policy, MP never loses packets
13+
/// While it's unlikely to happen with PassThroughCalculator,
14+
/// If our pipeline was too slow, the packets would
15+
/// queue indefinitely, increasing the lag, and eventually
16+
/// crashing the computer.
17+
/// Hopefully I'll make a real-time example later.
18+
19+
20+
#include <iostream>
21+
#include <string>
22+
#include <memory>
23+
#include <atomic>
24+
25+
#include "mediapipe/framework/calculator_framework.h"
26+
#include "mediapipe/framework/formats/image_frame.h"
27+
#include "mediapipe/framework/formats/image_frame_opencv.h"
28+
#include "mediapipe/framework/port/parse_text_proto.h"
29+
#include "mediapipe/framework/port/status.h"
30+
31+
#include "mediapipe/framework/port/opencv_highgui_inc.h"
32+
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
33+
34+
35+
//==============================================================================
36+
mediapipe::Status run() {
37+
using namespace std;
38+
using namespace mediapipe;
39+
40+
// A rather trivial graph by now
41+
string protoG = R"(
42+
input_stream: "in",
43+
output_stream: "out",
44+
node {
45+
calculator: "PassThroughCalculator",
46+
input_stream: "in",
47+
output_stream: "out",
48+
}
49+
)";
50+
// Parse config and create graph
51+
CalculatorGraphConfig config;
52+
if (!ParseTextProto<mediapipe::CalculatorGraphConfig>(protoG, &config)) {
53+
// mediapipe::Status is actually absl::Status (at least in the current mediapipe)
54+
// So we can create BAD statuses like this
55+
return absl::InternalError("Cannot parse the graph config !");
56+
}
57+
CalculatorGraph graph;
58+
MP_RETURN_IF_ERROR(graph.Initialize(config));
59+
60+
// Add observer to "out", then start the graph
61+
// This callback displays the frame on the screen
62+
auto cb = [](const Packet &packet)->Status{
63+
cout << packet.Timestamp() << ": RECEIVED VIDEO PACKET !" << endl;
64+
65+
// Get data from packet (you should be used to this by now)
66+
const ImageFrame & outputFrame = packet.Get<ImageFrame>();
67+
// Represent ImageFrame data as cv::Mat (MatView is a thin wrapper, no copying)
68+
cv::Mat ofMat = formats::MatView(&outputFrame);
69+
// Convert RGB->BGR
70+
cv::Mat frameOut;
71+
cvtColor(ofMat, frameOut, cv::COLOR_RGB2BGR);
72+
// Display frame on screen and quit on ESC
73+
// Returning non-OK status aborts graph execution
74+
// I'll make a nicer quit in later examples
75+
cv::imshow("frameOut", frameOut);
76+
if (27 == cv::waitKey(1))
77+
// I was not sure which Abseil error to use here ...
78+
return absl::CancelledError("It's time to QUIT !");
79+
else
80+
return OkStatus();
81+
};
82+
MP_RETURN_IF_ERROR(graph.ObserveOutputStream("out", cb));
83+
graph.StartRun({});
84+
85+
// Start the camera and check that it works
86+
cv::VideoCapture cap(cv::CAP_ANY);
87+
if (!cap.isOpened())
88+
return absl::NotFoundError("CANNOT OPEN CAMERA !");
89+
cv::Mat frameIn, frameInRGB;
90+
91+
// Endless loop over frames
92+
for (int i=0; ; ++i){
93+
// Read next frame from camera
94+
cap.read(frameIn);
95+
if (frameIn.empty())
96+
return absl::NotFoundError("CANNOT OPEN CAMERA !");
97+
// Convert BGR to RGB
98+
cv::cvtColor(frameIn, frameInRGB, cv::COLOR_BGR2RGB);
99+
// Create an empty (black?) RGB ImageFrame with the same size as our image
100+
// Note the raw pointer. Is there a memory leak?
101+
// NO, because of Adopt(), see below
102+
ImageFrame *inputFrame = new ImageFrame(
103+
ImageFormat::SRGB, frameInRGB.cols, frameInRGB.rows, ImageFrame::kDefaultAlignmentBoundary
104+
);
105+
// Copy data from cv::Mat to Imageframe, using
106+
// MatView: a cv::Mat representation of ImageFrame
107+
frameInRGB.copyTo(formats::MatView(inputFrame));
108+
109+
// Create and send a video packet
110+
uint64 ts = i;
111+
// Adopt() creates a new packet from a raw pointer, and takes this pointer under MP management
112+
// So that you must not call delete on the pointer after that
113+
// MP will delete your object automatically when the packet is destroyed
114+
// This is like creating shared_ptr from a raw pointer
115+
// This is useful for the classes which cannot be (easily) copied, like ImageFrame
116+
// Note that MakePacket<...>() we used previously contains a move or copy operation
117+
MP_RETURN_IF_ERROR(graph.AddPacketToInputStream("in",
118+
Adopt(inputFrame).At(Timestamp(ts))
119+
));
120+
}
121+
// We never reach here, the status is always error on exit
122+
// MP_RETURN_IF_ERROR(graph.WaitUntilDone());
123+
// return OkStatus();
124+
}
125+
126+
//==============================================================================
127+
int main(int argc, char** argv){
128+
using namespace std;
129+
130+
cout << "Example 2.1 : Video pipeline" << endl;
131+
mediapipe::Status status = run();
132+
cout << "status =" << status << endl;
133+
cout << "status.ok() = " << status.ok() << endl;
134+
return 0;
135+
}

‎first_steps/2_2/BUILD

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
cc_binary(
2+
name="2_2",
3+
srcs=["main.cpp"],
4+
deps=[
5+
"//mediapipe/framework:calculator_framework",
6+
"//mediapipe/calculators/core:pass_through_calculator",
7+
"//mediapipe/calculators/image:image_cropping_calculator",
8+
"//mediapipe/calculators/image:scale_image_calculator",
9+
"//mediapipe/framework/formats:image_frame",
10+
"//mediapipe/framework/formats:image_frame_opencv",
11+
"//mediapipe/framework/port:opencv_highgui",
12+
"//mediapipe/framework/port:opencv_imgproc",
13+
"//mediapipe/framework/port:parse_text_proto",
14+
"//mediapipe/framework/port:status",
15+
],
16+
)

‎first_steps/2_2/main.cpp

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
/// Example 2.2 : Video pipeline with ImageCroppingCalculator and ScaleImageCalculator
2+
/// By Oleksiy Grechnyev, IT-JIM
3+
/// Here I show how to use standard image claculators from MP
4+
/// ACHTUNG! This pipeline is still NOT real-time!
5+
6+
7+
#include <iostream>
8+
#include <string>
9+
#include <memory>
10+
#include <atomic>
11+
#include <mutex>
12+
13+
#include "mediapipe/framework/calculator_framework.h"
14+
#include "mediapipe/framework/formats/image_frame.h"
15+
#include "mediapipe/framework/formats/image_frame_opencv.h"
16+
#include "mediapipe/framework/port/parse_text_proto.h"
17+
#include "mediapipe/framework/port/status.h"
18+
19+
#include "mediapipe/framework/port/opencv_highgui_inc.h"
20+
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
21+
22+
23+
//==============================================================================
24+
mediapipe::Status run() {
25+
using namespace std;
26+
using namespace mediapipe;
27+
28+
// A graph with ImageCroppingCalculator and ScaleImageCalculator
29+
30+
// First, we use ImageCroppingCalculator
31+
// It crops (cuts) a rectangle from the image
32+
// The desired rectangle can be specified either staticaly (via options)
33+
// Or dynamically (via another stream)
34+
// Here we use the static way
35+
// Note that standard MP calculators often use tags, here it's "IMAGE"
36+
37+
// Second, it is ScaleImageCalculator to scale image to 640x480 (also set statically via the options)
38+
// Note: No tags this time. Doesn't work with tags. Why, google, why?
39+
// Making graphs is fun, isn't it?
40+
string protoG = R"(
41+
input_stream: "in"
42+
output_stream: "out"
43+
node {
44+
calculator: "ImageCroppingCalculator"
45+
input_stream: "IMAGE:in"
46+
output_stream: "IMAGE:out1"
47+
options: {
48+
[mediapipe.ImageCroppingCalculatorOptions.ext] {
49+
norm_width: 0.8
50+
norm_height: 0.4
51+
}
52+
}
53+
}
54+
node {
55+
calculator: "ScaleImageCalculator"
56+
input_stream: "out1"
57+
output_stream: "out"
58+
options: {
59+
[mediapipe.ScaleImageCalculatorOptions.ext] {
60+
target_width: 640
61+
target_height: 480
62+
preserve_aspect_ratio: false
63+
algorithm: CUBIC
64+
}
65+
}
66+
}
67+
)";
68+
69+
// Parse config and create graph
70+
CalculatorGraphConfig config;
71+
if (!ParseTextProto<mediapipe::CalculatorGraphConfig>(protoG, &config)) {
72+
// mediapipe::Status is actually absl::Status (at least in the current mediapipe)
73+
// So we can create BAD statuses like this
74+
return absl::InternalError("Cannot parse the graph config !");
75+
}
76+
CalculatorGraph graph;
77+
MP_RETURN_IF_ERROR(graph.Initialize(config));
78+
79+
// Now we use cv::imshow() in 2 different threads
80+
// For me it worked even without mutex, but let's protect imshow()
81+
// wth a mutex to be safe!
82+
mutex mutexImshow;
83+
84+
// We use an atomic stop flag for a soft quit, without giving graph errors
85+
atomic_bool flagStop(false);
86+
87+
// Add observer to "out", then start the graph
88+
// This callback displays the frame on the screen
89+
auto cb = [&mutexImshow, &flagStop](const Packet &packet)->Status{
90+
91+
// Get cv::Mat from the packet
92+
const ImageFrame & outputFrame = packet.Get<ImageFrame>();
93+
cv::Mat ofMat = formats::MatView(&outputFrame);
94+
cv::Mat frameOut;
95+
cvtColor(ofMat, frameOut, cv::COLOR_RGB2BGR);
96+
cout << packet.Timestamp() << ": RECEIVED VIDEO PACKET size = " << frameOut.size() << endl;
97+
98+
// Let's protect cv::imshow() by the mutex
99+
{
100+
lock_guard<mutex> lock(mutexImshow);
101+
// Display frame on screen and quit on ESC
102+
cv::imshow("frameOut", frameOut);
103+
if (27 == cv::waitKey(1)){
104+
cout << "It's time to QUIT !" << endl;
105+
// Set the stop flag, to finish the camera loop
106+
flagStop = true;
107+
}
108+
}
109+
// ALways return OK now: No errors!
110+
return OkStatus();
111+
};
112+
MP_RETURN_IF_ERROR(graph.ObserveOutputStream("out", cb));
113+
graph.StartRun({});
114+
115+
// Start the camera and check that it works
116+
cv::VideoCapture cap(cv::CAP_ANY);
117+
if (!cap.isOpened())
118+
return absl::NotFoundError("CANNOT OPEN CAMERA !");
119+
cv::Mat frameIn, frameInRGB;
120+
121+
// Camera loop, runs until we get flagStop == true
122+
for (int i=0; !flagStop ; ++i){
123+
// Read next frame from camera
124+
cap.read(frameIn);
125+
if (frameIn.empty())
126+
return absl::NotFoundError("CANNOT OPEN CAMERA !");
127+
128+
// Show input frame, in a different thread from the callback, mutex-protected
129+
// Note: no waitKey() here, we only put waitKey() in the callback
130+
cout << "SIZE_IN = " << frameIn.size() << endl;
131+
{
132+
lock_guard<mutex> lock(mutexImshow);
133+
cv::imshow("frameIn", frameIn);
134+
}
135+
136+
// Convert it to a packet and send
137+
cv::cvtColor(frameIn, frameInRGB, cv::COLOR_BGR2RGB);
138+
ImageFrame *inputFrame = new ImageFrame(
139+
ImageFormat::SRGB, frameInRGB.cols, frameInRGB.rows, ImageFrame::kDefaultAlignmentBoundary
140+
);
141+
frameInRGB.copyTo(formats::MatView(inputFrame));
142+
uint64 ts = i;
143+
MP_RETURN_IF_ERROR(graph.AddPacketToInputStream("in",
144+
Adopt(inputFrame).At(Timestamp(ts))
145+
));
146+
}
147+
// Now we can reach here!
148+
// Don't forget to close the input stream !
149+
graph.CloseInputStream("in");
150+
// Wait for the graph to finish
151+
MP_RETURN_IF_ERROR(graph.WaitUntilDone());
152+
return OkStatus();
153+
}
154+
155+
//==============================================================================
156+
int main(int argc, char** argv){
157+
using namespace std;
158+
159+
FLAGS_alsologtostderr = 1;
160+
google::SetLogDestination(google::GLOG_INFO, ".");
161+
google::InitGoogleLogging(argv[0]);
162+
163+
cout << "Example 2.2 : Video pipeline with ImageCroppingCalculator" << endl;
164+
mediapipe::Status status = run();
165+
cout << "status =" << status << endl;
166+
cout << "status.ok() = " << status.ok() << endl;
167+
return 0;
168+
}

0 commit comments

Comments
 (0)