added code for EuroPython 2017 upcoming demo

Author: Roberto De Ioris

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/eyes.py

@@ -0,0 +1,46 @@
+import unreal_engine as ue
+
+from unreal_engine.classes import SceneCaptureComponent2D, PyHUD
+from unreal_engine.enums import ESceneCaptureSource
+
+import os
+
+import cv2
+import numpy
+
+class Sight:
+
+    def __init__(self):
+        self.what_i_am_seeing = ue.create_transient_texture_render_target2d(512, 512)
+              
+    def pre_initialize_components(self):
+        # add a new root component (a SceneCaptureComponent2D one)
+        self.scene_capturer = self.uobject.add_actor_root_component(SceneCaptureComponent2D, 'Scene Capture')
+        # use the previously created texture as the render target
+        self.scene_capturer.TextureTarget = self.what_i_am_seeing
+        # store pixels as linear colors (non HDR)
+        self.scene_capturer.CaptureSource = ESceneCaptureSource.SCS_FinalColorLDR
+
+    def begin_play(self):
+        # get a reference to the pawn currently controlled by the player
+        mannequin = self.uobject.get_player_pawn()
+        # attach myself to the 'head' bone of the mannequin Mesh component
+        self.uobject.attach_to_component(mannequin.Mesh, 'head')
+
+        # spawn a new HUD (well, a PyHUD)
+        hud = self.uobject.actor_spawn(PyHUD, PythonModule='hud', PythonClass='FacesDetector')
+        # get a reference to its proxy class
+        self.py_hud = hud.get_py_proxy()
+
+        # set the texture to draw
+        self.py_hud.texture_to_draw = self.what_i_am_seeing
+
+        # use this new HUD as the player default one (so the engine will start drawing it)
+        self.uobject.set_player_hud(hud)
+
+    def tick(self, delta_time):
+        # read raw data from the render target
+        data = self.what_i_am_seeing.render_target_get_data()
+        # convert them to a numpy array with the format expected by OpenCV
+        npy_data = numpy.frombuffer(data, dtype=numpy.uint8).reshape(512, 512, 4)
+        self.py_hud.gray_data = cv2.cvtColor(npy_data, cv2.COLOR_BGRA2GRAY)

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/eyes_fifth.py

@@ -0,0 +1,45 @@
+import unreal_engine as ue
+
+from unreal_engine.classes import SceneCaptureComponent2D, PyHUD
+from unreal_engine.enums import ESceneCaptureSource
+
+import cv2
+import numpy
+
+class Sight:
+
+    def __init__(self):
+        self.what_i_am_seeing = ue.create_transient_texture_render_target2d(512, 512)
+
+    def pre_initialize_components(self):
+        # add a new root component (a SceneCaptureComponent2D one)
+        self.scene_capturer = self.uobject.add_actor_root_component(SceneCaptureComponent2D, 'Scene Capture')
+        # use the previously created texture as the render target
+        self.scene_capturer.TextureTarget = self.what_i_am_seeing
+        # store pixels as linear colors (non HDR)
+        self.scene_capturer.CaptureSource = ESceneCaptureSource.SCS_FinalColorLDR
+
+    def begin_play(self):
+        # get a reference to the pawn currently controlled by the player
+        mannequin = self.uobject.get_player_pawn()
+        # attach myself to the 'head' bone of the mannequin Mesh component
+        self.uobject.attach_to_component(mannequin.Mesh, 'head')
+
+        # spawn a new HUD (well, a PyHUD)
+        hud = self.uobject.actor_spawn(PyHUD, PythonModule='hud_second', PythonClass='FacesDetector')
+        # get a reference to its proxy class
+        self.py_hud = hud.get_py_proxy()
+
+        # set the texture to draw
+        self.py_hud.texture_to_draw = self.what_i_am_seeing
+
+        # use this new HUD as the player default one (so the engine will start drawing it)
+        self.uobject.set_player_hud(hud)
+
+    def tick(self, delta_time):
+        # read raw data from the render target
+        data = self.what_i_am_seeing.render_target_get_data()
+        # convert them to a numpy array with the format expected by OpenCV
+        npy_data = numpy.frombuffer(data, dtype=numpy.uint8).reshape(512, 512, 4)
+        # pass data to the HUD
+        self.py_hud.gray_data = cv2.cvtColor(npy_data, cv2.COLOR_BGRA2GRAY)

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/eyes_first.py

@@ -0,0 +1,17 @@
+import unreal_engine as ue
+
+
+class Sight:
+
+    def __init__(self):
+        ue.log_warning('Hello i am __init__')
+        self.timer = 1.0
+              
+    def begin_play(self):
+        ue.log_warning('Hello i am begin_play')        
+
+    def tick(self, delta_time):
+        self.timer -= delta_time
+        if self.timer <= 0:
+            ue.log_error('1 second elapsed !')
+            self.timer = 1.0

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/eyes_fourth.py

@@ -0,0 +1,34 @@
+import unreal_engine as ue
+
+from unreal_engine.classes import SceneCaptureComponent2D, PyHUD
+from unreal_engine.enums import ESceneCaptureSource
+
+class Sight:
+
+    def __init__(self):
+        self.what_i_am_seeing = ue.create_transient_texture_render_target2d(512, 512)
+
+    def pre_initialize_components(self):
+        # add a new root component (a SceneCaptureComponent2D one)
+        self.scene_capturer = self.uobject.add_actor_root_component(SceneCaptureComponent2D, 'Scene Capture')
+        # use the previously created texture as the render target
+        self.scene_capturer.TextureTarget = self.what_i_am_seeing
+        # store pixels as linear colors (non HDR)
+        self.scene_capturer.CaptureSource = ESceneCaptureSource.SCS_FinalColorLDR
+
+    def begin_play(self):
+        # get a reference to the pawn currently controlled by the player
+        mannequin = self.uobject.get_player_pawn()
+        # attach myself to the 'head' bone of the mannequin Mesh component
+        self.uobject.attach_to_component(mannequin.Mesh, 'head')
+
+        # spawn a new HUD (well, a PyHUD)
+        hud = self.uobject.actor_spawn(PyHUD, PythonModule='hud_first', PythonClass='FacesDetector')
+        # get a reference to its proxy class
+        self.py_hud = hud.get_py_proxy()
+
+        # set the texture to draw
+        self.py_hud.texture_to_draw = self.what_i_am_seeing
+
+        # use this new HUD as the player default one (so the engine will start drawing it)
+        self.uobject.set_player_hud(hud)

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/eyes_second.py

@@ -0,0 +1,17 @@
+import unreal_engine as ue
+
+from unreal_engine.classes import SceneCaptureComponent2D
+from unreal_engine.enums import ESceneCaptureSource
+
+class Sight:
+
+    def __init__(self):
+        self.what_i_am_seeing = ue.create_transient_texture_render_target2d(512, 512)
+
+    def pre_initialize_components(self):
+        # add a new root component (a SceneCaptureComponent2D one)
+        self.scene_capturer = self.uobject.add_actor_root_component(SceneCaptureComponent2D, 'Scene Capture')
+        # use the previously created texture as the render target
+        self.scene_capturer.TextureTarget = self.what_i_am_seeing
+        # store pixels as linear colors (non HDR)
+        self.scene_capturer.CaptureSource = ESceneCaptureSource.SCS_FinalColorLDR

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/eyes_third.py

@@ -0,0 +1,23 @@
+import unreal_engine as ue
+
+from unreal_engine.classes import SceneCaptureComponent2D
+from unreal_engine.enums import ESceneCaptureSource
+
+class Sight:
+
+    def __init__(self):
+        self.what_i_am_seeing = ue.create_transient_texture_render_target2d(512, 512)
+
+    def pre_initialize_components(self):
+        # add a new root component (a SceneCaptureComponent2D one)
+        self.scene_capturer = self.uobject.add_actor_root_component(SceneCaptureComponent2D, 'Scene Capture')
+        # use the previously created texture as the render target
+        self.scene_capturer.TextureTarget = self.what_i_am_seeing
+        # store pixels as linear colors (non HDR)
+        self.scene_capturer.CaptureSource = ESceneCaptureSource.SCS_FinalColorLDR
+
+    def begin_play(self):
+        # get a reference to the pawn currently controlled by the player
+        mannequin = self.uobject.get_player_pawn()
+        # attach myself to the 'head' bone of the mannequin Mesh component
+        self.uobject.attach_to_component(mannequin.Mesh, 'head')

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/hud.py

@@ -0,0 +1,42 @@
+import unreal_engine as ue
+
+from unreal_engine import FColor, FLinearColor
+
+from unreal_engine.enums import EBlendMode, EPixelFormat
+
+
+import os
+import cv2
+
+class FacesDetector:
+
+    def __init__(self):
+        self.texture_to_draw = None
+        self.gray_data = None
+        
+        self.texture = ue.create_transient_texture(512, 512, EPixelFormat.PF_G8)
+        self.texture.auto_root()
+        filename = os.path.join(ue.get_content_dir(), 'haarcascade_frontalface_default.xml')
+        self.cascade = cv2.CascadeClassifier(filename)
+        
+    def draw_hud(self):
+        # exit if we do not have enough data
+        if not self.texture_to_draw:
+            return
+
+        # upload grayscale data into the GPU
+        self.texture.texture_set_data(self.gray_data)
+        
+        # draw what the player pawn is seeing
+        self.uobject.hud_draw_texture(self.texture_to_draw, 0, 0, 256, 256)
+
+        # draw in opaque mode (ignore transparency)
+        self.uobject.hud_draw_texture(self.texture, 0, 256, 256, 256, 0, 0, 1, 1, FLinearColor(1, 1, 1, 1), EBlendMode.BLEND_Opaque)
+    
+        # detect faces
+        faces = self.cascade.detectMultiScale(self.gray_data)
+        # highlight detected areas
+        for x, y, w, h in faces:
+            self.uobject.hud_draw_rect(x/2.0, 256 + y/2.0, w/2.0, h/2.0, FLinearColor(0, 1, 0, 0.5))
+        # report the number of detected faces
+        self.uobject.hud_draw_text('I am seeing {0} faces'.format(len(faces)), 500, 50, FLinearColor(0, 1, 1, 1), 2)

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/hud_first.py

@@ -0,0 +1,14 @@
+import unreal_engine as ue
+
+class FacesDetector:
+
+    def __init__(self):
+        self.texture_to_draw = None
+               
+    def draw_hud(self):
+        # exit if we do not have enough data
+        if not self.texture_to_draw:
+            return
+
+        # draw what the player pawn is seeing
+        self.uobject.hud_draw_texture(self.texture_to_draw, 0, 0, 256, 256)

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/hud_second.py

@@ -0,0 +1,34 @@
+import unreal_engine as ue
+
+from unreal_engine import FColor, FLinearColor
+
+from unreal_engine.enums import EBlendMode, EPixelFormat
+
+
+import os
+import cv2
+
+class FacesDetector:
+
+    def __init__(self):
+        self.texture_to_draw = None
+        self.gray_data = None
+        
+        # create a transient texture with a single color channel
+        self.texture = ue.create_transient_texture(512, 512, EPixelFormat.PF_G8)
+        # this automatically destroys the texture when self.texture dies (this is required as UE does not know about this texture object)
+        self.texture.auto_root()
+                
+    def draw_hud(self):
+        # exit if we do not have enough data
+        if not self.texture_to_draw:
+            return
+
+        # upload grayscale data into the GPU
+        self.texture.texture_set_data(self.gray_data)
+        
+        # draw what the player pawn is seeing
+        self.uobject.hud_draw_texture(self.texture_to_draw, 0, 0, 256, 256)
+
+        # draw in opaque mode (ignore transparency)
+        self.uobject.hud_draw_texture(self.texture, 0, 256, 256, 256, 0, 0, 1, 1, FLinearColor(1, 1, 1, 1), EBlendMode.BLEND_Opaque)

tutorials/FaceRecognitionWithOpenCVAndUnrealEnginePython_Assets/utils.py

@@ -0,0 +1,31 @@
+import unreal_engine as ue
+
+from unreal_engine.classes import Material
+from unreal_engine.enums import EPixelFormat
+
+import cv2
+
+class WebCam:
+
+    def begin_play(self):
+        # open the first video capture device
+        self.capture = cv2.VideoCapture(0)
+        # get its size
+        width = int(self.capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(self.capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        # create a new texture with the desired size
+        self.texture = ue.create_transient_texture(width, height, EPixelFormat.PF_R8G8B8A8)
+
+        # assign the texture to the material mapped to the cube
+        self.material = self.uobject.create_material_instance_dynamic(ue.load_object(Material, '/Game/DumbMaterial'))
+        self.uobject.get_owner().StaticMeshComponent.set_material(0, self.material)
+        self.material.set_material_texture_parameter('Texture', self.texture)
+
+    def tick(self, delta_time):
+        # get webcam frame
+        ret, frame = self.capture.read()
+        if not ret:
+            return
+        # frames are in plain RGB, convert them to a texture-useful format
+        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGBA)
+        self.texture.texture_set_data(frame)