Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Fonctions proposant de traiter des vidéos
5avec des traitements compliqués type
6:epkg:`deep learning`.
7"""
8import os
9from moviepy.video.io.ImageSequenceClip import ImageSequenceClip
10from ..ai import DLImageSegmentation
11from .video import video_enumerate_frames
12from .moviepy_context import VideoContext
15def video_map_images(video_or_file, name, fLOG=None, **kwargs):
16 """
17 Applies one complex process to a video such as
18 extracting characters from videos and
19 removing the backaground. It is done image by image.
20 Applique un traitement compliqué sur une séquence
21 d'images telle que la séparation des personnages et du fond.
23 @param video_or_file string or :epkg:`VideoClip`
24 @param name name of the processing to do,
25 see the list below
26 @param fLOG logging function
27 @param kwargs additional parameters
28 @return :epkg:`VideoClip`
30 List of available treatments:
32 * ``'people'``: extracts characters from a movie.
33 The movie is composed with an image and a
34 `mask <https://zulko.github.io/moviepy/ref/AudioClip.html?highlight=mask#moviepy.audio.AudioClip.AudioClip.set_ismask>`_.
35 Parameters: see @fn video_map_images_people.
36 * ``'detect'`` : blurs or put a rectangle around faces, uses :epkg:`opencv`
38 .. warning:: A couple of errors timeout, out of memory...
39 The following processes might be quite time consuming
40 or memory consuming. If it is the case, you should think
41 of reducing the resolution, the number of frames per seconds
42 (*fps*). You can also split the video and process each piece
43 independently and finally concatenate them.
44 """
45 allowed = {'people'}
46 if name == 'people':
47 return video_map_images_people(video_or_file, fLOG=fLOG, **kwargs)
48 elif name == "detect":
49 return video_map_images_detect(video_or_file, fLOG=fLOG, **kwargs)
50 else:
51 raise ValueError("Unknown process '{}', should be among: {}".format(
52 name, ','.join(allowed)))
55def video_map_images_people(video_or_file, resize=('max2', 400), fps=None,
56 with_times=False, logger=None, dtype=None,
57 class_to_keep=15, fLOG=None, **kwargs):
58 """
59 Extracts characters from a movie.
60 The movie is composed with an image and a
61 `mask <https://zulko.github.io/moviepy/ref/AudioClip.html?highlight=mask#moviepy.audio.AudioClip.AudioClip.set_ismask>`_.
62 Extrait les personnages d'un film, le résultat est
63 composé d'une image et d'un masque transparent
64 qui laissera apparaître l'image d'en dessous si cette
65 vidéo est aposée sur une autre.
67 @param video_or_file string or :epkg:`VideoClip`
68 @param resize see :meth:`predict <code_beatrix.ai.image_segmentation.DLImageSegmentation.predict>`
69 @param fps see @see fn video_enumerate_frames
70 @param with_times see @see fn video_enumerate_frames
71 @param logger see @see fn video_enumerate_frames
72 @param dtype see @see fn video_enumerate_frames
73 @param class_to_keep class to keep from the image, it can
74 a number (15 for the background, a list of classes,
75 a function which takes an image and the prediction
76 and returns an image)
77 @param fLOG logging function
78 @param kwargs see @see cl DLImageSegmentation
79 @return :epkg:`VideoClip`
81 .. warning:: A couple of errors timeout, out of memory...
82 The following processes might be quite time consuming
83 or memory consuming. If it is the case, you should think
84 of reducing the resolution, the number of frames per seconds
85 (*fps*). You can also split the video and process each piece
86 independently and finally concatenate them.
88 .. exref::
89 :title: Extract characters from a video.
91 The following example shows how to extract a movie with
92 people and without the background. It works better
93 if the contrast between the characters and the background is
94 high.
96 ::
98 from code_beatrix.art.video import video_extract_video, video_save
99 from code_beatrix.art.videodl import video_map_images
101 vide = video_extract_video("something.mp4", 0, 5)
102 vid2 = video_map_images(vide, fps=10, name="people", logger='bar')
103 video_save(vid2, "people.mp4")
105 The function returns something like the the following.
106 The character is wearing black and the background is quite
107 dark too. That explains that the kind of large halo
108 around the character.
110 .. video:: videodl.mp4
111 """
112 if isinstance(class_to_keep, int):
113 def local_mask(img, pred):
114 img[pred != class_to_keep] = 0
115 return img
116 elif isinstance(class_to_keep, (set, tuple, list)):
117 def local_mask(img, pred):
118 dist = set(pred.ravel())
119 rem = set(class_to_keep)
120 for cl in dist:
121 if cl not in rem:
122 img[pred == cl] = 0
123 return img
124 elif callable(class_to_keep):
125 local_mask = class_to_keep
126 else:
127 raise TypeError("class_to_keep should be an int, a list or a function not {0}".format(
128 type(class_to_keep)))
130 if fLOG:
131 fLOG('[video_map_images_people] loads deep learning model')
132 model = DLImageSegmentation(fLOG=fLOG, **kwargs)
133 iter = video_enumerate_frames(video_or_file, fps=fps, with_times=with_times,
134 logger=logger, dtype=dtype, clean=False)
135 if fLOG is not None:
136 if fps is not None:
137 every = max(fps, 1)
138 unit = 's'
139 else:
140 every = 20
141 unit = 'i'
143 if fLOG:
144 fLOG('[video_map_images_people] starts extracting characters')
145 seq = []
146 for i, img in enumerate(iter):
147 if not logger and fLOG is not None and i % every == 0:
148 fLOG('[video_map_images_people] process %d%s images' % (i, unit))
149 if resize is not None and isinstance(resize[0], str):
150 if len(img.shape) == 2:
151 resize = DLImageSegmentation._new_size(img.shape, resize)
152 else:
153 resize = DLImageSegmentation._new_size(img.shape[:2], resize)
154 img, pred = model.predict(img, resize=resize)
155 img2 = local_mask(img, pred)
156 seq.append(img2)
157 if fLOG:
158 fLOG('[video_map_images_people] done.')
160 return ImageSequenceClip(seq, fps=fps)
163def video_map_images_detect(video_or_file, fps=None, with_times=False, logger=None, dtype=None,
164 scaleFactor=1.3, minNeighbors=4, minSize=(30, 30),
165 action='blur', color=(255, 255, 0), haar=None, fLOG=None):
166 """
167 Blurs people faces.
168 Uses function `detectmultiscale <https://docs.opencv.org/2.4/modules/objdetect/
169 doc/cascade_classification.html#cascadeclassifier-detectmultiscale>`_.
170 Relies on :epkg:`opencv`.
171 Floute les visages.
173 @param video_or_file string or :epkg:`VideoClip`
174 @param fps see @see fn video_enumerate_frames,
175 faces are detected in each frame returned by
176 @see fn video_enumerate_frames
177 @param with_times see @see fn video_enumerate_frames
178 @param logger see @see fn video_enumerate_frames
179 @param dtype see @see fn video_enumerate_frames
180 @param fLOG logging function
181 @param scaleFactor see `detectmultiscale <https://docs.opencv.org/2.4/modules/objdetect/doc/
182 cascade_classification.html#cascadeclassifier-detectmultiscale>`_
183 @param minNeighbors see `detectmultiscale <https://docs.opencv.org/2.4/modules/objdetect/doc/
184 cascade_classification.html#cascadeclassifier-detectmultiscale>`_
185 @param minSize see `detectmultiscale <https://docs.opencv.org/2.4/modules/objdetect/doc/
186 cascade_classification.html#cascadeclassifier-detectmultiscale>`_
187 @param haar shape classifier to load, face by default, see below
188 @param action to blur, to put a rectangle around the detected zone... see below
189 @param color rectangle color if *action* is ``'rect'``
190 @return :epkg:`VideoClip`
192 Only ``haarcascade_frontalface_alt.xml`` is provided but you can
193 get more at `haarcascades <https://github.com/opencv/opencv/blob/master/data/haarcascades/>`_.
195 Parameter *action* can be:
197 * ``'blur'``: to blur faces (or detector zones)
198 * ``'rect'``: to draw a rectangle around faces (or detector zones)
200 .. exref::
201 :title: Faces in a yellow box in a video
203 The following example uses :epkg:`opencv` to detect faces
204 on each image of a video and put a yellow box around each of them.
206 ::
208 from code_beatrix.art.videodl import video_map_images
209 from code_beatrix.art.video import video_save, video_extract_video
211 vide = video_extract_video(vid, 0, 5 if __name__ == "__main__" else 1)
212 vid2 = video_map_images(
213 vide, fps=10, name='detect', action='rect',
214 logger='bar', fLOG=fLOG)
215 exp = os.path.join(temp, "people.mp4")
216 video_save(vid2, exp, fps=10)
218 The following video is taken from
219 `Charlie Chaplin's movies <source: https://www.youtube.com/watch?v=n_1apYo6-Ow>`_.
221 .. video:: face.mp4
222 """
223 from cv2 import CascadeClassifier, CASCADE_SCALE_IMAGE # pylint: disable=E0401
224 from .video_drawing import blur, rectangle
226 def fl_blur(gf, t, rects):
227 im = gf(t).copy()
228 ti = min(int(t * fps), len(rects) - 1)
229 rects = all_rects[ti]
230 for rect in rects:
231 x1, y1, dx, dy = rect
232 blur(im, (x1, y1), (x1 + dx, y1 + dy))
233 return im
235 def fl_rect(gf, t, rects):
236 im = gf(t).copy()
237 ti = min(int(t * fps), len(rects) - 1)
238 rects = all_rects[ti]
239 for rect in rects:
240 x1, y1, dx, dy = rect
241 rectangle(im, (x1, y1), (x1 + dx, y1 + dy), color)
242 return im
244 fcts = dict(blur=fl_blur, rect=fl_rect)
246 if action not in fcts:
247 raise ValueError("action='{0}' should be in {1}".format(
248 action, list(sorted(fcts.keys()))))
250 if fLOG:
251 fLOG('[video_map_images_blur] detect faces')
253 if haar is None:
254 this = os.path.abspath(os.path.dirname(__file__))
255 cascade_fn = os.path.join(
256 this, 'data', 'haarcascade_frontalface_alt.xml')
257 elif not os.path.exists(haar):
258 raise FileNotFoundError(haar)
259 else:
260 cascade_fn = haar
262 cascade = CascadeClassifier(cascade_fn)
264 iter = video_enumerate_frames(video_or_file, fps=fps, with_times=with_times,
265 logger=logger, dtype=dtype, clean=False)
267 if fLOG:
268 fLOG("[video_map_images_people] starts detecting and burring faces with: {0}".format(
269 cascade_fn))
270 if fps is not None:
271 every = max(fps, 1)
272 unit = 's'
273 else:
274 every = 20
275 unit = 'i'
277 all_rects = []
278 for i, img in enumerate(iter):
279 if not logger and fLOG is not None and i % every == 0:
280 fLOG('[video_map_images_face] process %d%s images' % (i, unit))
282 try:
283 rects = cascade.detectMultiScale(img, scaleFactor=1.3,
284 minNeighbors=minNeighbors, minSize=minSize,
285 flags=CASCADE_SCALE_IMAGE)
286 except Exception as e:
287 if fLOG:
288 fLOG('Unable to retrieve any shape due to ', e)
289 rects = []
290 all_rects.append(rects)
292 if fLOG:
293 non = sum(map(len, (filter(lambda x: len(x) > 0, all_rects))))
294 fLOG('[video_map_images_blur] creates video nb image: {1}, nb faces: {0}'.format(
295 non, len(all_rects)))
297 with VideoContext(video_or_file) as video:
298 return video.video.fl(lambda im, t: fcts[action](im, t, all_rects), keep_duration=True)