Coverage for src/code_beatrix/art/videodl.py: 64%

110 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-04-29 13:45 +0200

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Fonctions proposant de traiter des vidéos 

5avec des traitements compliqués type 

6:epkg:`deep learning`. 

7""" 

8import os 

9from moviepy.video.io.ImageSequenceClip import ImageSequenceClip 

10from ..ai import DLImageSegmentation 

11from .video import video_enumerate_frames 

12from .moviepy_context import VideoContext 

13 

14 

15def video_map_images(video_or_file, name, fLOG=None, **kwargs): 

16 """ 

17 Applies one complex process to a video such as 

18 extracting characters from videos and 

19 removing the backaground. It is done image by image. 

20 Applique un traitement compliqué sur une séquence 

21 d'images telle que la séparation des personnages et du fond. 

22 

23 @param video_or_file string or :epkg:`VideoClip` 

24 @param name name of the processing to do, 

25 see the list below 

26 @param fLOG logging function 

27 @param kwargs additional parameters 

28 @return :epkg:`VideoClip` 

29 

30 List of available treatments: 

31 

32 * ``'people'``: extracts characters from a movie. 

33 The movie is composed with an image and a 

34 `mask <https://zulko.github.io/moviepy/ref/AudioClip.html?highlight=mask#moviepy.audio.AudioClip.AudioClip.set_ismask>`_. 

35 Parameters: see @fn video_map_images_people. 

36 * ``'detect'`` : blurs or put a rectangle around faces, uses :epkg:`opencv` 

37 

38 .. warning:: A couple of errors timeout, out of memory... 

39 The following processes might be quite time consuming 

40 or memory consuming. If it is the case, you should think 

41 of reducing the resolution, the number of frames per seconds 

42 (*fps*). You can also split the video and process each piece 

43 independently and finally concatenate them. 

44 """ 

45 allowed = {'people'} 

46 if name == 'people': 

47 return video_map_images_people(video_or_file, fLOG=fLOG, **kwargs) 

48 elif name == "detect": 

49 return video_map_images_detect(video_or_file, fLOG=fLOG, **kwargs) 

50 else: 

51 raise ValueError("Unknown process '{}', should be among: {}".format( 

52 name, ','.join(allowed))) 

53 

54 

55def video_map_images_people(video_or_file, resize=('max2', 400), fps=None, 

56 with_times=False, logger=None, dtype=None, 

57 class_to_keep=15, fLOG=None, **kwargs): 

58 """ 

59 Extracts characters from a movie. 

60 The movie is composed with an image and a 

61 `mask <https://zulko.github.io/moviepy/ref/AudioClip.html?highlight=mask#moviepy.audio.AudioClip.AudioClip.set_ismask>`_. 

62 Extrait les personnages d'un film, le résultat est 

63 composé d'une image et d'un masque transparent 

64 qui laissera apparaître l'image d'en dessous si cette 

65 vidéo est aposée sur une autre. 

66 

67 @param video_or_file string or :epkg:`VideoClip` 

68 @param resize see :meth:`predict <code_beatrix.ai.image_segmentation.DLImageSegmentation.predict>` 

69 @param fps see @see fn video_enumerate_frames 

70 @param with_times see @see fn video_enumerate_frames 

71 @param logger see @see fn video_enumerate_frames 

72 @param dtype see @see fn video_enumerate_frames 

73 @param class_to_keep class to keep from the image, it can 

74 a number (15 for the background, a list of classes, 

75 a function which takes an image and the prediction 

76 and returns an image) 

77 @param fLOG logging function 

78 @param kwargs see @see cl DLImageSegmentation 

79 @return :epkg:`VideoClip` 

80 

81 .. warning:: A couple of errors timeout, out of memory... 

82 The following processes might be quite time consuming 

83 or memory consuming. If it is the case, you should think 

84 of reducing the resolution, the number of frames per seconds 

85 (*fps*). You can also split the video and process each piece 

86 independently and finally concatenate them. 

87 

88 .. exref:: 

89 :title: Extract characters from a video. 

90 

91 The following example shows how to extract a movie with 

92 people and without the background. It works better 

93 if the contrast between the characters and the background is 

94 high. 

95 

96 :: 

97 

98 from code_beatrix.art.video import video_extract_video, video_save 

99 from code_beatrix.art.videodl import video_map_images 

100 

101 vide = video_extract_video("something.mp4", 0, 5) 

102 vid2 = video_map_images(vide, fps=10, name="people", logger='bar') 

103 video_save(vid2, "people.mp4") 

104 

105 The function returns something like the the following. 

106 The character is wearing black and the background is quite 

107 dark too. That explains that the kind of large halo 

108 around the character. 

109 

110 .. video:: videodl.mp4 

111 """ 

112 if isinstance(class_to_keep, int): 

113 def local_mask(img, pred): 

114 img[pred != class_to_keep] = 0 

115 return img 

116 elif isinstance(class_to_keep, (set, tuple, list)): 

117 def local_mask(img, pred): 

118 dist = set(pred.ravel()) 

119 rem = set(class_to_keep) 

120 for cl in dist: 

121 if cl not in rem: 

122 img[pred == cl] = 0 

123 return img 

124 elif callable(class_to_keep): 

125 local_mask = class_to_keep 

126 else: 

127 raise TypeError("class_to_keep should be an int, a list or a function not {0}".format( 

128 type(class_to_keep))) 

129 

130 if fLOG: 

131 fLOG('[video_map_images_people] loads deep learning model') 

132 model = DLImageSegmentation(fLOG=fLOG, **kwargs) 

133 iter = video_enumerate_frames(video_or_file, fps=fps, with_times=with_times, 

134 logger=logger, dtype=dtype, clean=False) 

135 if fLOG is not None: 

136 if fps is not None: 

137 every = max(fps, 1) 

138 unit = 's' 

139 else: 

140 every = 20 

141 unit = 'i' 

142 

143 if fLOG: 

144 fLOG('[video_map_images_people] starts extracting characters') 

145 seq = [] 

146 for i, img in enumerate(iter): 

147 if not logger and fLOG is not None and i % every == 0: 

148 fLOG('[video_map_images_people] process %d%s images' % (i, unit)) 

149 if resize is not None and isinstance(resize[0], str): 

150 if len(img.shape) == 2: 

151 resize = DLImageSegmentation._new_size(img.shape, resize) 

152 else: 

153 resize = DLImageSegmentation._new_size(img.shape[:2], resize) 

154 img, pred = model.predict(img, resize=resize) 

155 img2 = local_mask(img, pred) 

156 seq.append(img2) 

157 if fLOG: 

158 fLOG('[video_map_images_people] done.') 

159 

160 return ImageSequenceClip(seq, fps=fps) 

161 

162 

163def video_map_images_detect(video_or_file, fps=None, with_times=False, logger=None, dtype=None, 

164 scaleFactor=1.3, minNeighbors=4, minSize=(30, 30), 

165 action='blur', color=(255, 255, 0), haar=None, fLOG=None): 

166 """ 

167 Blurs people faces. 

168 Uses function `detectmultiscale <https://docs.opencv.org/2.4/modules/objdetect/ 

169 doc/cascade_classification.html#cascadeclassifier-detectmultiscale>`_. 

170 Relies on :epkg:`opencv`. 

171 Floute les visages. 

172 

173 @param video_or_file string or :epkg:`VideoClip` 

174 @param fps see @see fn video_enumerate_frames, 

175 faces are detected in each frame returned by 

176 @see fn video_enumerate_frames 

177 @param with_times see @see fn video_enumerate_frames 

178 @param logger see @see fn video_enumerate_frames 

179 @param dtype see @see fn video_enumerate_frames 

180 @param fLOG logging function 

181 @param scaleFactor see `detectmultiscale <https://docs.opencv.org/2.4/modules/objdetect/doc/ 

182 cascade_classification.html#cascadeclassifier-detectmultiscale>`_ 

183 @param minNeighbors see `detectmultiscale <https://docs.opencv.org/2.4/modules/objdetect/doc/ 

184 cascade_classification.html#cascadeclassifier-detectmultiscale>`_ 

185 @param minSize see `detectmultiscale <https://docs.opencv.org/2.4/modules/objdetect/doc/ 

186 cascade_classification.html#cascadeclassifier-detectmultiscale>`_ 

187 @param haar shape classifier to load, face by default, see below 

188 @param action to blur, to put a rectangle around the detected zone... see below 

189 @param color rectangle color if *action* is ``'rect'`` 

190 @return :epkg:`VideoClip` 

191 

192 Only ``haarcascade_frontalface_alt.xml`` is provided but you can 

193 get more at `haarcascades <https://github.com/opencv/opencv/blob/master/data/haarcascades/>`_. 

194 

195 Parameter *action* can be: 

196 

197 * ``'blur'``: to blur faces (or detector zones) 

198 * ``'rect'``: to draw a rectangle around faces (or detector zones) 

199 

200 .. exref:: 

201 :title: Faces in a yellow box in a video 

202 

203 The following example uses :epkg:`opencv` to detect faces 

204 on each image of a video and put a yellow box around each of them. 

205 

206 :: 

207 

208 from code_beatrix.art.videodl import video_map_images 

209 from code_beatrix.art.video import video_save, video_extract_video 

210 

211 vide = video_extract_video(vid, 0, 5 if __name__ == "__main__" else 1) 

212 vid2 = video_map_images( 

213 vide, fps=10, name='detect', action='rect', 

214 logger='bar', fLOG=fLOG) 

215 exp = os.path.join(temp, "people.mp4") 

216 video_save(vid2, exp, fps=10) 

217 

218 The following video is taken from 

219 `Charlie Chaplin's movies <source: https://www.youtube.com/watch?v=n_1apYo6-Ow>`_. 

220 

221 .. video:: face.mp4 

222 """ 

223 from cv2 import CascadeClassifier, CASCADE_SCALE_IMAGE # pylint: disable=E0401 

224 from .video_drawing import blur, rectangle 

225 

226 def fl_blur(gf, t, rects): 

227 im = gf(t).copy() 

228 ti = min(int(t * fps), len(rects) - 1) 

229 rects = all_rects[ti] 

230 for rect in rects: 

231 x1, y1, dx, dy = rect 

232 blur(im, (x1, y1), (x1 + dx, y1 + dy)) 

233 return im 

234 

235 def fl_rect(gf, t, rects): 

236 im = gf(t).copy() 

237 ti = min(int(t * fps), len(rects) - 1) 

238 rects = all_rects[ti] 

239 for rect in rects: 

240 x1, y1, dx, dy = rect 

241 rectangle(im, (x1, y1), (x1 + dx, y1 + dy), color) 

242 return im 

243 

244 fcts = dict(blur=fl_blur, rect=fl_rect) 

245 

246 if action not in fcts: 

247 raise ValueError("action='{0}' should be in {1}".format( 

248 action, list(sorted(fcts.keys())))) 

249 

250 if fLOG: 

251 fLOG('[video_map_images_blur] detect faces') 

252 

253 if haar is None: 

254 this = os.path.abspath(os.path.dirname(__file__)) 

255 cascade_fn = os.path.join( 

256 this, 'data', 'haarcascade_frontalface_alt.xml') 

257 elif not os.path.exists(haar): 

258 raise FileNotFoundError(haar) 

259 else: 

260 cascade_fn = haar 

261 

262 cascade = CascadeClassifier(cascade_fn) 

263 

264 iter = video_enumerate_frames(video_or_file, fps=fps, with_times=with_times, 

265 logger=logger, dtype=dtype, clean=False) 

266 

267 if fLOG: 

268 fLOG("[video_map_images_people] starts detecting and burring faces with: {0}".format( 

269 cascade_fn)) 

270 if fps is not None: 

271 every = max(fps, 1) 

272 unit = 's' 

273 else: 

274 every = 20 

275 unit = 'i' 

276 

277 all_rects = [] 

278 for i, img in enumerate(iter): 

279 if not logger and fLOG is not None and i % every == 0: 

280 fLOG('[video_map_images_face] process %d%s images' % (i, unit)) 

281 

282 try: 

283 rects = cascade.detectMultiScale(img, scaleFactor=1.3, 

284 minNeighbors=minNeighbors, minSize=minSize, 

285 flags=CASCADE_SCALE_IMAGE) 

286 except Exception as e: 

287 if fLOG: 

288 fLOG('Unable to retrieve any shape due to ', e) 

289 rects = [] 

290 all_rects.append(rects) 

291 

292 if fLOG: 

293 non = sum(map(len, (filter(lambda x: len(x) > 0, all_rects)))) 

294 fLOG('[video_map_images_blur] creates video nb image: {1}, nb faces: {0}'.format( 

295 non, len(all_rects))) 

296 

297 with VideoContext(video_or_file) as video: 

298 return video.video.fl(lambda im, t: fcts[action](im, t, all_rects), keep_duration=True)