diff --git a/exact/exact/images/models.py b/exact/exact/images/models.py index de09cfcd..20620992 100644 --- a/exact/exact/images/models.py +++ b/exact/exact/images/models.py @@ -27,7 +27,8 @@ from openslide import OpenSlide, open_slide from czifile import czi2tif from util.cellvizio import ReadableCellVizioMKTDataset # just until data access is pip installable - +# for video handling +from util.video_handler import ReadableVideoDataset from PIL import Image as PIL_Image from datetime import datetime @@ -193,69 +194,20 @@ def save_file(self, path:Path): self.filename = path.name # Videos elif Path(path).suffix.lower() in [".avi", ".mp4"]: - dtype_to_format = { - 'uint8': 'uchar', - 'int8': 'char', - 'uint16': 'ushort', - 'int16': 'short', - 'uint32': 'uint', - 'int32': 'int', - 'float32': 'float', - 'float64': 'double', - 'complex64': 'complex', - 'complex128': 'dpcomplex', - } - - folder_path = Path(self.image_set.root_path()) / path.stem - os.makedirs(str(folder_path), exist_ok =True) - os.chmod(str(folder_path), 0o777) - self.save() # initially save - - cap = cv2.VideoCapture(str(Path(path))) - frame_id = 0 - while cap.isOpened(): - ret, frame = cap.read() - if not ret: - # if video has just one frame copy file to top layer - if frame_id == 1: - copy_path = Path(path).with_suffix('.tiff') - shutil.copyfile(str(target_file), str(copy_path)) - self.filename = copy_path.name - break - - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - height, width, bands = frame.shape - linear = frame.reshape(width * height * bands) - - vi = pyvips.Image.new_from_memory(np.ascontiguousarray(linear.data), width, height, bands, - dtype_to_format[str(frame.dtype)]) - if dtype_to_format[str(frame.dtype)] not in ["uchar"]: - vi = vi.scaleimage() - - height, width, channels = vi.height, vi.width, vi.bands - self.channels = channels - - target_file = folder_path / "{}_{}_{}".format(1, frame_id + 1, path.name) #z-axis frame image - vi.tiffsave(str(target_file), tile=True, compression='lzw', bigtiff=True, pyramid=True, tile_width=256, tile_height=256) - - # save first frame as default file for thumbnail etc. - if frame_id == 0: - self.filename = target_file.name - - # save FrameDescription object for each frame + reader = ReadableVideoDataset(str(path)) + self.frames = reader.nFrames + self.width, self.height = reader.dimensions + self.channels = 3 + self.filename = path.name + self.save() + for frame_id in range(reader.nFrames): FrameDescription.objects.create( Image=self, frame_id=frame_id, - file_path=target_file, + file_path=self.filename, frame_type=FrameType.TIMESERIES, - description='%.2f s (%d)' % ((float(frame_id-1)/cap.get(cv2.CAP_PROP_FPS)), frame_id) + description=reader.frame_descriptors[frame_id] ) - - - frame_id += 1 - - self.frames = frame_id - # check if file is philips iSyntax elif Path(path).suffix.lower().endswith(".isyntax"): diff --git a/exact/exact/images/views.py b/exact/exact/images/views.py index fe1fec71..6116351f 100644 --- a/exact/exact/images/views.py +++ b/exact/exact/images/views.py @@ -303,7 +303,7 @@ def upload_image(request, imageset_id): image_set=imageset).first() print('Image:',image) if image is None: - + os.makedirs(os.path.dirname(filename), exist_ok=True) with open(filename, 'wb') as out: for chunk in f.chunks(): out.write(chunk) diff --git a/exact/util/slide_server.py b/exact/util/slide_server.py index 25e82e88..3a13556e 100644 --- a/exact/util/slide_server.py +++ b/exact/util/slide_server.py @@ -21,6 +21,7 @@ from PIL import Image import numpy as np from util.cellvizio import ReadableCellVizioMKTDataset +from util.video_handler import ReadableVideoDataset from openslide import OpenSlideError import tifffile from util.tiffzstack import OMETiffSlide, OMETiffZStack @@ -454,12 +455,6 @@ class JPEGEXIFFileType(FileType): extensions = ['jpg','jpeg'] handler = ImageSlideWrapper -class MP4MovieFileType(FileType): - extensions = ['mp4'] - magic_number = b'\x66\x74\x79\x70' - magic_number_offset = 4 - handler = MovieWrapperCV2 - class PNGFileType(FileType): magic_number = b'\x89\x50\x4e\x47' extensions = ['png'] @@ -491,9 +486,19 @@ class MKTFileType(FileType): extensions = 'mkt' handler = ReadableCellVizioMKTDataset +class VideoMP4FileType(FileType): + magic_number = b'\x66\x74\x79\x70' + extensions = ['mp4'] + magic_number_offset = 4 + handler = ReadableVideoDataset +class VideoAVIFileType(FileType): + magic_number = b'\x52\x49\x46\x46' # RIFF + magic_number_offset = 0 + extensions = ['avi'] + handler = ReadableVideoDataset -SupportedFileTypes = [MKTFileType, MP4MovieFileType, DicomFileType, MiraxFileType, PhilipsISyntaxFileType, PNGFileType, JPEGEXIFFileType, JPEGJFIFFileType, OlympusVSIFileType, NormalTiffFileType, BigTiffFileType, ZeissCZIFile] +SupportedFileTypes = [MKTFileType, VideoMP4FileType, VideoAVIFileType, DicomFileType, MiraxFileType, PhilipsISyntaxFileType, PNGFileType, JPEGEXIFFileType, JPEGJFIFFileType, OlympusVSIFileType, NormalTiffFileType, BigTiffFileType, ZeissCZIFile] diff --git a/exact/util/video_handler.py b/exact/util/video_handler.py new file mode 100644 index 00000000..a6d5157e --- /dev/null +++ b/exact/util/video_handler.py @@ -0,0 +1,235 @@ +""" +Scripts for MP4 files's support + +""" +import threading +from collections import OrderedDict + +import openslide +from openslide import OpenSlideError +import numpy as np +import cv2 +from PIL import Image +try : + from util.enums import FrameType +except ImportError: + from enums import FrameType + + +class ReadableVideoDataset(openslide.ImageSlide): + def __init__(self, filename, cache_size=32, max_cache_bytes=None): + self.slide_path = filename + + self._cap = None + self._cap_lock = threading.RLock() + self._frame_cache = OrderedDict() + self._cache_size = cache_size + + self._max_cache_bytes = max_cache_bytes # optional based on memory + self._cache_bytes = 0 + + cap = cv2.VideoCapture(filename) + if not cap.isOpened(): + raise OpenSlideError(f"Could not open video file: {filename}") + # Get video properties + self._width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + self._height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + self.numberOfLayers = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # total number of frames + self.fps = cap.get(cv2.CAP_PROP_FPS) + + cap.release() + + self._dimensions = (self._width, self._height) + + def __reduce__(self): + return (self.__class__, (self.slide_path,)) + + def close(self): + with self._cap_lock: + if self._cap is not None: + self._cap.release() + self._cap = None + + self._frame_cache.clear() + self._cache_bytes = 0 + + def __del__(self): + self.close() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + @property + def properties(self): + return { + openslide.PROPERTY_NAME_BACKGROUND_COLOR: '000000', + openslide.PROPERTY_NAME_MPP_X: 0, + openslide.PROPERTY_NAME_MPP_Y: 0, + openslide.PROPERTY_NAME_OBJECTIVE_POWER: 1, + openslide.PROPERTY_NAME_VENDOR: 'MP4' + } + + + @property + def dimensions(self): + return self._dimensions + + @property + def frame_descriptors(self) -> list[str]: + """ returns a list of strings, used as descriptor for each frame + """ + return ['%.2f' % (x/self.fps) for x in range(self.nFrames)] + + @property + def frame_type(self): + return FrameType.TIMESERIES + + @property + def default_frame(self) -> list[str]: + return 0 + + @property + def nFrames(self): + return self.numberOfLayers + + @property + def level_dimensions(self): + return (self.dimensions,) + + @property + def level_count(self): + return 1 + + def _get_capture(self): + if self._cap is None or not self._cap.isOpened(): + self._cap = cv2.VideoCapture(self.slide_path) + return self._cap + + def _frame_num_bytes(self, frame_arr: np.ndarray) -> int: + """ + + Calculate the number of bytes used by a frame array. + :param frame_arr: Description + """ + try: + return int(frame_arr.nbytes) + except Exception: + return 0 + + def _evict_if_needed(self): + """ + Evict frames by LRU + """ + + # Evict frames if exceeding max frame count + while self._cache_size is not None and len(self._frame_cache) > self._cache_size: + old_idx, old_frame = self._frame_cache.popitem(last=False) + self._cache_bytes -= self._frame_num_bytes(old_frame) + # Evict based on byte size + if self._max_cache_bytes is not None: + while self._cache_bytes > self._max_cache_bytes and len(self._frame_cache) > 0: + old_idx, old_frame = self._frame_cache.popitem(last=False) + self._cache_bytes -= self._frame_num_bytes(old_frame) + + def _read_frame(self, frame_idx: int): + """ + Before reading the frame, check the cache first with thread safety. + Followed by LRU cache eviction policy. + + :param frame_idx: + """ + with self._cap_lock: + cached = self._frame_cache.get(frame_idx) + if cached is not None: + self._frame_cache.move_to_end(frame_idx) + return cached + + cap = self._get_capture() + cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx) + success, img = cap.read() + if not success: + return None + # Convert BGR to RGBA + img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGBA) + self._frame_cache[frame_idx] = img_rgb + self._frame_cache.move_to_end(frame_idx, last=True) + self._cache_bytes += self._frame_num_bytes(img_rgb) + + self._evict_if_needed() + return img_rgb + + + def get_thumbnail(self, size): + return self.read_region((0,0),0, self.dimensions).resize(size) + + def read_region(self, location, level, size, frame=0): + + """ + Reads a region from a specific video frame. + Return a PIL.Image containing the contents of the region. + Reference: https://github.com/DeepMicroscopy/Exact/commit/4d52b614fa41328bf08367d99e088c1e838fb05a + + + location: (x, y) tuple giving the top left pixel in the level 0 + reference frame. + level: the level number. + size: (width, height) tuple giving the region size. + frame: the frame index to read from the video. + + """ + if level != 0: + raise OpenSlideError("Only level 0 is supported for video files.") + + if any(s < 0 for s in size): + raise OpenSlideError(f"Size {size} must be non-negative") + + # Clamp frame index + frame = max(0, min(frame, self.numberOfLayers - 1)) + img_rgb = self._read_frame(frame) + if img_rgb is None: + # Return a transparent tile if frame read fails + return Image.new("RGBA", size, (0, 0, 0, 0)) + + x, y = location + w, h = size + + # Create the transparent canvas + tile = Image.new("RGBA", size, (0, 0, 0, 0)) + + # Calculate crop boundaries within the source image + img_h, img_w = img_rgb.shape[:2] + + # Source coordinates + src_x1 = max(0, min(x, img_w)) + src_y1 = max(0, min(y, img_h)) + src_x2 = max(0, min(x + w, img_w)) + src_y2 = max(0, min(y + h, img_h)) + + # Destination coordinates (where to paste on the tile) + dst_x1 = max(0, -x) if x < 0 else 0 + dst_y1 = max(0, -y) if y < 0 else 0 + + # Extract the crop using numpy slicing + crop_data = img_rgb[src_y1:src_y2, src_x1:src_x2] + + if crop_data.size > 0: + crop_img = Image.fromarray(crop_data) + tile.paste(crop_img, (dst_x1, dst_y1)) + + return tile + + def get_duration(self): + """Get the length of the video in seconds.""" + return self.numberOfLayers / self.fps if self.fps > 0 else 0 + + def time_to_frame(self, time_seconds: float) -> int: + """Convert time in seconds to frame index.""" + frame_idx = int(time_seconds * self.fps) + return max(0, min(frame_idx, self.numberOfFrames - 1)) + + def frame_to_time(self, frame_idx: int) -> float: + """Convert frame index to time in seconds.""" + return frame_idx / self.fps if self.fps > 0 else 0 \ No newline at end of file