fileio related interfaces should keep bc (open-mmlab#2539)

* fileio related interfaces should keep bc

* fix format

* update unit tests of transforms

* update format

* lazily infer client until calling get function of fileio

Author: zhouzaida

mmcv/image/io.py

@@ -145,6 +145,7 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
            flag: str = 'color',
            channel_order: str = 'bgr',
            backend: Optional[str] = None,
+           file_client_args: Optional[dict] = None,
            *,
            backend_args: Optional[dict] = None) -> np.ndarray:
     """Read an image.
@@ -166,12 +167,18 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
             `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`.
             If backend is None, the global imread_backend specified by
             ``mmcv.use_backend()`` will be used. Default: None.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmengine.fileio.FileClient` for details.
+            Default: None. It will be deprecated in future. Please use
+            ``backend_args`` instead.
+            Deprecated in version 2.0.0rc4.
         backend_args (dict, optional): Instantiates the corresponding file
             backend. It may contain `backend` key to specify the file
             backend. If it contains, the file backend corresponding to this
             value will be used and initialized with the remaining values,
             otherwise the corresponding file backend will be selected
             based on the prefix of the file path. Defaults to None.
+            New in version 2.0.0rc4.
 
     Returns:
         ndarray: Loaded image array.
@@ -195,13 +202,27 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
         >>> img = mmcv.imread(http_img_path, backend_args={
         ...     'backend': 'http'})
     """
+    if file_client_args is not None:
+        warnings.warn(
+            '"file_client_args" will be deprecated in future. '
+            'Please use "backend_args" instead', DeprecationWarning)
+        if backend_args is not None:
+            raise ValueError(
+                '"file_client_args" and "backend_args" cannot be set at the '
+                'same time.')
+
     if isinstance(img_or_path, Path):
         img_or_path = str(img_or_path)
 
     if isinstance(img_or_path, np.ndarray):
         return img_or_path
     elif is_str(img_or_path):
-        img_bytes = fileio.get(img_or_path, backend_args=backend_args)
+        if file_client_args is not None:
+            file_client = fileio.FileClient.infer_client(
+                file_client_args, img_or_path)
+            img_bytes = file_client.get(img_or_path)
+        else:
+            img_bytes = fileio.get(img_or_path, backend_args=backend_args)
         return imfrombytes(img_bytes, flag, channel_order, backend)
     else:
         raise TypeError('"img" must be a numpy array or a str or '
@@ -271,6 +292,7 @@ def imwrite(img: np.ndarray,
             file_path: str,
             params: Optional[list] = None,
             auto_mkdir: Optional[bool] = None,
+            file_client_args: Optional[dict] = None,
             *,
             backend_args: Optional[dict] = None) -> bool:
     """Write image to file.
@@ -285,12 +307,18 @@ def imwrite(img: np.ndarray,
         params (None or list): Same as opencv :func:`imwrite` interface.
         auto_mkdir (bool): If the parent folder of `file_path` does not exist,
             whether to create it automatically. It will be deprecated.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmengine.fileio.FileClient` for details.
+            Default: None. It will be deprecated in future. Please use
+            ``backend_args`` instead.
+            Deprecated in version 2.0.0rc4.
         backend_args (dict, optional): Instantiates the corresponding file
             backend. It may contain `backend` key to specify the file
             backend. If it contains, the file backend corresponding to this
             value will be used and initialized with the remaining values,
             otherwise the corresponding file backend will be selected
             based on the prefix of the file path. Defaults to None.
+            New in version 2.0.0rc4.
 
     Returns:
         bool: Successful or not.
@@ -304,6 +332,15 @@ def imwrite(img: np.ndarray,
         >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', backend_args={
         ...     'backend': 'petrel'})
     """
+    if file_client_args is not None:
+        warnings.warn(
+            '"file_client_args" will be deprecated in future. '
+            'Please use "backend_args" instead', DeprecationWarning)
+        if backend_args is not None:
+            raise ValueError(
+                '"file_client_args" and "backend_args" cannot be set at the '
+                'same time.')
+
     assert is_filepath(file_path)
     file_path = str(file_path)
     if auto_mkdir is not None:
@@ -317,6 +354,11 @@ def imwrite(img: np.ndarray,
     # format is '.jpg'.
     flag, img_buff = cv2.imencode(img_ext, img, params)
 
-    fileio.put(img_buff.tobytes(), file_path, backend_args=backend_args)
+    if file_client_args is not None:
+        file_client = fileio.FileClient.infer_client(file_client_args,
+                                                     file_path)
+        file_client.put(img_buff.tobytes(), file_path)
+    else:
+        fileio.put(img_buff.tobytes(), file_path, backend_args=backend_args)
 
     return flag

mmcv/transforms/loading.py

@@ -1,4 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
+import warnings
 from typing import Optional
 
 import mmengine.fileio as fileio
@@ -33,6 +34,11 @@ class LoadImageFromFile(BaseTransform):
             argument for :func:`mmcv.imfrombytes`.
             See :func:`mmcv.imfrombytes` for details.
             Defaults to 'cv2'.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmengine.fileio.FileClient` for details.
+            Defaults to None. It will be deprecated in future. Please use
+            ``backend_args`` instead.
+            Deprecated in version 2.0.0rc4.
         ignore_empty (bool): Whether to allow loading empty image or file path
             not existent. Defaults to False.
         backend_args (dict, optional): Instantiates the corresponding file
@@ -41,20 +47,36 @@ class LoadImageFromFile(BaseTransform):
             value will be used and initialized with the remaining values,
             otherwise the corresponding file backend will be selected
             based on the prefix of the file path. Defaults to None.
+            New in version 2.0.0rc4.
     """
 
     def __init__(self,
                  to_float32: bool = False,
                  color_type: str = 'color',
                  imdecode_backend: str = 'cv2',
+                 file_client_args: Optional[dict] = None,
                  ignore_empty: bool = False,
                  *,
                  backend_args: Optional[dict] = None) -> None:
         self.ignore_empty = ignore_empty
         self.to_float32 = to_float32
         self.color_type = color_type
         self.imdecode_backend = imdecode_backend
-        self.backend_args = backend_args
+
+        self.file_client_args: Optional[dict] = None
+        self.backend_args: Optional[dict] = None
+        if file_client_args is not None:
+            warnings.warn(
+                '"file_client_args" will be deprecated in future. '
+                'Please use "backend_args" instead', DeprecationWarning)
+            if backend_args is not None:
+                raise ValueError(
+                    '"file_client_args" and "backend_args" cannot be set '
+                    'at the same time.')
+
+            self.file_client_args = file_client_args.copy()
+        if backend_args is not None:
+            self.backend_args = backend_args.copy()
 
     def transform(self, results: dict) -> Optional[dict]:
         """Functions to load image.
@@ -69,7 +91,13 @@ def transform(self, results: dict) -> Optional[dict]:
 
         filename = results['img_path']
         try:
-            img_bytes = fileio.get(filename, backend_args=self.backend_args)
+            if self.file_client_args is not None:
+                file_client = fileio.FileClient.infer_client(
+                    self.file_client_args, filename)
+                img_bytes = file_client.get(filename)
+            else:
+                img_bytes = fileio.get(
+                    filename, backend_args=self.backend_args)
             img = mmcv.imfrombytes(
                 img_bytes, flag=self.color_type, backend=self.imdecode_backend)
         except Exception as e:
@@ -90,12 +118,12 @@ def __repr__(self):
                     f'ignore_empty={self.ignore_empty}, '
                     f'to_float32={self.to_float32}, '
                     f"color_type='{self.color_type}', "
-                    f"imdecode_backend='{self.imdecode_backend}'")
+                    f"imdecode_backend='{self.imdecode_backend}', ")
 
-        if self.backend_args is not None:
-            repr_str += f', backend_args={self.backend_args})'
+        if self.file_client_args is not None:
+            repr_str += f'file_client_args={self.file_client_args})'
         else:
-            repr_str += ')'
+            repr_str += f'backend_args={self.backend_args})'
 
         return repr_str
 
@@ -177,12 +205,18 @@ class LoadAnnotations(BaseTransform):
             argument for :func:`mmcv.imfrombytes`.
             See :func:`mmcv.imfrombytes` for details.
             Defaults to 'cv2'.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmengine.fileio.FileClient` for details.
+            Defaults to None. It will be deprecated in future. Please use
+            ``backend_args`` instead.
+            Deprecated in version 2.0.0rc4.
         backend_args (dict, optional): Instantiates the corresponding file
             backend. It may contain `backend` key to specify the file
             backend. If it contains, the file backend corresponding to this
             value will be used and initialized with the remaining values,
             otherwise the corresponding file backend will be selected
             based on the prefix of the file path. Defaults to None.
+            New in version 2.0.0rc4.
     """
 
     def __init__(
@@ -192,6 +226,7 @@ def __init__(
         with_seg: bool = False,
         with_keypoints: bool = False,
         imdecode_backend: str = 'cv2',
+        file_client_args: Optional[dict] = None,
         *,
         backend_args: Optional[dict] = None,
     ) -> None:
@@ -201,7 +236,21 @@ def __init__(
         self.with_seg = with_seg
         self.with_keypoints = with_keypoints
         self.imdecode_backend = imdecode_backend
-        self.backend_args = backend_args
+
+        self.file_client_args: Optional[dict] = None
+        self.backend_args: Optional[dict] = None
+        if file_client_args is not None:
+            warnings.warn(
+                '"file_client_args" will be deprecated in future. '
+                'Please use "backend_args" instead', DeprecationWarning)
+            if backend_args is not None:
+                raise ValueError(
+                    '"file_client_args" and "backend_args" cannot be set '
+                    'at the same time.')
+
+            self.file_client_args = file_client_args.copy()
+        if backend_args is not None:
+            self.backend_args = backend_args.copy()
 
     def _load_bboxes(self, results: dict) -> None:
         """Private function to load bounding box annotations.
@@ -245,9 +294,14 @@ def _load_seg_map(self, results: dict) -> None:
         Returns:
             dict: The dict contains loaded semantic segmentation annotations.
         """
+        if self.file_client_args is not None:
+            file_client = fileio.FileClient.infer_client(
+                self.file_client_args, results['seg_map_path'])
+            img_bytes = file_client.get(results['seg_map_path'])
+        else:
+            img_bytes = fileio.get(
+                results['seg_map_path'], backend_args=self.backend_args)
 
-        img_bytes = fileio.get(
-            results['seg_map_path'], backend_args=self.backend_args)
         results['gt_seg_map'] = mmcv.imfrombytes(
             img_bytes, flag='unchanged',
             backend=self.imdecode_backend).squeeze()
@@ -296,11 +350,11 @@ def __repr__(self) -> str:
         repr_str += f'with_label={self.with_label}, '
         repr_str += f'with_seg={self.with_seg}, '
         repr_str += f'with_keypoints={self.with_keypoints}, '
-        repr_str += f"imdecode_backend='{self.imdecode_backend}'"
+        repr_str += f"imdecode_backend='{self.imdecode_backend}', "
 
-        if self.backend_args is not None:
-            repr_str += f', backend_args={self.backend_args})'
+        if self.file_client_args is not None:
+            repr_str += f'file_client_args={self.file_client_args})'
         else:
-            repr_str += ')'
+            repr_str += f'backend_args={self.backend_args})'
 
         return repr_str

tests/test_image/test_io.py

@@ -60,6 +60,15 @@ def test_imread(self):
         # backend cv2
         mmcv.use_backend('cv2')
 
+        # file_client_args and backend_args can not be both set
+        with pytest.raises(
+                ValueError,
+                match='"file_client_args" and "backend_args" cannot be set'):
+            mmcv.imread(
+                self.img_path,
+                file_client_args={'backend': 'disk'},
+                backend_args={'backend': 'disk'})
+
         # HardDiskBackend
         img_cv2_color_bgr = mmcv.imread(self.img_path)
         assert img_cv2_color_bgr.shape == (300, 400, 3)
@@ -95,6 +104,16 @@ def test_imread(self):
                 PetrelBackend, 'get',
                 return_value=img_cv2_color_bgr) as mock_method:
             img_cv2_color_bgr_petrel = mmcv.imread(self.s3_path, backend='cv2')
+            img_cv2_color_bgr_petrel_with_args = mmcv.imread(
+                self.s3_path,
+                backend='cv2',
+                file_client_args={'backend': 'petrel'})
+            mock_method.assert_called()
+            assert_array_equal(img_cv2_color_bgr_petrel,
+                               img_cv2_color_bgr_petrel_with_args)
+
+            mock_method.reset_mock()
+
             img_cv2_color_bgr_petrel_with_args = mmcv.imread(
                 self.s3_path,
                 backend='cv2',
@@ -109,6 +128,16 @@ def test_imread(self):
                 HTTPBackend, 'get',
                 return_value=img_cv2_color_bgr) as mock_method:
             img_cv2_color_bgr_http = mmcv.imread(self.http_path, backend='cv2')
+            img_cv2_color_bgr_http_with_args = mmcv.imread(
+                self.http_path,
+                backend='cv2',
+                file_client_args={'backend': 'http'})
+            mock_method.assert_called()
+            assert_array_equal(img_cv2_color_bgr_http,
+                               img_cv2_color_bgr_http_with_args)
+
+            mock_method.reset_mock()
+
             img_cv2_color_bgr_http_with_args = mmcv.imread(
                 self.http_path,
                 backend='cv2',
@@ -358,6 +387,16 @@ def test_imwrite(self):
         img = mmcv.imread(self.img_path)
         out_file = osp.join(tempfile.gettempdir(), 'mmcv_test.jpg')
 
+        # file_client_args and backend_args can not be both set
+        with pytest.raises(
+                ValueError,
+                match='"file_client_args" and "backend_args" cannot be set'):
+            mmcv.imwrite(
+                img,
+                out_file,
+                file_client_args={'backend': 'disk'},
+                backend_args={'backend': 'disk'})
+
         mmcv.imwrite(img, out_file)
         rewrite_img = mmcv.imread(out_file)
         os.remove(out_file)
@@ -367,7 +406,13 @@ def test_imwrite(self):
         with patch.object(
                 PetrelBackend, 'put', return_value=None) as mock_method:
             ret = mmcv.imwrite(img, self.s3_path)
+            ret_with_args = mmcv.imwrite(
+                img, self.s3_path, file_client_args={'backend': 'petrel'})
             assert ret
+            assert ret_with_args
+            mock_method.assert_called()
+
+            mock_method.reset_mock()
 
             ret_with_args = mmcv.imwrite(
                 img, self.s3_path, backend_args={'backend': 'petrel'})