revert unlabeled auto train detection back. Fixed integration tests

Author: kprokofi

otx/cli/manager/config_manager.py

@@ -3,12 +3,12 @@
 # Copyright (C) 2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 #
+import logging
 import os
 import shutil
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, List, Optional
-import logging
 
 from datumaro.components.dataset import Dataset
 from datumaro.components.dataset_base import IDataset
@@ -251,73 +251,72 @@ def _configure_train_type(self):
         If unlabeled_images presented in dataset structure and it is sufficient to start Semi-SL -> Semi-SL
         Overwise set Incremental training type.
         """
-
-        def _check_is_only_images(dir):
-            """Check if a directory contains only images."""
+        def _count_imgs_in_dir(dir, recursive=False):
+            """count number of images in directory recursively"""
             import glob
             valid_suff = ["jpg", "png", "jpeg", "gif"]
             num_valid_imgs = 0
-            for files in glob.iglob(f'{dir}/*'):
+            for files in glob.iglob(f"{dir}/**", recursive=recursive):
                 suff = files.split(".")[-1]
                 if suff.lower() in valid_suff:
                     num_valid_imgs += 1
-            return num_valid_imgs > 0
 
-        def _check_semisl_requirements(train_dir, unlabeled_dir, thershold=0.07):
+            return num_valid_imgs
+
+        def _check_semisl_requirements(unlabeled_dir):
             """Check if quantity of unlabeled images is sufficient for Semi-SL learning."""
             if unlabeled_dir is None:
-                unlabeled_folder_name = [
-                    os.path.join(train_dir, item)
-                    for item in os.listdir(train_dir)
-                    if (item.startswith("unlabeled") and os.path.isdir(os.path.join(train_dir, item)))
-                ]
-                if unlabeled_folder_name:
-                    unlabeled_valid_path = unlabeled_folder_name[0]
-                else:
-                    return False
-            else:
-                if not os.path.isdir(unlabeled_dir) or not os.listdir(unlabeled_dir):
-                    raise ValueError(
-                        "unlabeled-data-roots isn't a directory, it doesn't exist or it is empty. "
-                        "Please, check command line and directory path."
-                    )
-                unlabeled_valid_path = unlabeled_dir
+                return False
 
-            all_unlabeled_images = len(os.listdir(unlabeled_valid_path))
-            all_train_images = len(os.listdir(train_dir))
+            if not os.path.isdir(unlabeled_dir) or not os.listdir(unlabeled_dir):
+                raise ValueError(
+                    "unlabeled-data-roots isn't a directory, it doesn't exist or it is empty. "
+                    "Please, check command line and directory path."
+                )
+
+            all_unlabeled_images = _count_imgs_in_dir(unlabeled_dir, recursive=True)
             # check if number of unlabeled images is more than relative thershold
-            if all_unlabeled_images > 1 and all_unlabeled_images >= thershold * all_train_images:
-                return unlabeled_valid_path
+            if all_unlabeled_images > 1:
+                return unlabeled_dir
 
             logging.warning(
-                        "WARNING: There are none or too litle images to start Semi-SL training. "
-                        "It should be more than relative threshold (at least 7% of labeled images) "
-                        "Start Supervised training instead."
-                    )
+                "WARNING: There are none or too litle images to start Semi-SL training. "
+                "It should be more than relative threshold (at least 7% of labeled images) "
+                "Start Supervised training instead."
+            )
 
         # if user explicitly passed train type via args
         if self.args.train_type is not None:
             self.train_type = self.args.train_type
             return
 
-        path_to_train_data = Path(self.args.train_data_roots)
-        if not Path.is_dir(path_to_train_data) or not os.listdir(path_to_train_data):
+        if self.mode == "build" and self.args.train_data_roots is None:
+            # Case, when we want to build environment with tempate without dataset path
+            # Set train_type to Incremental by default
+            self.train_type = "Incremental"
+            return
+
+        if (
+            self.args.train_data_roots is None
+            or not os.path.isdir(self.args.train_data_roots)
+            or not os.listdir(self.args.train_data_roots)
+        ):
             raise ValueError(
                 "train-data-roots isn't a directory, it doesn't exist or it is empty. "
                 "Please, check command line and directory path."
             )
 
-        if _check_is_only_images(path_to_train_data):
+        if _count_imgs_in_dir(self.args.train_data_roots):
             # If train folder with images only was passed to args
             # Then we start self-supervised training
             print("[*] Selfsupervised training type detected")
             self.train_type = "Selfsupervised"
             return
 
         # if user explicitly passed unlabeled images folder
-        valid_unlabeled_path = _check_semisl_requirements(path_to_train_data, self.args.unlabeled_data_roots)
+        valid_unlabeled_path = _check_semisl_requirements(self.args.unlabeled_data_roots)
         if valid_unlabeled_path:
-            print(f"[*] Semisupervised training type detected with unalabeled data: {valid_unlabeled_path}")
+            print(f"[*] Semisupervised training type detected with unlabeled data: {valid_unlabeled_path}")
             self.train_type = "Semisupervised"
             return
 

tests/assets/common_semantic_segmentation_dataset/train_with_unlabeled/images/0001.png

@@ -50,15 +50,8 @@
 # Warmstart using data w/ 'intel', 'openvino', 'opencv' classes
 args_selfsl = {
     "--train-data-roots": "tests/assets/classification_dataset",
-    "train_params": [
-        "params",
-        "--learning_parameters.num_iters",
-        "1",
-        "--learning_parameters.batch_size",
-        "4",
-        "--algo_backend.train_type",
-        "Selfsupervised",
-    ],
+    "--train-type": "Selfsupervised",
+    "train_params": ["params", "--learning_parameters.num_iters", "1", "--learning_parameters.batch_size", "4"],
 }
 
 # Training params for resume, num_iters*2
@@ -241,8 +234,9 @@ def test_otx_train_semisl(self, template, tmp_dir_path):
         tmp_dir_path = tmp_dir_path / "multi_class_cls/test_semisl"
         args_semisl = copy.deepcopy(args)
         args_semisl["--unlabeled-data-roots"] = args["--train-data-roots"]
-        args_semisl["train_params"].extend(["--algo_backend.train_type", "Semisupervised"])
         otx_train_testing(template, tmp_dir_path, otx_dir, args_semisl)
+        template_dir = get_template_dir(template, tmp_dir_path)
+        assert os.path.exists(f"{template_dir}/semisl")
 
     @e2e_pytest_component
     @pytest.mark.skipif(MULTI_GPU_UNAVAILABLE, reason="The number of gpu is insufficient")
@@ -251,15 +245,18 @@ def test_otx_multi_gpu_train_semisl(self, template, tmp_dir_path):
         tmp_dir_path = tmp_dir_path / "multi_class_cls/test_multi_gpu_semisl"
         args_semisl_multigpu = copy.deepcopy(args)
         args_semisl_multigpu["--unlabeled-data-roots"] = args["--train-data-roots"]
-        args_semisl_multigpu["train_params"].extend(["--algo_backend.train_type", "Semisupervised"])
         args_semisl_multigpu["--gpus"] = "0,1"
         otx_train_testing(template, tmp_dir_path, otx_dir, args_semisl_multigpu)
+        template_dir = get_template_dir(template, tmp_dir_path)
+        assert os.path.exists(f"{template_dir}/semisl")
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids)
     def test_otx_train_selfsl(self, template, tmp_dir_path):
         tmp_dir_path = tmp_dir_path / "multi_class_cls/test_selfsl"
         otx_train_testing(template, tmp_dir_path, otx_dir, args_selfsl)
+        template_dir = get_template_dir(template, tmp_dir_path)
+        assert os.path.exists(f"{template_dir}/selfsl")
 
     @e2e_pytest_component
     @pytest.mark.skipif(MULTI_GPU_UNAVAILABLE, reason="The number of gpu is insufficient")
@@ -269,6 +266,8 @@ def test_otx_multi_gpu_train_selfsl(self, template, tmp_dir_path):
         args_selfsl_multigpu = copy.deepcopy(args_selfsl)
         args_selfsl_multigpu["--gpus"] = "0,1"
         otx_train_testing(template, tmp_dir_path, otx_dir, args_selfsl_multigpu)
+        template_dir = get_template_dir(template, tmp_dir_path)
+        assert os.path.exists(f"{template_dir}/selfsl")
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)
@@ -426,8 +425,9 @@ def test_otx_train_semisl(self, template, tmp_dir_path):
         tmp_dir_path = tmp_dir_path / "multi_label_cls" / "test_semisl"
         args_semisl = copy.deepcopy(args_m)
         args_semisl["--unlabeled-data-roots"] = args_m["--train-data-roots"]
-        args_semisl["train_params"].extend(["--algo_backend.train_type", "Semisupervised"])
         otx_train_testing(template, tmp_dir_path, otx_dir, args_semisl)
+        template_dir = get_template_dir(template, tmp_dir_path)
+        assert os.path.exists(f"{template_dir}/semisl")
 
 
 args_h = {

tests/assets/common_semantic_segmentation_dataset/train_with_unlabeled/images/0002.png

@@ -44,15 +44,7 @@
     "--test-data-roots": "tests/assets/car_tree_bug",
     "--unlabeled-data-roots": "tests/assets/car_tree_bug",
     "--input": "tests/assets/car_tree_bug/images/train",
-    "train_params": [
-        "params",
-        "--learning_parameters.num_iters",
-        "1",
-        "--learning_parameters.batch_size",
-        "4",
-        "--algo_backend.train_type",
-        "Semisupervised",
-    ],
+    "train_params": ["params", "--learning_parameters.num_iters", "1", "--learning_parameters.batch_size", "4"],
 }
 
 # Training params for resume, num_iters*2
@@ -206,6 +198,8 @@ def test_otx_multi_gpu_train(self, template, tmp_dir_path):
     def test_otx_train_semisl(self, template, tmp_dir_path):
         tmp_dir_path = tmp_dir_path / "detection/test_semisl"
         otx_train_testing(template, tmp_dir_path, otx_dir, args_semisl)
+        template_dir = get_template_dir(template, tmp_dir_path)
+        assert os.path.exists(f"{template_dir}/semisl")
 
     @e2e_pytest_component
     @pytest.mark.skipif(MULTI_GPU_UNAVAILABLE, reason="The number of gpu is insufficient")
@@ -215,6 +209,8 @@ def test_otx_multi_gpu_train_semisl(self, template, tmp_dir_path):
         args_semisl_multigpu = copy.deepcopy(args_semisl)
         args_semisl_multigpu["--gpus"] = "0,1"
         otx_train_testing(template, tmp_dir_path, otx_dir, args_semisl_multigpu)
+        template_dir = get_template_dir(template, tmp_dir_path)
+        assert os.path.exists(f"{template_dir}/semisl")
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", default_templates, ids=default_templates_ids)

tests/assets/common_semantic_segmentation_dataset/train_with_unlabeled/images/0003.png

@@ -46,36 +46,14 @@
     "--val-data-roots": "tests/assets/common_semantic_segmentation_dataset/val",
     "--test-data-roots": "tests/assets/common_semantic_segmentation_dataset/val",
     "--unlabeled-data-roots": "tests/assets/common_semantic_segmentation_dataset/train",
-    "train_params": [
-        "params",
-        "--learning_parameters.num_iters",
-        "1",
-        "--learning_parameters.batch_size",
-        "4",
-        "--algo_backend.train_type",
-        "Semisupervised",
-    ],
-}
-
-args_semisl_auto_config = {
-    "--train-data-roots": "tests/assets/common_semantic_segmentation_dataset/train_with_unlabeled",
-    "--val-data-roots": "tests/assets/common_semantic_segmentation_dataset/val",
-    "--test-data-roots": "tests/assets/common_semantic_segmentation_dataset/val",
     "train_params": ["params", "--learning_parameters.num_iters", "1", "--learning_parameters.batch_size", "4"],
 }
 
 args_selfsl = {
     "--train-data-roots": "tests/assets/common_semantic_segmentation_dataset/train",
     "--input": "tests/assets/segmentation/custom/images/training",
-    "train_params": [
-        "params",
-        "--learning_parameters.num_iters",
-        "1",
-        "--learning_parameters.batch_size",
-        "4",
-        "--algo_backend.train_type",
-        "Selfsupervised",
-    ],
+    "--train-type": "Selfsupervised",
+    "train_params": ["params", "--learning_parameters.num_iters", "1", "--learning_parameters.batch_size", "4"],
 }
 
 # Training params for resume, num_iters*2
@@ -205,12 +183,6 @@ def test_otx_multi_gpu_train(self, template, tmp_dir_path):
     def test_otx_train_semisl(self, template, tmp_dir_path):
         tmp_dir_path = tmp_dir_path / "segmentation/test_semisl"
         otx_train_testing(template, tmp_dir_path, otx_dir, args_semisl)
-
-    @e2e_pytest_component
-    @pytest.mark.parametrize("template", templates_inc_segnext, ids=templates_ids_inc_segnext)
-    def test_otx_train_semisl_auto_config(self, template, tmp_dir_path):
-        tmp_dir_path = tmp_dir_path / "segmentation/test_semisl_auto"
-        otx_train_testing(template, tmp_dir_path, otx_dir, args_semisl_auto_config)
         template_dir = get_template_dir(template, tmp_dir_path)
         # Check that semi-sl launched
         assert os.path.exists(f"{template_dir}/semisl")
@@ -223,6 +195,9 @@ def test_otx_multi_gpu_train_semisl(self, template, tmp_dir_path):
         args_semisl_multigpu = copy.deepcopy(args_semisl)
         args_semisl_multigpu["--gpus"] = "0,1"
         otx_train_testing(template, tmp_dir_path, otx_dir, args_semisl_multigpu)
+        template_dir = get_template_dir(template, tmp_dir_path)
+        # Check that semi-sl launched
+        assert os.path.exists(f"{template_dir}/semisl")
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

tests/assets/common_semantic_segmentation_dataset/train_with_unlabeled/masks/0001.png

@@ -140,6 +140,8 @@ def otx_train_testing(template, root, otx_dir, args, deterministic=True):
         command_line.extend(["--gpus", args["--gpus"]])
         if "--multi-gpu-port" in args:
             command_line.extend(["--multi-gpu-port", args["--multi-gpu-port"]])
+    if "--train-type" in args:
+        command_line.extend(["--train-type", args["--train-type"]])
     if deterministic:
         command_line.extend(["--deterministic"])
     if "train_params" in args:

tests/assets/common_semantic_segmentation_dataset/train_with_unlabeled/masks/0002.png

@@ -502,23 +502,17 @@ def test_auto_semisl_detection(self, mocker):
         config_manager.args.unlabeled_data_roots = "non_exist_dir"
         with pytest.raises(ValueError):
             config_manager._get_train_type(ignore_args=False)
-        try:
-            config_manager.args.unlabeled_data_roots = None
-            os.mkdir("tests/assets/classification_dataset/unlabeled_images")
-            # unlabeled root is empty
-            config_manager.args.train_data_roots = "tests/assets/classification_dataset"
-            assert config_manager._get_train_type(ignore_args=False) == "Incremental"
-            Path('tests/assets/classification_dataset/unlabeled_images/file.jpg').touch()
-            # number of images in unlabeled root is unsufficient
-            assert config_manager._get_train_type(ignore_args=False) == "Incremental"
-            config_manager.args.unlabeled_data_roots = "tests/assets/classification_dataset/unlabeled_images"
-            assert config_manager._get_train_type(ignore_args=False) == "Incremental"
-            # number of images in unlabeled root is sufficient
-            Path('tests/assets/classification_dataset/unlabeled_images/file2.jpg').touch()
-            Path('tests/assets/classification_dataset/unlabeled_images/file3.jpg').touch()
-            assert config_manager._get_train_type(ignore_args=False) == "Semisupervised"
-        finally:
-            shutil.rmtree("tests/assets/classification_dataset/unlabeled_images")
+        tempdir = tempfile.mkdtemp()
+        # unlabeled root is empty
+        config_manager.args.unlabeled_data_roots = str(tempdir)
+        with pytest.raises(ValueError):
+            config_manager._get_train_type(ignore_args=False)
+        Path(f"{tempdir}/file.jpg").touch()
+        # number of images in unlabeled root is unsufficient
+        assert config_manager._get_train_type(ignore_args=False) == "Incremental"
+        Path(f"{tempdir}/file1.jpg").touch()
+        Path(f"{tempdir}/file2.jpg").touch()
+        assert config_manager._get_train_type(ignore_args=False) == "Semisupervised"
 
     @e2e_pytest_unit
     def test_auto_selfsl_detection(self, mocker):