Split data_for_key

Author: yngve-sk

src/ert/config/ext_param_config.py

@@ -11,7 +11,7 @@
 
 from ert.substitutions import substitute_runpath_name
 
-from .parameter_config import ParameterConfig
+from .parameter_config import ParameterConfig, ParameterMetadata
 
 if TYPE_CHECKING:
     import numpy.typing as npt
@@ -32,6 +32,21 @@ class ExtParamConfig(ParameterConfig):
     If a list of strings is given, the order is preserved.
     """
 
+    @property
+    def parameter_keys(self) -> list[str]:
+        if isinstance(self.input_keys, dict):
+            flattened = []
+            for key, subkeys in self.input_keys.items():
+                for subkey in subkeys:
+                    flattened.append(key + subkey)
+            return flattened
+        else:
+            return self.input_keys
+
+    @property
+    def metadata(self) -> list[ParameterMetadata]:
+        return []
+
     input_keys: list[str] | dict[str, list[str]] = field(default_factory=list)
     forward_init: bool = False
     output_file: str = ""

src/ert/config/field.py

@@ -15,7 +15,7 @@
 from ert.utils import log_duration
 
 from ._str_to_bool import str_to_bool
-from .parameter_config import ParameterConfig
+from .parameter_config import ParameterConfig, ParameterMetadata
 from .parsing import ConfigValidationError, ConfigWarning
 
 if TYPE_CHECKING:
@@ -41,6 +41,26 @@ class Field(ParameterConfig):
     grid_file: str
     mask_file: Path | None = None
 
+    @property
+    def parameter_keys(self) -> list[str]:
+        return []
+
+    @property
+    def metadata(self) -> list[ParameterMetadata]:
+        return [
+            ParameterMetadata(
+                key=self.name,
+                transformation=self.output_transformation,
+                dimensionality=3,
+                userdata={
+                    "data_origin": "FIELD",
+                    "nx": self.nx,
+                    "ny": self.ny,
+                    "nz": self.nz,
+                },
+            )
+        ]
+
     @classmethod
     def from_config_list(
         cls,

src/ert/config/gen_data_config.py

@@ -11,7 +11,7 @@
 from ert.validation import rangestring_to_list
 
 from .parsing import ConfigDict, ConfigValidationError, ErrorInfo
-from .response_config import InvalidResponseFile, ResponseConfig
+from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
 from .responses_index import responses_index
 
 
@@ -21,6 +21,21 @@ class GenDataConfig(ResponseConfig):
     report_steps_list: list[list[int] | None] = dataclasses.field(default_factory=list)
     has_finalized_keys: bool = True
 
+    @property
+    def metadata(self) -> list[ResponseMetadata]:
+        return [
+            ResponseMetadata(
+                response_type=self.name,
+                response_key=response_key,
+                filter_on={"report_steps": report_steps}
+                if report_steps is not None
+                else None,
+            )
+            for response_key, report_steps in zip(
+                self.keys, self.report_steps_list, strict=False
+            )
+        ]
+
     def __post_init__(self) -> None:
         if len(self.report_steps_list) == 0:
             self.report_steps_list = [[0] for _ in self.keys]

src/ert/config/gen_kw_config.py

@@ -19,7 +19,7 @@
 from ert.substitutions import substitute_runpath_name
 
 from ._str_to_bool import str_to_bool
-from .parameter_config import ParameterConfig
+from .parameter_config import ParameterConfig, ParameterMetadata
 from .parsing import ConfigValidationError, ConfigWarning, ErrorInfo
 
 if TYPE_CHECKING:
@@ -85,6 +85,26 @@ def __contains__(self, item: str) -> bool:
     def __len__(self) -> int:
         return len(self.transform_functions)
 
+    @property
+    def parameter_keys(self) -> list[str]:
+        keys = []
+        for tf in self.transform_functions:
+            keys.append(tf.name)
+
+        return keys
+
+    @property
+    def metadata(self) -> list[ParameterMetadata]:
+        return [
+            ParameterMetadata(
+                key=f"{self.name}:{tf.name}",
+                transformation=tf.transform_function_name,
+                dimensionality=2,
+                userdata={"data_origin": "GEN_KW"},
+            )
+            for tf in self.transform_functions
+        ]
+
     @classmethod
     def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
         gen_kw_key = cast(str, gen_kw[0])

src/ert/config/parameter_config.py

@@ -3,10 +3,11 @@
 import dataclasses
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
 
 import numpy as np
 import xarray as xr
+from pydantic import BaseModel
 
 if TYPE_CHECKING:
     import numpy.typing as npt
@@ -28,6 +29,13 @@ def __init__(self, data: list[tuple[Any, Any]]) -> None:
         super().__init__(data)
 
 
+class ParameterMetadata(BaseModel):
+    key: str
+    transformation: str | None
+    dimensionality: Literal[1, 2, 3] = 1
+    userdata: dict[str, Any]
+
+
 @dataclasses.dataclass
 class ParameterConfig(ABC):
     name: str
@@ -42,6 +50,21 @@ def sample_or_load(
     ) -> xr.Dataset:
         return self.read_from_runpath(Path(), real_nr, 0)
 
+    @property
+    @abstractmethod
+    def parameter_keys(self) -> list[str]:
+        """
+        Returns a list of parameter keys within this parameter group
+        """
+
+    @property
+    @abstractmethod
+    def metadata(self) -> list[ParameterMetadata]:
+        """
+        Returns metadata describing this parameter
+
+        """
+
     @abstractmethod
     def __len__(self) -> int:
         """Number of parameters"""

src/ert/config/response_config.py

@@ -3,6 +3,7 @@
 from typing import Any, Self
 
 import polars as pl
+from pydantic import BaseModel
 
 from .parameter_config import CustomDict
 from .parsing import ConfigDict
@@ -15,13 +16,28 @@ class InvalidResponseFile(Exception):
     """
 
 
+class ResponseMetadata(BaseModel):
+    response_type: str
+    response_key: str
+    # ex: { "report_step": [0, 199, 299] }
+    filter_on: dict[str, list[Any]] | None  # Dimensions
+
+
 @dataclasses.dataclass
 class ResponseConfig(ABC):
     name: str
     input_files: list[str] = dataclasses.field(default_factory=list)
     keys: list[str] = dataclasses.field(default_factory=list)
     has_finalized_keys: bool = False
 
+    @property
+    @abstractmethod
+    def metadata(self) -> list[ResponseMetadata]:
+        """
+        Returns metadata describing this response
+
+        """
+
     @abstractmethod
     def read_from_file(self, run_path: str, iens: int, iter: int) -> pl.DataFrame:
         """Reads the data for the response from run_path.

src/ert/config/summary_config.py

@@ -13,7 +13,7 @@
 from .ensemble_config import Refcase
 from .parsing import ConfigDict, ConfigKeys
 from .parsing.config_errors import ConfigValidationError, ConfigWarning
-from .response_config import InvalidResponseFile, ResponseConfig
+from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
 from .responses_index import responses_index
 
 logger = logging.getLogger(__name__)
@@ -32,6 +32,17 @@ def __post_init__(self) -> None:
         if len(self.keys) < 1:
             raise ValueError("SummaryConfig must be given at least one key")
 
+    @property
+    def metadata(self) -> list[ResponseMetadata]:
+        return [
+            ResponseMetadata(
+                response_type=self.name,
+                response_key=response_key,
+                filter_on=None,
+            )
+            for response_key in self.keys
+        ]
+
     @property
     def expected_input_files(self) -> list[str]:
         base = self.input_files[0]

src/ert/config/surface_config.py

@@ -11,7 +11,7 @@
 from ert.substitutions import substitute_runpath_name
 
 from ._str_to_bool import str_to_bool
-from .parameter_config import ParameterConfig
+from .parameter_config import ParameterConfig, ParameterMetadata
 from .parsing import ConfigValidationError, ErrorInfo
 
 if TYPE_CHECKING:
@@ -34,6 +34,14 @@ class SurfaceConfig(ParameterConfig):
     output_file: Path
     base_surface_path: str
 
+    @property
+    def parameter_keys(self) -> list[str]:
+        return []
+
+    @property
+    def metadata(self) -> list[ParameterMetadata]:
+        return []
+
     @classmethod
     def from_config_list(cls, surface: list[str | dict[str, str]]) -> Self:
         name = cast(str, surface[0])