Source code for esmvalcore.preprocessor._bias

"""Preprocessor functions to calculate biases from data."""
import logging

import dask.array as da
import iris.cube

from ._io import concatenate

logger = logging.getLogger(__name__)

[docs] def bias(products, bias_type='absolute', denominator_mask_threshold=1e-3, keep_reference_dataset=False): """Calculate biases. Notes ----- This preprocessor requires a reference dataset. For this, exactly one input dataset needs to have the facet ``reference_for_bias: true`` defined in the recipe. In addition, all input datasets need to have identical dimensional coordinates. This can for example be ensured with the preprocessors :func:`esmvalcore.preprocessor.regrid` and/or :func:`esmvalcore.preprocessor.regrid_time`. Parameters ---------- products: set of esmvalcore.preprocessor.PreprocessorFile Input datasets. Exactly one datasets needs the facet ``reference_for_bias: true``. bias_type: str, optional (default: 'absolute') Bias type that is calculated. Must be one of ``'absolute'`` (dataset - ref) or ``'relative'`` ((dataset - ref) / ref). denominator_mask_threshold: float, optional (default: 1e-3) Threshold to mask values close to zero in the denominator (i.e., the reference dataset) during the calculation of relative biases. All values in the reference dataset with absolute value less than the given threshold are masked out. This setting is ignored when ``bias_type`` is set to ``'absolute'``. Please note that for some variables with very small absolute values (e.g., carbon cycle fluxes, which are usually :math:`< 10^{-6}` kg m :math:`^{-2}` s :math:`^{-1}`) it is absolutely essential to change the default value in order to get reasonable results. keep_reference_dataset: bool, optional (default: False) If ``True``, keep the reference dataset in the output. If ``False``, drop the reference dataset. Returns ------- set of esmvalcore.preprocessor.PreprocessorFile Output datasets. Raises ------ ValueError Not exactly one input datasets contains the facet ``reference_for_bias: true``; ``bias_type`` is not one of ``'absolute'`` or ``'relative'``. """ # Get reference product reference_product = [] for product in products: if product.attributes.get('reference_for_bias', False): reference_product.append(product) if len(reference_product) != 1: raise ValueError( f"Expected exactly 1 dataset with 'reference_for_bias: true', " f"found {len(reference_product):d}") reference_product = reference_product[0] # Extract reference cube # Note: For technical reasons, product objects contain the member # ``cubes``, which is a list of cubes. However, this is expected to be a # list with exactly one element due to the call of concatenate earlier in # the preprocessing chain of ESMValTool. To make sure that this # preprocessor can also be used outside the ESMValTool preprocessing chain, # an additional concatenate call is added here. ref_cube = concatenate(reference_product.cubes) if bias_type == 'relative': ref_cube = ref_cube.copy() =, -denominator_mask_threshold, denominator_mask_threshold) # Iterate over all input datasets and calculate bias output_products = set() for product in products: if product == reference_product: continue cube = concatenate(product.cubes) cube_metadata = cube.metadata # Calculate bias if bias_type == 'absolute': cube = cube - ref_cube new_units = str(cube.units) elif bias_type == 'relative': cube = (cube - ref_cube) / ref_cube new_units = '1' else: raise ValueError( f"Expected one of ['absolute', 'relative'] for bias_type, got " f"'{bias_type}'") # Adapt cube metadata and provenance information cube.metadata = cube_metadata cube.units = new_units product.attributes['units'] = new_units product.wasderivedfrom(reference_product) product.cubes = iris.cube.CubeList([cube]) output_products.add(product) # Add reference dataset to output if desired if keep_reference_dataset: output_products.add(reference_product) return output_products