From 7986f182e451b1116db7e2212ee1c58f812a773f Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 7 Jul 2026 12:12:42 +0100 Subject: [PATCH 1/8] Remove the DECODE_TO_STRINGS_ON_READ from public API. --- lib/iris/fileformats/netcdf/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/iris/fileformats/netcdf/__init__.py b/lib/iris/fileformats/netcdf/__init__.py index 60b4b5895a..795a9c5af0 100644 --- a/lib/iris/fileformats/netcdf/__init__.py +++ b/lib/iris/fileformats/netcdf/__init__.py @@ -48,7 +48,6 @@ "CFNameCoordMap", "CF_CONVENTIONS_VERSION", "DEBUG", - "DECODE_TO_STRINGS_ON_READ", "DEFAULT_READ_ENCODING", "DEFAULT_WRITE_ENCODING", "MESH_ELEMENTS", From b4f1a3fcef2ab6f54cd11d4f9fa41bd27bfb0fe1 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 7 Jul 2026 12:14:03 +0100 Subject: [PATCH 2/8] First-draft chardata docs. --- .../src/user_manual/explanation/netcdf_io.rst | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/docs/src/user_manual/explanation/netcdf_io.rst b/docs/src/user_manual/explanation/netcdf_io.rst index f5f608f5c8..7291e456da 100644 --- a/docs/src/user_manual/explanation/netcdf_io.rst +++ b/docs/src/user_manual/explanation/netcdf_io.rst @@ -133,6 +133,129 @@ Iris' optimisation all together, and will take its chunksizes from Dask's behavi (70, 37, 49) +Character and String datatypes +------------------------------ + +In NetCDF +~~~~~~~~~ +In the NetCDF v4 implementation, there are three specific areas where the datatype and +storage characteristics of character data are relevant: + +* The **names** of file components (variables, dimensions, and attributes) are + natively unicode-capable strings of arbitrary (variable) length. + +* **Attributes** with string content likewise *appear* to be natively unicode. However, + the actual datatype may vary, being either 'char' or 'string'. + +* The **content of variables** can be either 'char' or 'string'. + + - 'string' type variables contain a variable-length unicode string at each array element. + + - 'char' type variables contain one-byte characters, and generally have a fixed-length + 'string dimension'. If they contain *only* ascii character values, this is + uncomplicated, but they may also be used to contain non-ascii data (i.e. + including unicode characters). There is no universally defined agreement on how + to indicate a variable containing non-ascii data, but many datasets have used a + suitable ``_Encoding`` attribute. + +The NetCDF documentation also mentions that an '_Encoding' attribute may be used to +represent non-ascii strings in a 'char' type array. However this is described as +"reserved for future use", and its valid values and effects are not explicitly defined. + +However, it is also notable that the standard ``ncgen`` and ``ncdump`` tools *do* +correctly interpret an ``_Encoding`` attribute in most cases, despite this not being an +"official" solution. + + +In CF +~~~~~ +The CF Conventions define a subset of "allowed" datatypes, and describe various data +elements stored as variables, such as data variables, auxiliary coordinates, cell +methods, etc. + +CF currently supports the use of either netcdf 'string' or 'char' arrays for any +variables. + +However, historically CF has had more limited support, and also 'unofficial' +conventions for the use of 'char' arrays have been used which may be encountered in +older datasets. + +Since v1.8, CF has allowed the use of 'string' data in all variables, but prior to that +it was required to use 'char' type only, without providing any official means of storing +non-ascii data. + +Since v1.12, CF also mandates a *default* assumption of utf-8 encoding to store non-ascii +data in 'char form, but it also notes that some data has used an "_Encoding" attribute +in the past -- though this has never been an official CF usage. + +Where strings are stored as 'char' type, the array must have a 'string dimension', +which is a normal file dimension. Thus, these strings always have a *fixed byte width*. + + +In Iris +~~~~~~~ +In Iris, the 'string' data type is not fully supported at present, though this is +planned for future releases. See the following section `Variable-length datatypes`_ for +an interim solution enabling you to *load* variable-length string data. + +Iris stores string data in arrays of dtype "U", where is a maximum character +width. However, this data is currently **only** read and written as 'char' type arrays +in netCDF files. + +Iris provides a set of valid encodings for non-ascii data : +"ascii" / "utf8" / "utf16" and "utf32". These (or valid aliases) will appear as the +"_Encoding" attribute of a file variable, and likewise as a regular attribute of the +corresponding Iris component object (e.g. cube or coordinate). + +**On loading**, if there is a valid ``_Encoding`` attribute this is used to decode the data, +otherwise a default encoding of "utf8" is applied: This works transparently if only +ascii bytes are present, and also allows the ``_Encoding`` attribute to be omitted as +long as utf8 was used. + +**On saving**, any string data with only ascii characters does not require an +``_Encoding`` attribute. However if there are non-ascii bytes, and no ``_Encoding`` +attribute, then an error will be raised. + +So effectively, the **"default" encodings are 'utf8' for load and 'ascii' for save**. + +For each valid encoding there is a definite relation between the string dimension length +in the file (actually, the number of *bytes*), and the maximum character length in the +array dtype, i.e. the "" in the "U" dtype. + +The lengths of string dimensions created on write are calculated as follows: + +* ascii : n-bytes = n-characters +* utf8 : n-bytes = n-characters +* utf16 : n-bytes = 2 * (n-characters + 1) +* utf32 : n-bytes = 4 * (n-characters + 1) + +For reading, the inverse relations are applied to determine the '"U"' dtype in which +the data is presented. This will round-trip correctly, i.e. is unchanged if written and +then read back. + +For 'ascii' and 'utf32' this relationship is simple + fixed, but for 'utf8' and +'utf16', the number of encoded bytes depends on the actual characters present +**and can exceed the numbers given above**. If any string in the *actual* data encodes +to more bytes than the above-calculated string dimension, then Iris will raise an +:class:`iris.exceptions.TranslationError`. In this case, the user must manually +enforce a longer string dimension by converting the data to a longer "U" dtype : +for example, ``cube.data = cube.core_data().astype("U<20>")``. + + +In the netCDF4 Python module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* attributes with string content always appear as Python 'str' (i.e. unicode strings). + It is not possible to distinguish or control the 'char' and 'string' type in the file, + this is hidden from the user by the Python implementation. +* variable data of type 'string' is presented (read and written) as numpy arrays of dtype + "U", where is a (maximum) string length. +* variable data of type 'char' is presented (read and written) as numpy arrays of + dtype "S1" -- that is, an array of length-1 Python "bytes" objects. + The netCDF4 package can also automatically translate this to 'U' type string + arrays, if the variable has an `_Encoding` attribute, but Iris turns this feature + *off*, in order to implement its own wider-ranging support (see below). + + Variable-length datatypes ------------------------- From 6387a3e439706cfe468a4a23b63652e8b7881fe1 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 7 Jul 2026 12:25:04 +0100 Subject: [PATCH 3/8] Fix titles and re-order slightly. --- .../src/user_manual/explanation/netcdf_io.rst | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/docs/src/user_manual/explanation/netcdf_io.rst b/docs/src/user_manual/explanation/netcdf_io.rst index 7291e456da..a9ed5e90e4 100644 --- a/docs/src/user_manual/explanation/netcdf_io.rst +++ b/docs/src/user_manual/explanation/netcdf_io.rst @@ -137,7 +137,7 @@ Character and String datatypes ------------------------------ In NetCDF -~~~~~~~~~ +^^^^^^^^^ In the NetCDF v4 implementation, there are three specific areas where the datatype and storage characteristics of character data are relevant: @@ -168,7 +168,7 @@ correctly interpret an ``_Encoding`` attribute in most cases, despite this not b In CF -~~~~~ +^^^^^ The CF Conventions define a subset of "allowed" datatypes, and describe various data elements stored as variables, such as data variables, auxiliary coordinates, cell methods, etc. @@ -192,8 +192,25 @@ Where strings are stored as 'char' type, the array must have a 'string dimension which is a normal file dimension. Thus, these strings always have a *fixed byte width*. +In the netCDF4 Python module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* attributes with string content always appear as Python 'str' (i.e. unicode strings). + It is not possible to distinguish or control the 'char' and 'string' type in the file, + this is hidden from the user by the Python implementation. + +* variable data of type 'string' is presented (read and written) as numpy arrays of dtype + "U", where is a (maximum) string length. + +* variable data of type 'char' is presented (read and written) as numpy arrays of + dtype "S1" -- that is, an array of length-1 Python "bytes" objects. + The netCDF4 package can also automatically translate this to 'U' type string + arrays, if the variable has an `_Encoding` attribute, but Iris turns this feature + *off*, in order to implement its own wider-ranging support (see below). + + In Iris -~~~~~~~ +^^^^^^^ In Iris, the 'string' data type is not fully supported at present, though this is planned for future releases. See the following section `Variable-length datatypes`_ for an interim solution enabling you to *load* variable-length string data. @@ -242,19 +259,6 @@ enforce a longer string dimension by converting the data to a longer "U" dty for example, ``cube.data = cube.core_data().astype("U<20>")``. -In the netCDF4 Python module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* attributes with string content always appear as Python 'str' (i.e. unicode strings). - It is not possible to distinguish or control the 'char' and 'string' type in the file, - this is hidden from the user by the Python implementation. -* variable data of type 'string' is presented (read and written) as numpy arrays of dtype - "U", where is a (maximum) string length. -* variable data of type 'char' is presented (read and written) as numpy arrays of - dtype "S1" -- that is, an array of length-1 Python "bytes" objects. - The netCDF4 package can also automatically translate this to 'U' type string - arrays, if the variable has an `_Encoding` attribute, but Iris turns this feature - *off*, in order to implement its own wider-ranging support (see below). - Variable-length datatypes ------------------------- From e645ea16401d50aa66a41bae93b6f66f40374973 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 7 Jul 2026 12:52:42 +0100 Subject: [PATCH 4/8] Disable warn-to-fail. --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 176ce0cd1d..c8d76a89f5 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -23,7 +23,7 @@ conda: sphinx: configuration: docs/src/conf.py - fail_on_warning: true + fail_on_warning: false python: install: From 77ab73f4128128922b51bfc85b996d0235f0c135 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 7 Jul 2026 13:27:02 +0100 Subject: [PATCH 5/8] Improves. --- .../src/user_manual/explanation/netcdf_io.rst | 81 ++++++++++--------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/docs/src/user_manual/explanation/netcdf_io.rst b/docs/src/user_manual/explanation/netcdf_io.rst index a9ed5e90e4..b71e5cc9f7 100644 --- a/docs/src/user_manual/explanation/netcdf_io.rst +++ b/docs/src/user_manual/explanation/netcdf_io.rst @@ -141,24 +141,24 @@ In NetCDF In the NetCDF v4 implementation, there are three specific areas where the datatype and storage characteristics of character data are relevant: -* The **names** of file components (variables, dimensions, and attributes) are +* The **names** of file components (variables, dimensions, and attributes) are natively unicode-capable strings of arbitrary (variable) length. -* **Attributes** with string content likewise *appear* to be natively unicode. However, - the actual datatype may vary, being either 'char' or 'string'. +* **Attributes** with string content likewise *appear* to be natively unicode. However, + the actual datatype of the attribute may vary, being either 'char' or 'string'. -* The **content of variables** can be either 'char' or 'string'. +* The **content of variables** can be either 'char' or 'string'. - - 'string' type variables contain a variable-length unicode string at each array element. + * 'string' type variables contain a variable-length unicode string at each array element. - - 'char' type variables contain one-byte characters, and generally have a fixed-length - 'string dimension'. If they contain *only* ascii character values, this is - uncomplicated, but they may also be used to contain non-ascii data (i.e. - including unicode characters). There is no universally defined agreement on how - to indicate a variable containing non-ascii data, but many datasets have used a - suitable ``_Encoding`` attribute. + * 'char' type variables contain one-byte characters, and generally have a fixed-length + "string dimension". If they contain *only* ascii character values, this is + uncomplicated, but they may also be used to contain non-ascii data (i.e. + including unicode characters). There is no universally defined agreement on how + to indicate a variable containing non-ascii data, but many datasets have used a + suitable ``_Encoding`` attribute. -The NetCDF documentation also mentions that an '_Encoding' attribute may be used to +The NetCDF documentation also mentions that an ``_Encoding`` attribute may be used to represent non-ascii strings in a 'char' type array. However this is described as "reserved for future use", and its valid values and effects are not explicitly defined. @@ -185,43 +185,45 @@ it was required to use 'char' type only, without providing any official means of non-ascii data. Since v1.12, CF also mandates a *default* assumption of utf-8 encoding to store non-ascii -data in 'char form, but it also notes that some data has used an "_Encoding" attribute -in the past -- though this has never been an official CF usage. +data in 'char' form, but it also notes that some data in the past has used an +``_Encoding`` attribute -- though was never an official CF usage. -Where strings are stored as 'char' type, the array must have a 'string dimension', +Where strings are stored as 'char' type, the array must have a "string dimension", which is a normal file dimension. Thus, these strings always have a *fixed byte width*. +(However, that is not the same as a fixed *character* width, since in most encodings +non-ascii characters require more bytes to store.) In the netCDF4 Python module ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* attributes with string content always appear as Python 'str' (i.e. unicode strings). - It is not possible to distinguish or control the 'char' and 'string' type in the file, - this is hidden from the user by the Python implementation. +* attributes with string content always appear as Python 'str' (i.e. unicode strings). + It is not possible to distinguish or control the 'char' and 'string' type in the file, + this is hidden from the user by the Python implementation. -* variable data of type 'string' is presented (read and written) as numpy arrays of dtype - "U", where is a (maximum) string length. +* variable data of type 'string' is presented (read and written) as numpy arrays of dtype + "U", where is a (maximum) string length. -* variable data of type 'char' is presented (read and written) as numpy arrays of - dtype "S1" -- that is, an array of length-1 Python "bytes" objects. - The netCDF4 package can also automatically translate this to 'U' type string - arrays, if the variable has an `_Encoding` attribute, but Iris turns this feature - *off*, in order to implement its own wider-ranging support (see below). +* variable data of type 'char' is presented (read and written) as numpy arrays of + dtype "S1" -- that is, an array of length-1 Python "bytes" objects. + The netCDF4 package can also automatically translate this to string arrays of dtype + "U", if the variable has an ``_Encoding`` attribute, but Iris turns this feature + *off* in order to implement its own wider-ranging encoding support (see below). In Iris ^^^^^^^ -In Iris, the 'string' data type is not fully supported at present, though this is -planned for future releases. See the following section `Variable-length datatypes`_ for -an interim solution enabling you to *load* variable-length string data. +In Iris, the 'string' data type is supported at present, though this is planned for +future releases. See the following section `Variable-length datatypes`_ for +an interim solution enabling you at least to *load* variable-length string data. Iris stores string data in arrays of dtype "U", where is a maximum character -width. However, this data is currently **only** read and written as 'char' type arrays -in netCDF files. +width. However, this data is currently **only** read and written in netCDF files as +'char' type variables. Iris provides a set of valid encodings for non-ascii data : -"ascii" / "utf8" / "utf16" and "utf32". These (or valid aliases) will appear as the -"_Encoding" attribute of a file variable, and likewise as a regular attribute of the +"ascii", "utf8", "utf16" and "utf32". These (or valid aliases) will appear in the +``_Encoding`` attribute of a file variable, and likewise as an attribute of the corresponding Iris component object (e.g. cube or coordinate). **On loading**, if there is a valid ``_Encoding`` attribute this is used to decode the data, @@ -241,10 +243,10 @@ array dtype, i.e. the "" in the "U" dtype. The lengths of string dimensions created on write are calculated as follows: -* ascii : n-bytes = n-characters -* utf8 : n-bytes = n-characters -* utf16 : n-bytes = 2 * (n-characters + 1) -* utf32 : n-bytes = 4 * (n-characters + 1) +* ascii : n-bytes = n-characters +* utf8 : n-bytes = n-characters +* utf16 : n-bytes = 2 * (n-characters + 1) +* utf32 : n-bytes = 4 * (n-characters + 1) For reading, the inverse relations are applied to determine the '"U"' dtype in which the data is presented. This will round-trip correctly, i.e. is unchanged if written and @@ -254,10 +256,9 @@ For 'ascii' and 'utf32' this relationship is simple + fixed, but for 'utf8' and 'utf16', the number of encoded bytes depends on the actual characters present **and can exceed the numbers given above**. If any string in the *actual* data encodes to more bytes than the above-calculated string dimension, then Iris will raise an -:class:`iris.exceptions.TranslationError`. In this case, the user must manually -enforce a longer string dimension by converting the data to a longer "U" dtype : -for example, ``cube.data = cube.core_data().astype("U<20>")``. - +:class:`iris.exceptions.TranslationError`. In this case, the user must **explicitly +specify** a longer string dimension, by converting the data to a longer "U" dtype : +for example, ``cube.data = cube.core_data().astype("U20")``. Variable-length datatypes From 93ffd64bdb18df1bf06ab332d59bd97f2a704a79 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 7 Jul 2026 13:30:00 +0100 Subject: [PATCH 6/8] Pin mpl for docs build, reinstate warn-to-fail. --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index c8d76a89f5..176ce0cd1d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -23,7 +23,7 @@ conda: sphinx: configuration: docs/src/conf.py - fail_on_warning: false + fail_on_warning: true python: install: From a52e50d60bbe65d234df671baa01ca22d1f7ea0e Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 7 Jul 2026 17:10:01 +0100 Subject: [PATCH 7/8] Fix bullets --- docs/src/user_manual/explanation/netcdf_io.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/user_manual/explanation/netcdf_io.rst b/docs/src/user_manual/explanation/netcdf_io.rst index b71e5cc9f7..ffc79ad101 100644 --- a/docs/src/user_manual/explanation/netcdf_io.rst +++ b/docs/src/user_manual/explanation/netcdf_io.rst @@ -152,11 +152,11 @@ storage characteristics of character data are relevant: * 'string' type variables contain a variable-length unicode string at each array element. * 'char' type variables contain one-byte characters, and generally have a fixed-length - "string dimension". If they contain *only* ascii character values, this is - uncomplicated, but they may also be used to contain non-ascii data (i.e. - including unicode characters). There is no universally defined agreement on how - to indicate a variable containing non-ascii data, but many datasets have used a - suitable ``_Encoding`` attribute. + "string dimension". If they contain *only* ascii character values, this is + uncomplicated, but they may also be used to contain non-ascii data (i.e. + including unicode characters). There is no universally defined agreement on how + to indicate a variable containing non-ascii data, but many datasets have used a + suitable ``_Encoding`` attribute. The NetCDF documentation also mentions that an ``_Encoding`` attribute may be used to represent non-ascii strings in a 'char' type array. However this is described as From 473db52092e8d275c2ec76344992383de082a289 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 8 Jul 2026 11:13:16 +0100 Subject: [PATCH 8/8] More improvements --- .../src/user_manual/explanation/netcdf_io.rst | 171 ++++++++++++------ 1 file changed, 119 insertions(+), 52 deletions(-) diff --git a/docs/src/user_manual/explanation/netcdf_io.rst b/docs/src/user_manual/explanation/netcdf_io.rst index ffc79ad101..04227b9f28 100644 --- a/docs/src/user_manual/explanation/netcdf_io.rst +++ b/docs/src/user_manual/explanation/netcdf_io.rst @@ -135,9 +135,45 @@ Iris' optimisation all together, and will take its chunksizes from Dask's behavi Character and String datatypes ------------------------------ +Summary +^^^^^^^ + +* Iris currently *only* fully supports fixed-width 'char' type data in netCDF variables + + * the 'string' type (variable-width unicode strings) will be added in a future release + +* 'char' variable data is represented as numpy string arrays in Iris objects, such as + cubes and coordinates. + + * the numpy dtype is of the type "U", where is a character width. + * the character width relates to a string *dimension* of the netCDF variable, + which is not in the dimensions of the Iris object or its data array. + * the dtype 'width' controls the length of string dimensions created when saving + +* Iris also uses a variable ``_Encoding`` attribute to enable storage of non-ascii + characters in 'char' type arrays. + + * it appears as a regular attribute of the Iris object + * it is not needed for ascii-only data + * it is not needed to *read* 'utf-8' encoded data correctly + * it **is** required to *save* any non-ascii characters + +The following describes the nature of character and string data handling in : +netCDF itself; the CF conventions; the netCDF4 Python module and the Iris implementation. +In practice all these are connected. + +The details are generally much simpler when strings may contain only ASCII characters. +When strings may include non-ascii characters, this requires a specific encoding to be +adopted when translating to and from bytes, and rules for determining what the encoding +is or was. + +Another significant factor is the way in which all the relevant projects have changed +their features and methods over time, so that historic datasets may often use +non-standard approaches to record string data. + -In NetCDF -^^^^^^^^^ +In the NetCDF file format +^^^^^^^^^^^^^^^^^^^^^^^^^ In the NetCDF v4 implementation, there are three specific areas where the datatype and storage characteristics of character data are relevant: @@ -154,44 +190,53 @@ storage characteristics of character data are relevant: * 'char' type variables contain one-byte characters, and generally have a fixed-length "string dimension". If they contain *only* ascii character values, this is uncomplicated, but they may also be used to contain non-ascii data (i.e. - including unicode characters). There is no universally defined agreement on how - to indicate a variable containing non-ascii data, but many datasets have used a - suitable ``_Encoding`` attribute. + including unicode characters). There is no universally defined agreement for + how to indicate that bytes are encoded non-ascii data, but many older datasets + have used a variable attribute ``_Encoding`` indicating the encoding name. The NetCDF documentation also mentions that an ``_Encoding`` attribute may be used to -represent non-ascii strings in a 'char' type array. However this is described as -"reserved for future use", and its valid values and effects are not explicitly defined. +represent non-ascii strings. However this is described as "reserved for future use", +and its valid values and effects are not explicitly defined. However, it is also notable that the standard ``ncgen`` and ``ncdump`` tools *do* correctly interpret an ``_Encoding`` attribute in most cases, despite this not being an "official" solution. -In CF -^^^^^ -The CF Conventions define a subset of "allowed" datatypes, and describe various data -elements stored as variables, such as data variables, auxiliary coordinates, cell -methods, etc. +In the netCDF CF Conventions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The `CF Conventions `_ define a subset of +"allowed" datatypes, and various types of data elements represented by variables +-- such as data variables, auxiliary coordinates, cell methods, etc. CF currently supports the use of either netcdf 'string' or 'char' arrays for any variables. +However, *historically*, CF has had more limited support, and also "unofficial +conventions" have been used for string data encoded as bytes, which may be encountered +in older datasets. + +**Prior to v1.8**, CF required to use 'char' type only, and provided +**no official means** of representing non-ascii data. -However, historically CF has had more limited support, and also 'unofficial' -conventions for the use of 'char' arrays have been used which may be encountered in -older datasets. +**Since v1.8**, CF has allowed the use of 'string' data in all variables. +However, up to v1.12 there was still no official way of encoding non-ascii data in +'char' arrays. -Since v1.8, CF has allowed the use of 'string' data in all variables, but prior to that -it was required to use 'char' type only, without providing any official means of storing -non-ascii data. +**Since v1.12**, CF now mandates a *default* assumption of utf-8 encoding to store +non-ascii data in 'char' form. It does also note that some data in the past has used an +``_Encoding`` attribute -- though this was never an official CF usage. -Since v1.12, CF also mandates a *default* assumption of utf-8 encoding to store non-ascii -data in 'char' form, but it also notes that some data in the past has used an -``_Encoding`` attribute -- though was never an official CF usage. +Characteristics +~~~~~~~~~~~~~~~ +Where strings are stored as 'char' type, which is the more common traditional approach, +the array must have a "string dimension", which is a normal file dimension. Thus, these +strings always have a *fixed byte width*. (However, that is not the same as a fixed +*character* width, since in most encodings non-ascii characters require more bytes to +store). -Where strings are stored as 'char' type, the array must have a "string dimension", -which is a normal file dimension. Thus, these strings always have a *fixed byte width*. -(However, that is not the same as a fixed *character* width, since in most encodings -non-ascii characters require more bytes to store.) +Although the variable-length 'string' data is now supported in CF, the use of +fixed-width 'char' arrays is obviously more efficient for storage and access, and it is +still the most common approach in practice. In the netCDF4 Python module @@ -206,42 +251,57 @@ In the netCDF4 Python module * variable data of type 'char' is presented (read and written) as numpy arrays of dtype "S1" -- that is, an array of length-1 Python "bytes" objects. - The netCDF4 package can also automatically translate this to string arrays of dtype - "U", if the variable has an ``_Encoding`` attribute, but Iris turns this feature - *off* in order to implement its own wider-ranging encoding support (see below). + + .. note:: + + The netCDF4 package can also automatically translate this to string arrays of + dtype "U", if the variable has an ``_Encoding`` attribute. **However,** + Iris turns this feature *off*, in order to implement its own wider-ranging + encoding support (described below). In Iris ^^^^^^^ -In Iris, the 'string' data type is supported at present, though this is planned for -future releases. See the following section `Variable-length datatypes`_ for -an interim solution enabling you at least to *load* variable-length string data. +.. note:: + + In Iris, **the 'string' data type is not supported at present**, though this is + planned for future releases. See the following section `Variable-length datatypes`_ + for an interim solution enabling you at least to *load* variable-length string data. Iris stores string data in arrays of dtype "U", where is a maximum character width. However, this data is currently **only** read and written in netCDF files as -'char' type variables. +'char' type variables (i.e. byte arrays). -Iris provides a set of valid encodings for non-ascii data : -"ascii", "utf8", "utf16" and "utf32". These (or valid aliases) will appear in the +Iris supports a specific set of valid encodings for non-ascii data : +"ascii", "utf8", "utf16" and "utf32". These (or aliases) will appear in the ``_Encoding`` attribute of a file variable, and likewise as an attribute of the corresponding Iris component object (e.g. cube or coordinate). -**On loading**, if there is a valid ``_Encoding`` attribute this is used to decode the data, -otherwise a default encoding of "utf8" is applied: This works transparently if only -ascii bytes are present, and also allows the ``_Encoding`` attribute to be omitted as -long as utf8 was used. - -**On saving**, any string data with only ascii characters does not require an -``_Encoding`` attribute. However if there are non-ascii bytes, and no ``_Encoding`` -attribute, then an error will be raised. - -So effectively, the **"default" encodings are 'utf8' for load and 'ascii' for save**. - +When loading +~~~~~~~~~~~~ +If there is a valid ``_Encoding`` attribute this is used to decode the +data, otherwise a default encoding of "utf8" is applied: This works transparently when +only ascii characters are present, and also allows the ``_Encoding`` attribute to be +omitted as long as utf8 was used. An invalid or unsupported encoding name will be +ignored, with a warning, but the attribute will still be added to the Iris component +object. + +When saving +~~~~~~~~~~~ +Any string data with only ascii characters does not require an ``_Encoding`` attribute. +However if there are any non-ascii characters, and no ``_Encoding`` +attribute, then an error will be raised. An invalid or unsupported encoding name will +be ignored, with a warning, but the attribute will still be stored to the file. + +So effectively, the **default encoding is 'utf8' for load and 'ascii' for save**. + +String width dimensions +~~~~~~~~~~~~~~~~~~~~~~~ For each valid encoding there is a definite relation between the string dimension length -in the file (actually, the number of *bytes*), and the maximum character length in the -array dtype, i.e. the "" in the "U" dtype. +in the file (actually, the number of *bytes*), and the maximum character length, aka +string width, in the array dtype : i.e. the "" in the "U" dtype. -The lengths of string dimensions created on write are calculated as follows: +The **lengths of string dimensions created on write** are calculated as follows: * ascii : n-bytes = n-characters * utf8 : n-bytes = n-characters @@ -249,17 +309,24 @@ The lengths of string dimensions created on write are calculated as follows: * utf32 : n-bytes = 4 * (n-characters + 1) For reading, the inverse relations are applied to determine the '"U"' dtype in which -the data is presented. This will round-trip correctly, i.e. is unchanged if written and -then read back. +the data is presented. This will always round-trip correctly, i.e. the dimension length +is unchanged if data is read and then written back. -For 'ascii' and 'utf32' this relationship is simple + fixed, but for 'utf8' and -'utf16', the number of encoded bytes depends on the actual characters present +For 'ascii' and 'utf32' this character-to-byte relationship is simple + fixed, but for +'utf8' and 'utf16', the number of encoded bytes depends on the actual characters present **and can exceed the numbers given above**. If any string in the *actual* data encodes to more bytes than the above-calculated string dimension, then Iris will raise an :class:`iris.exceptions.TranslationError`. In this case, the user must **explicitly specify** a longer string dimension, by converting the data to a longer "U" dtype : for example, ``cube.data = cube.core_data().astype("U20")``. +.. warning:: + + When processing string arrays, Numpy does not routinely preserve the "" width part + of "U" type data : instead, some operations will reduce it to the maximum width + occurring. So in these cases also, it may be necessary to explicitly re-assert the + desired "string width" before saving -- use ``.astype()``, as above. + Variable-length datatypes -------------------------