Commit 0f0cac5731add015ad00f6a4f4bd8b180ea78372 - coda

+9

-0

CHANGES less more

	0	2.23 2021-12-17
	1	~~~~~~~~~~~~~~~
	2
	3	* Fixed issue with reading compressed variables from CDF where variable would
	4	also contain VVR records besides CVVR records.
	5
	6	* Added constituentType for GRIB2 data with productDefinitionTemplate 40 and
	7	fixed reading of these grib messages.
	8
0	9	2.22.2 2021-10-20
1	10	~~~~~~~~~~~~~~~~~
2	11

+2

-2

CMakeLists.txt less more

4	4
5	5	project(CODA)
6	6
7		set(VERSION 2.22.2)
	7	set(VERSION 2.23)
8	8	set(CODA_VERSION \"${VERSION}\")
9	9	set(CPACK_PACKAGE_VERSION ${VERSION})
10	10
11	11	# Set dynamic library version
12	12	set(LIBCODA_CURRENT 15)
13		set(LIBCODA_REVISION 5)
	13	set(LIBCODA_REVISION 6)
14	14	set(LIBCODA_AGE 0)
15	15	math(EXPR LIBCODA_MAJOR "${LIBCODA_CURRENT} - ${LIBCODA_AGE}")
16	16	set(LIBCODA_MINOR ${LIBCODA_AGE})

+1

-1

INSTALL less more

116	116	CODA is also available as a conda package for Windows, Linux, and macOS
117	117	(only 64bit and Python3). You can install CODA using conda with::
118	118
119		$ conda install -c stcorp coda
	119	$ conda install -c conda-forge coda
120	120
121	121	Note that this version of CODA does not include the Fortran, Java, MATLAB,
122	122	and IDL interfaces. It only includes the Python interfaces, C library, and

+1

-1

README less more

0	0	Copyright (C) 2007-2021 S[&]T, The Netherlands
1	1
2		CODA 2.22.2 Release Notes
	2	CODA 2.23 Release Notes
3	3
4	4
5	5	CODA is the Common Data Access framework that allows reading of scientific data

+2

-2

configure.ac less more

0	0	# autoconf file for CODA
1		AC_INIT([CODA],[2.22.2])
	1	AC_INIT([CODA],[2.23])
2	2	AC_CONFIG_AUX_DIR([.])
3	3	AM_INIT_AUTOMAKE(foreign subdir-objects 1.12 -Wall)
4	4

171	171	# 6. If any interfaces have been removed or changed since the last public
172	172	# release, then set age to 0.
173	173	LIBCODA_CURRENT=15
174		LIBCODA_REVISION=5
	174	LIBCODA_REVISION=6
175	175	LIBCODA_AGE=0
176	176	AC_SUBST(LIBCODA_CURRENT)
177	177	AC_SUBST(LIBCODA_REVISION)

+1

-0

doc/html/codadef/codadef-grib.html less more

163	163	<tr><td>/[]/grib2/data[]/gridRecordIndex</td><td>integer</td><td>uint32</td><td>index into /[]/grid[] to find associated grid definition data</td></tr>
164	164	<tr><td>/[]/grib2/data[]/parameterCategory</td><td>integer</td><td>uint8</td><td></td></tr>
165	165	<tr><td>/[]/grib2/data[]/parameterNumber</td><td>integer</td><td>uint8</td><td></td></tr>
	166	<tr><td>/[]/grib2/data[]/constituentType</td><td>integer</td><td>uint16</td><td>optional</td></tr>
166	167	<tr><td>/[]/grib2/data[]/typeOfGeneratingProcess</td><td>integer</td><td>uint8</td><td></td></tr>
167	168	<tr><td>/[]/grib2/data[]/backgroundProcess</td><td>integer</td><td>uint8</td><td></td></tr>
168	169	<tr><td>/[]/grib2/data[]/generatingProcessIdentifier</td><td>integer</td><td>uint8</td><td></td></tr>

+7

-2

libcoda/coda-cdf-cursor.c less more

424	424	{
425	425	value_size = size_boundary;
426	426	}
427		if (variable->data != NULL)
428		{
	427	if ((variable->data != NULL) && (value_size > 0))
	428	{
	429	if (offset > (variable->num_records * variable->num_values_per_record * variable->value_size))
	430	{
	431	coda_set_error(CODA_ERROR_UNSUPPORTED_PRODUCT, "Offset too large in accessing data of CDF variable");
	432	return -1;
	433	}
429	434	memcpy(dst, &variable->data[offset], value_size);
430	435	}
431	436	else

+2

-1

libcoda/coda-cdf-internal.h less more

70	70	int num_values_per_record;
71	71	int value_size;
72	72	int sparse_rec_method; /* 0: no sparse records, 1: padded sparse records, 2: previous sparse records */
	73	int has_compression;
73	74	int64_t offset; / file offset for each record - will be offset into 'data' if 'data != NULL' */
74	75	int8_t *data;
75	76	} coda_cdf_variable;

102	103	coda_dynamic_type *coda_cdf_variable_new(int32_t data_type, int32_t max_rec, int32_t rec_varys, int32_t num_dims,
103	104	int32_t dim[CODA_MAX_NUM_DIMS], int32_t dim_varys[CODA_MAX_NUM_DIMS],
104	105	coda_array_ordering array_ordering, int32_t num_elements,
105		int sparse_rec_method, coda_cdf_variable **variable);
	106	int sparse_rec_method, int has_compression, coda_cdf_variable **variable);
106	107
107	108	int coda_cdf_variable_add_attribute(coda_cdf_variable type, const char real_name, coda_dynamic_type *attribute_type,
108	109	int update_definition);

+2

-1

libcoda/coda-cdf-type.c less more

257	257	coda_dynamic_type *coda_cdf_variable_new(int32_t data_type, int32_t max_rec, int32_t rec_varys, int32_t num_dims,
258	258	int32_t dim[CODA_MAX_NUM_DIMS], int32_t dim_varys[CODA_MAX_NUM_DIMS],
259	259	coda_array_ordering array_ordering, int32_t num_elements,
260		int sparse_rec_method, coda_cdf_variable **variable)
	260	int sparse_rec_method, int has_compression, coda_cdf_variable **variable)
261	261	{
262	262	coda_cdf_variable *type;
263	263	int is_scalar = 0;

282	282	type->num_values_per_record = 1;
283	283	type->value_size = -1;
284	284	type->sparse_rec_method = sparse_rec_method;
	285	type->has_compression = has_compression;
285	286	type->offset = NULL;
286	287	type->data = NULL;
287	288

+42

-6

libcoda/coda-cdf.c less more

648	648	{
649	649	last = variable->num_records - 1;
650	650	}
651		for (i = first; i <= last; i++)
652		{
653		variable->offset[i] = offset + 12 + (i - first) * variable->num_values_per_record * variable->value_size;
	651	if (variable->has_compression)
	652	{
	653	if (variable->data == NULL)
	654	{
	655	variable->data = malloc(variable->num_records * variable->num_values_per_record * variable->value_size);
	656	if (variable->data == NULL)
	657	{
	658	coda_set_error(CODA_ERROR_OUT_OF_MEMORY, "out of memory (could not allocate %lu bytes) (%s:%u)",
	659	(long)variable->num_records * variable->num_values_per_record * variable->value_size,
	660	__FILE__, __LINE__);
	661	return -1;
	662	}
	663	}
	664	if (read_bytes(product_file->raw_product, offset + 12,
	665	(last - first + 1) * variable->num_values_per_record * variable->value_size,
	666	&variable->data[first * variable->num_values_per_record * variable->value_size]) < 0)
	667	{
	668	return -1;
	669	}
	670
	671	for (i = first; i <= last; i++)
	672	{
	673	variable->offset[i] = i * variable->num_values_per_record * variable->value_size;
	674	}
	675	}
	676	else
	677	{
	678	for (i = first; i <= last; i++)
	679	{
	680	variable->offset[i] = offset + 12 +
	681	(i - first) * variable->num_values_per_record * variable->value_size;
	682	}
654	683	}
655	684	}
656	685	else if (record_type == 13)

662	691	int result;
663	692	int i;
664	693
	694	if (!variable->has_compression)
	695	{
	696	coda_set_error(CODA_ERROR_FILE_READ, "unexpected CVVR record for uncompressed CDF variable");
	697	return -1;
	698	}
665	699	if (first >= variable->num_records)
666	700	{
667	701	/* completely skip this record */

691	725
692	726	if (csize < 20)
693	727	{
694		coda_set_error(CODA_ERROR_PRODUCT, "Invalid compressed data block for CDF variable");
	728	coda_set_error(CODA_ERROR_PRODUCT, "invalid compressed data block for CDF variable");
695	729	return -1;
696	730	}
697	731	buffer = malloc((size_t)csize);

1167	1201	if (is_zvar)
1168	1202	{
1169	1203	variable_type = coda_cdf_variable_new(data_type, max_rec, record_varys, num_dims, zdim_sizes, dim_varys,
1170		product_file->array_ordering, num_elems, srecords, &variable);
	1204	product_file->array_ordering, num_elems, srecords, has_compression,
	1205	&variable);
1171	1206	}
1172	1207	else
1173	1208	{
1174	1209	variable_type = coda_cdf_variable_new(data_type, max_rec, record_varys, num_dims, product_file->rdim_sizes,
1175		dim_varys, product_file->array_ordering, num_elems, srecords, &variable);
	1210	dim_varys, product_file->array_ordering, num_elems, srecords,
	1211	has_compression, &variable);
1176	1212	}
1177	1213	if (variable_type == NULL)
1178	1214	{

+86

-2

libcoda/coda-grib.c less more

168	168	grib2_interpretationOfListOfNumbers,
169	169	grib2_parameterCategory,
170	170	grib2_parameterNumber,
	171	grib2_constituentType,
171	172	grib2_typeOfGeneratingProcess,
172	173	grib2_backgroundProcess,
173	174	grib2_generatingProcessIdentifier,

1100	1101	coda_type_set_read_type(grib_type[grib2_parameterNumber], coda_native_type_uint8);
1101	1102	coda_type_set_bit_size(grib_type[grib2_parameterNumber], 8);
1102	1103	coda_type_set_description(grib_type[grib2_parameterNumber], "Parameter Number");
	1104
	1105	grib_type[grib2_constituentType] = (coda_type *)coda_type_number_new(coda_format_grib, coda_integer_class);
	1106	coda_type_number_set_endianness((coda_type_number *)grib_type[grib2_constituentType], endianness);
	1107	coda_type_set_read_type(grib_type[grib2_constituentType], coda_native_type_uint16);
	1108	coda_type_set_bit_size(grib_type[grib2_constituentType], 16);
	1109	coda_type_set_description(grib_type[grib2_constituentType], "Constituent Number");
1103	1110
1104	1111	grib_type[grib2_typeOfGeneratingProcess] = (coda_type *)coda_type_number_new(coda_format_grib, coda_integer_class);
1105	1112	coda_type_number_set_endianness((coda_type_number *)grib_type[grib2_typeOfGeneratingProcess], endianness);

1311	1318	field = coda_type_record_field_new("parameterNumber");
1312	1319	coda_type_record_field_set_type(field, grib_type[grib2_parameterNumber]);
1313	1320	coda_type_record_add_field((coda_type_record *)grib_type[grib2_data], field);
	1321	field = coda_type_record_field_new("constituentType");
	1322	coda_type_record_field_set_type(field, grib_type[grib2_constituentType]);
	1323	coda_type_record_add_field((coda_type_record *)grib_type[grib2_data], field);
	1324	coda_type_record_field_set_optional(field);
1314	1325	field = coda_type_record_field_new("typeOfGeneratingProcess");
1315	1326	coda_type_record_field_set_type(field, grib_type[grib2_typeOfGeneratingProcess]);
1316	1327	coda_type_record_add_field((coda_type_record *)grib_type[grib2_data], field);

2184	2195	int64_t coordinate_values_offset = -1;
2185	2196	uint8_t parameterCategory = 0;
2186	2197	uint8_t parameterNumber = 0;
	2198	int has_constituentType = 0;
	2199	uint16_t constituentType = 0;
2187	2200	uint8_t typeOfGeneratingProcess = 0;
2188	2201	uint8_t backgroundProcess = 0;
2189	2202	uint8_t generatingProcessIdentifier = 0;

2642	2655	file_offset += 4;
2643	2656
2644	2657	/* we could possibly add support for 41, 44, 45 and 48 in the future as well */
2645		if (productDefinitionTemplate <= 6 \|\| productDefinitionTemplate == 15 \|\| productDefinitionTemplate == 40 \|\|
2646		productDefinitionTemplate == 51)
	2658	if (productDefinitionTemplate <= 6 \|\| productDefinitionTemplate == 15 \|\| productDefinitionTemplate == 51)
2647	2659	{
2648	2660	if (read_bytes(product->raw_product, file_offset, 25, buffer) < 0)
2649	2661	{

2705	2717	file_offset += 25;
2706	2718	coordinate_values_offset = num_coordinate_values > 0 ? file_offset : -1;
2707	2719	}
	2720	else if (productDefinitionTemplate == 40)
	2721	{
	2722	if (read_bytes(product->raw_product, file_offset, 25, buffer) < 0)
	2723	{
	2724	return -1;
	2725	}
	2726	parameterCategory = buffer[0];
	2727	parameterNumber = buffer[1];
	2728	has_constituentType = 1;
	2729	constituentType = buffer[2] * 256 + buffer[3];
	2730	typeOfGeneratingProcess = buffer[4];
	2731	backgroundProcess = buffer[5];
	2732	generatingProcessIdentifier = buffer[6];
	2733	hoursAfterDataCutoff = buffer[7] * 256 + buffer[8];
	2734	minutesAfterDataCutoff = buffer[9];
	2735	indicatorOfUnitOfTimeRange = buffer[10];
	2736	forecastTime = (((uint32_t)buffer[11] * 256 + buffer[12]) * 256 + buffer[13]) * 256 + buffer[14];
	2737	typeOfFirstFixedSurface = buffer[15];
	2738	if (typeOfFirstFixedSurface != 255)
	2739	{
	2740	int8_t scaleFactor = ((int8_t *)buffer)[16];
	2741
	2742	firstFixedSurface = (((uint32_t)buffer[17] * 256 + buffer[18]) * 256 + buffer[19]) * 256 +
	2743	buffer[20];
	2744	while (scaleFactor < 0)
	2745	{
	2746	firstFixedSurface *= 10;
	2747	scaleFactor++;
	2748	}
	2749	while (scaleFactor > 0)
	2750	{
	2751	firstFixedSurface /= 10;
	2752	scaleFactor--;
	2753	}
	2754	}
	2755	else
	2756	{
	2757	firstFixedSurface = coda_NaN();
	2758	}
	2759	typeOfSecondFixedSurface = buffer[21];
	2760	if (typeOfSecondFixedSurface != 255)
	2761	{
	2762	int8_t scaleFactor = ((int8_t *)buffer)[22];
	2763
	2764	secondFixedSurface = (((uint32_t)buffer[23] * 256 + buffer[24]) * 256 + buffer[25]) * 256 +
	2765	buffer[26];
	2766	while (scaleFactor < 0)
	2767	{
	2768	secondFixedSurface *= 10;
	2769	scaleFactor++;
	2770	}
	2771	while (scaleFactor > 0)
	2772	{
	2773	secondFixedSurface /= 10;
	2774	scaleFactor--;
	2775	}
	2776	}
	2777	else
	2778	{
	2779	secondFixedSurface = coda_NaN();
	2780	}
	2781	file_offset += 25;
	2782	coordinate_values_offset = num_coordinate_values > 0 ? file_offset : -1;
	2783	}
2708	2784	else
2709	2785	{
2710	2786	coda_set_error(CODA_ERROR_PRODUCT, "unsupported Product Definition Template (%d)",

2852	2928	gtype = grib_type[grib2_parameterNumber];
2853	2929	type = (coda_dynamic_type )coda_mem_uint8_new((coda_type_number )gtype, NULL, cproduct, parameterNumber);
2854	2930	coda_mem_record_add_field(data, "parameterNumber", type, 0);
	2931
	2932	if (has_constituentType)
	2933	{
	2934	gtype = grib_type[grib2_constituentType];
	2935	type = (coda_dynamic_type )coda_mem_uint16_new((coda_type_number )gtype, NULL, cproduct,
	2936	constituentType);
	2937	coda_mem_record_add_field(data, "constituentType", type, 0);
	2938	}
2855	2939
2856	2940	gtype = grib_type[grib2_typeOfGeneratingProcess];
2857	2941	type = (coda_dynamic_type )coda_mem_uint8_new((coda_type_number )gtype, NULL, cproduct,