NetCDF  4.7.4
nc4hdf.c
Go to the documentation of this file.
1 /* Copyright 2018, University Corporation for Atmospheric
2  * Research. See the COPYRIGHT file for copying and redistribution
3  * conditions. */
17 #include "config.h"
18 #include "netcdf.h"
19 #include "nc4internal.h"
20 #include "ncdispatch.h"
21 #include "hdf5internal.h"
22 #include <math.h>
23 
24 #ifdef HAVE_INTTYPES_H
25 #define __STDC_FORMAT_MACROS
26 #include <inttypes.h>
27 #endif
28 
29 #define NC_HDF5_MAX_NAME 1024
39 static int
40 flag_atts_dirty(NCindex *attlist) {
41 
42  NC_ATT_INFO_T *att = NULL;
43  int i;
44 
45  if(attlist == NULL) {
46  return NC_NOERR;
47  }
48 
49  for(i=0;i<ncindexsize(attlist);i++) {
50  att = (NC_ATT_INFO_T*)ncindexith(attlist,i);
51  if(att == NULL) continue;
52  att->dirty = NC_TRUE;
53  }
54 
55  return NC_NOERR;
56 }
57 
74 int
75 rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
76 {
77  NC_VAR_INFO_T *var;
78  NC_GRP_INFO_T *child_grp;
79  int d, i;
80  int retval;
81 
82  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
83  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
84 
85  /* If there are any child groups, attach dimscale there, if needed. */
86  for (i = 0; i < ncindexsize(grp->children); i++)
87  {
88  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
89  assert(child_grp);
90  if ((retval = rec_reattach_scales(child_grp, dimid, dimscaleid)))
91  return retval;
92  }
93 
94  /* Find any vars that use this dimension id. */
95  for (i = 0; i < ncindexsize(grp->vars); i++)
96  {
97  NC_HDF5_VAR_INFO_T *hdf5_var;
98 
99  var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
100  assert(var && var->format_var_info);
101  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
102 
103  for (d = 0; d < var->ndims; d++)
104  {
105  if (var->dimids[d] == dimid && !var->dimscale)
106  {
107  LOG((2, "%s: attaching scale for dimid %d to var %s",
108  __func__, var->dimids[d], var->hdr.name));
109  if (var->created)
110  {
111  if (H5DSattach_scale(hdf5_var->hdf_datasetid,
112  dimscaleid, d) < 0)
113  return NC_EHDFERR;
114  var->dimscale_attached[d] = NC_TRUE;
115  }
116  }
117  }
118  }
119  return NC_NOERR;
120 }
121 
138 int
139 rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
140 {
141  NC_VAR_INFO_T *var;
142  NC_GRP_INFO_T *child_grp;
143  int d, i;
144  int retval;
145 
146  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
147  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
148 
149  /* If there are any child groups, detach dimscale there, if needed. */
150  for(i=0;i<ncindexsize(grp->children);i++) {
151  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
152  if(child_grp == NULL) continue;
153  if ((retval = rec_detach_scales(child_grp, dimid, dimscaleid)))
154  return retval;
155  }
156 
157  /* Find any vars that use this dimension id. */
158  for (i = 0; i < ncindexsize(grp->vars); i++)
159  {
160  NC_HDF5_VAR_INFO_T *hdf5_var;
161  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
162  assert(var && var->format_var_info);
163  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
164 
165  for (d = 0; d < var->ndims; d++)
166  {
167  if (var->dimids[d] == dimid && !var->dimscale)
168  {
169  LOG((2, "%s: detaching scale for dimid %d to var %s",
170  __func__, var->dimids[d], var->hdr.name));
171  if (var->created)
172  {
173  if (var->dimscale_attached && var->dimscale_attached[d])
174  {
175  if (H5DSdetach_scale(hdf5_var->hdf_datasetid,
176  dimscaleid, d) < 0)
177  return NC_EHDFERR;
178  var->dimscale_attached[d] = NC_FALSE;
179  }
180  }
181  }
182  }
183  }
184  return NC_NOERR;
185 }
186 
198 int
199 nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset)
200 {
201  NC_VAR_INFO_T *var;
202  NC_HDF5_VAR_INFO_T *hdf5_var;
203 
204  assert(grp && grp->format_grp_info && dataset);
205 
206  /* Find the requested varid. */
207  if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
208  return NC_ENOTVAR;
209  assert(var && var->hdr.id == varid && var->format_var_info);
210  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
211 
212  /* Open this dataset if necessary. */
213  if (!hdf5_var->hdf_datasetid)
214  {
215  NC_HDF5_GRP_INFO_T *hdf5_grp;
216  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
217 
218  if ((hdf5_var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid,
219  var->hdr.name, H5P_DEFAULT)) < 0)
220  return NC_ENOTVAR;
221  }
222 
223  *dataset = hdf5_var->hdf_datasetid;
224 
225  return NC_NOERR;
226 }
227 
239 int
240 nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp)
241 {
242  size_t size;
243  int retval;
244 
245  /* Find out how much space we need for this type's fill value. */
246  if (var->type_info->nc_type_class == NC_VLEN)
247  size = sizeof(nc_vlen_t);
248  else if (var->type_info->nc_type_class == NC_STRING)
249  size = sizeof(char *);
250  else
251  {
252  if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size)))
253  return retval;
254  }
255  assert(size);
256 
257  /* Allocate the space. */
258  if (!((*fillp) = calloc(1, size)))
259  return NC_ENOMEM;
260 
261  /* If the user has set a fill_value for this var, use, otherwise
262  * find the default fill value. */
263  if (var->fill_value)
264  {
265  LOG((4, "Found a fill value for var %s", var->hdr.name));
266  if (var->type_info->nc_type_class == NC_VLEN)
267  {
268  nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value), *fv_vlen = (nc_vlen_t *)(*fillp);
269  size_t basetypesize = 0;
270 
271  if((retval=nc4_get_typelen_mem(h5, var->type_info->u.v.base_nc_typeid, &basetypesize)))
272  return retval;
273 
274  fv_vlen->len = in_vlen->len;
275  if (!(fv_vlen->p = malloc(basetypesize * in_vlen->len)))
276  {
277  free(*fillp);
278  *fillp = NULL;
279  return NC_ENOMEM;
280  }
281  memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * basetypesize);
282  }
283  else if (var->type_info->nc_type_class == NC_STRING)
284  {
285  if (*(char **)var->fill_value)
286  if (!(**(char ***)fillp = strdup(*(char **)var->fill_value)))
287  {
288  free(*fillp);
289  *fillp = NULL;
290  return NC_ENOMEM;
291  }
292  }
293  else
294  memcpy((*fillp), var->fill_value, size);
295  }
296  else
297  {
298  if (nc4_get_default_fill_value(var->type_info, *fillp))
299  {
300  /* Note: release memory, but don't return error on failure */
301  free(*fillp);
302  *fillp = NULL;
303  }
304  }
305 
306  return NC_NOERR;
307 }
308 
326 int
327 nc4_get_hdf_typeid(NC_FILE_INFO_T *h5, nc_type xtype,
328  hid_t *hdf_typeid, int endianness)
329 {
330  NC_TYPE_INFO_T *type;
331  hid_t typeid = 0;
332  int retval = NC_NOERR;
333 
334  assert(hdf_typeid && h5);
335 
336  *hdf_typeid = -1;
337 
338  /* Determine an appropriate HDF5 datatype */
339  if (xtype == NC_NAT)
340  return NC_EBADTYPE;
341  else if (xtype == NC_CHAR || xtype == NC_STRING)
342  {
343  /* NC_CHAR & NC_STRING types create a new HDF5 datatype */
344  if (xtype == NC_CHAR)
345  {
346  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
347  return NC_EHDFERR;
348  if (H5Tset_strpad(typeid, H5T_STR_NULLTERM) < 0)
349  BAIL(NC_EVARMETA);
350  if(H5Tset_cset(typeid, H5T_CSET_ASCII) < 0)
351  BAIL(NC_EVARMETA);
352 
353  /* Take ownership of the newly created HDF5 datatype */
354  *hdf_typeid = typeid;
355  typeid = 0;
356  }
357  else
358  {
359  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
360  return NC_EHDFERR;
361  if (H5Tset_size(typeid, H5T_VARIABLE) < 0)
362  BAIL(NC_EVARMETA);
363  if(H5Tset_cset(typeid, H5T_CSET_UTF8) < 0)
364  BAIL(NC_EVARMETA);
365 
366  /* Take ownership of the newly created HDF5 datatype */
367  *hdf_typeid = typeid;
368  typeid = 0;
369  }
370  }
371  else
372  {
373  /* All other types use an existing HDF5 datatype */
374  switch (xtype)
375  {
376  case NC_BYTE: /* signed 1 byte integer */
377  if (endianness == NC_ENDIAN_LITTLE)
378  typeid = H5T_STD_I8LE;
379  else if (endianness == NC_ENDIAN_BIG)
380  typeid = H5T_STD_I8BE;
381  else
382  typeid = H5T_NATIVE_SCHAR;
383  break;
384 
385  case NC_SHORT: /* signed 2 byte integer */
386  if (endianness == NC_ENDIAN_LITTLE)
387  typeid = H5T_STD_I16LE;
388  else if (endianness == NC_ENDIAN_BIG)
389  typeid = H5T_STD_I16BE;
390  else
391  typeid = H5T_NATIVE_SHORT;
392  break;
393 
394  case NC_INT:
395  if (endianness == NC_ENDIAN_LITTLE)
396  typeid = H5T_STD_I32LE;
397  else if (endianness == NC_ENDIAN_BIG)
398  typeid = H5T_STD_I32BE;
399  else
400  typeid = H5T_NATIVE_INT;
401  break;
402 
403  case NC_UBYTE:
404  if (endianness == NC_ENDIAN_LITTLE)
405  typeid = H5T_STD_U8LE;
406  else if (endianness == NC_ENDIAN_BIG)
407  typeid = H5T_STD_U8BE;
408  else
409  typeid = H5T_NATIVE_UCHAR;
410  break;
411 
412  case NC_USHORT:
413  if (endianness == NC_ENDIAN_LITTLE)
414  typeid = H5T_STD_U16LE;
415  else if (endianness == NC_ENDIAN_BIG)
416  typeid = H5T_STD_U16BE;
417  else
418  typeid = H5T_NATIVE_USHORT;
419  break;
420 
421  case NC_UINT:
422  if (endianness == NC_ENDIAN_LITTLE)
423  typeid = H5T_STD_U32LE;
424  else if (endianness == NC_ENDIAN_BIG)
425  typeid = H5T_STD_U32BE;
426  else
427  typeid = H5T_NATIVE_UINT;
428  break;
429 
430  case NC_INT64:
431  if (endianness == NC_ENDIAN_LITTLE)
432  typeid = H5T_STD_I64LE;
433  else if (endianness == NC_ENDIAN_BIG)
434  typeid = H5T_STD_I64BE;
435  else
436  typeid = H5T_NATIVE_LLONG;
437  break;
438 
439  case NC_UINT64:
440  if (endianness == NC_ENDIAN_LITTLE)
441  typeid = H5T_STD_U64LE;
442  else if (endianness == NC_ENDIAN_BIG)
443  typeid = H5T_STD_U64BE;
444  else
445  typeid = H5T_NATIVE_ULLONG;
446  break;
447 
448  case NC_FLOAT:
449  if (endianness == NC_ENDIAN_LITTLE)
450  typeid = H5T_IEEE_F32LE;
451  else if (endianness == NC_ENDIAN_BIG)
452  typeid = H5T_IEEE_F32BE;
453  else
454  typeid = H5T_NATIVE_FLOAT;
455  break;
456 
457  case NC_DOUBLE:
458  if (endianness == NC_ENDIAN_LITTLE)
459  typeid = H5T_IEEE_F64LE;
460  else if (endianness == NC_ENDIAN_BIG)
461  typeid = H5T_IEEE_F64BE;
462  else
463  typeid = H5T_NATIVE_DOUBLE;
464  break;
465 
466  default:
467  /* Maybe this is a user defined type? */
468  if (nc4_find_type(h5, xtype, &type))
469  return NC_EBADTYPE;
470  if (!type)
471  return NC_EBADTYPE;
472  typeid = ((NC_HDF5_TYPE_INFO_T *)type->format_type_info)->hdf_typeid;
473  break;
474  }
475  assert(typeid);
476 
477  /* Copy the HDF5 datatype, so the function operates uniformly */
478  if ((*hdf_typeid = H5Tcopy(typeid)) < 0)
479  return NC_EHDFERR;
480  typeid = 0;
481  }
482  assert(*hdf_typeid != -1);
483 
484 exit:
485  if (typeid > 0 && H5Tclose(typeid) < 0)
486  BAIL2(NC_EHDFERR);
487  return retval;
488 }
489 
504 static int
505 put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att)
506 {
507  NC_HDF5_GRP_INFO_T *hdf5_grp;
508  hid_t datasetid = 0, locid;
509  hid_t attid = 0, spaceid = 0, file_typeid = 0;
510  hid_t existing_att_typeid = 0, existing_attid = 0, existing_spaceid = 0;
511  hsize_t dims[1]; /* netcdf attributes always 1-D. */
512  htri_t attr_exists;
513  void *data;
514  int phoney_data = 99;
515  int retval = NC_NOERR;
516 
517  assert(att->hdr.name && grp && grp->format_grp_info);
518  LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d "
519  "att->len %d", __func__, varid, att->hdr.id, att->hdr.name,
520  att->nc_typeid, att->len));
521 
522  /* Get HDF5-specific group info. */
523  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
524 
525  /* If the file is read-only, return an error. */
526  if (grp->nc4_info->no_write)
527  BAIL(NC_EPERM);
528 
529  /* Get the hid to attach the attribute to, or read it from. */
530  if (varid == NC_GLOBAL)
531  locid = hdf5_grp->hdf_grpid;
532  else
533  {
534  if ((retval = nc4_open_var_grp2(grp, varid, &datasetid)))
535  BAIL(retval);
536  locid = datasetid;
537  }
538 
539  /* Get the length ready, and find the HDF type we'll be
540  * writing. */
541  dims[0] = att->len;
542  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, att->nc_typeid,
543  &file_typeid, 0)))
544  BAIL(retval);
545 
546  /* Even if the length is zero, HDF5 won't let me write with a
547  * NULL pointer. So if the length of the att is zero, point to
548  * some phoney data (which won't be written anyway.)*/
549  if (!dims[0])
550  data = &phoney_data;
551  else if (att->data)
552  data = att->data;
553  else if (att->stdata)
554  data = att->stdata;
555  else
556  data = att->vldata;
557 
558  /* NC_CHAR types require some extra work. The space ID is set to
559  * scalar, and the type is told how long the string is. If it's
560  * really zero length, set the size to 1. (The fact that it's
561  * really zero will be marked by the NULL dataspace, but HDF5
562  * doesn't allow me to set the size of the type to zero.)*/
563  if (att->nc_typeid == NC_CHAR)
564  {
565  size_t string_size = dims[0];
566  if (!string_size)
567  {
568  string_size = 1;
569  if ((spaceid = H5Screate(H5S_NULL)) < 0)
570  BAIL(NC_EATTMETA);
571  }
572  else
573  {
574  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
575  BAIL(NC_EATTMETA);
576  }
577  if (H5Tset_size(file_typeid, string_size) < 0)
578  BAIL(NC_EATTMETA);
579  if (H5Tset_strpad(file_typeid, H5T_STR_NULLTERM) < 0)
580  BAIL(NC_EATTMETA);
581  }
582  else
583  {
584  if (!att->len)
585  {
586  if ((spaceid = H5Screate(H5S_NULL)) < 0)
587  BAIL(NC_EATTMETA);
588  }
589  else
590  {
591  if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0)
592  BAIL(NC_EATTMETA);
593  }
594  }
595 
596  /* Does the att exists already? */
597  if ((attr_exists = H5Aexists(locid, att->hdr.name)) < 0)
598  BAIL(NC_EHDFERR);
599  if (attr_exists)
600  {
601  hssize_t npoints;
602 
603  /* Open the attribute. */
604  if ((existing_attid = H5Aopen(locid, att->hdr.name, H5P_DEFAULT)) < 0)
605  BAIL(NC_EATTMETA);
606 
607  /* Find the type of the existing attribute. */
608  if ((existing_att_typeid = H5Aget_type(existing_attid)) < 0)
609  BAIL(NC_EATTMETA);
610 
611  /* How big is the attribute? */
612  if ((existing_spaceid = H5Aget_space(existing_attid)) < 0)
613  BAIL(NC_EATTMETA);
614  if ((npoints = H5Sget_simple_extent_npoints(existing_spaceid)) < 0)
615  BAIL(NC_EATTMETA);
616 
617  /* For text attributes the size is specified in the datatype
618  and it is enough to compare types using H5Tequal(). */
619  if (!H5Tequal(file_typeid, existing_att_typeid) ||
620  (att->nc_typeid != NC_CHAR && npoints != att->len))
621  {
622  /* The attribute exists but we cannot re-use it. */
623 
624  /* Delete the attribute. */
625  if (H5Adelete(locid, att->hdr.name) < 0)
626  BAIL(NC_EHDFERR);
627 
628  /* Re-create the attribute with the type and length
629  reflecting the new value (or values). */
630  if ((attid = H5Acreate(locid, att->hdr.name, file_typeid, spaceid,
631  H5P_DEFAULT)) < 0)
632  BAIL(NC_EATTMETA);
633 
634  /* Write the values, (even if length is zero). */
635  if (H5Awrite(attid, file_typeid, data) < 0)
636  BAIL(NC_EATTMETA);
637  }
638  else
639  {
640  /* The attribute exists and we can re-use it. */
641 
642  /* Write the values, re-using the existing attribute. */
643  if (H5Awrite(existing_attid, file_typeid, data) < 0)
644  BAIL(NC_EATTMETA);
645  }
646  }
647  else
648  {
649  /* The attribute does not exist yet. */
650 
651  /* Create the attribute. */
652  if ((attid = H5Acreate(locid, att->hdr.name, file_typeid, spaceid,
653  H5P_DEFAULT)) < 0)
654  BAIL(NC_EATTMETA);
655 
656  /* Write the values, (even if length is zero). */
657  if (H5Awrite(attid, file_typeid, data) < 0)
658  BAIL(NC_EATTMETA);
659  }
660 
661 exit:
662  if (file_typeid && H5Tclose(file_typeid))
663  BAIL2(NC_EHDFERR);
664  if (attid > 0 && H5Aclose(attid) < 0)
665  BAIL2(NC_EHDFERR);
666  if (existing_att_typeid && H5Tclose(existing_att_typeid))
667  BAIL2(NC_EHDFERR);
668  if (existing_attid > 0 && H5Aclose(existing_attid) < 0)
669  BAIL2(NC_EHDFERR);
670  if (spaceid > 0 && H5Sclose(spaceid) < 0)
671  BAIL2(NC_EHDFERR);
672  if (existing_spaceid > 0 && H5Sclose(existing_spaceid) < 0)
673  BAIL2(NC_EHDFERR);
674  return retval;
675 }
676 
688 static int
689 write_attlist(NCindex *attlist, int varid, NC_GRP_INFO_T *grp)
690 {
691  NC_ATT_INFO_T *att;
692  int retval;
693  int i;
694 
695  for(i = 0; i < ncindexsize(attlist); i++)
696  {
697  att = (NC_ATT_INFO_T *)ncindexith(attlist, i);
698  assert(att);
699  if (att->dirty)
700  {
701  LOG((4, "%s: writing att %s to varid %d", __func__, att->hdr.name, varid));
702  if ((retval = put_att_grpa(grp, varid, att)))
703  return retval;
704  att->dirty = NC_FALSE;
705  att->created = NC_TRUE;
706  }
707  }
708  return NC_NOERR;
709 }
710 
724 static int
725 write_coord_dimids(NC_VAR_INFO_T *var)
726 {
727  NC_HDF5_VAR_INFO_T *hdf5_var;
728  hsize_t coords_len[1];
729  hid_t c_spaceid = -1, c_attid = -1;
730  int retval = NC_NOERR;
731 
732  assert(var && var->format_var_info);
733 
734  /* Get HDF5-specific var info. */
735  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
736 
737  /* Set up space for attribute. */
738  coords_len[0] = var->ndims;
739  if ((c_spaceid = H5Screate_simple(1, coords_len, coords_len)) < 0)
740  BAIL(NC_EHDFERR);
741 
742  /* Create the attribute. */
743  if ((c_attid = H5Acreate(hdf5_var->hdf_datasetid, COORDINATES,
744  H5T_NATIVE_INT, c_spaceid, H5P_DEFAULT)) < 0)
745  BAIL(NC_EHDFERR);
746 
747  /* Write our attribute. */
748  if (H5Awrite(c_attid, H5T_NATIVE_INT, var->dimids) < 0)
749  BAIL(NC_EHDFERR);
750 
751 exit:
752  if (c_spaceid >= 0 && H5Sclose(c_spaceid) < 0)
753  BAIL2(NC_EHDFERR);
754  if (c_attid >= 0 && H5Aclose(c_attid) < 0)
755  BAIL2(NC_EHDFERR);
756  return retval;
757 }
758 
769 static int
770 write_netcdf4_dimid(hid_t datasetid, int dimid)
771 {
772  hid_t dimid_spaceid = -1, dimid_attid = -1;
773  htri_t attr_exists;
774  int retval = NC_NOERR;
775 
776  /* Create the space. */
777  if ((dimid_spaceid = H5Screate(H5S_SCALAR)) < 0)
778  BAIL(NC_EHDFERR);
779 
780  /* Does the attribute already exist? If so, don't try to create it. */
781  if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0)
782  BAIL(NC_EHDFERR);
783  if (attr_exists)
784  dimid_attid = H5Aopen_by_name(datasetid, ".", NC_DIMID_ATT_NAME,
785  H5P_DEFAULT, H5P_DEFAULT);
786  else
787  /* Create the attribute if needed. */
788  dimid_attid = H5Acreate(datasetid, NC_DIMID_ATT_NAME,
789  H5T_NATIVE_INT, dimid_spaceid, H5P_DEFAULT);
790  if (dimid_attid < 0)
791  BAIL(NC_EHDFERR);
792 
793 
794  /* Write it. */
795  LOG((4, "%s: writing secret dimid %d", __func__, dimid));
796  if (H5Awrite(dimid_attid, H5T_NATIVE_INT, &dimid) < 0)
797  BAIL(NC_EHDFERR);
798 
799 exit:
800  /* Close stuff*/
801  if (dimid_spaceid >= 0 && H5Sclose(dimid_spaceid) < 0)
802  BAIL2(NC_EHDFERR);
803  if (dimid_attid >= 0 && H5Aclose(dimid_attid) < 0)
804  BAIL2(NC_EHDFERR);
805 
806  return retval;
807 }
808 
823 static int
824 var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid)
825 {
826  NC_HDF5_GRP_INFO_T *hdf5_grp;
827  NC_HDF5_VAR_INFO_T *hdf5_var;
828  hid_t plistid = 0, access_plistid = 0, typeid = 0, spaceid = 0;
829  hsize_t chunksize[H5S_MAX_RANK], dimsize[H5S_MAX_RANK], maxdimsize[H5S_MAX_RANK];
830  int d;
831  void *fillp = NULL;
832  NC_DIM_INFO_T *dim = NULL;
833  char *name_to_use;
834  int retval;
835 
836  assert(grp && grp->format_grp_info && var && var->format_var_info);
837 
838  LOG((3, "%s:: name %s", __func__, var->hdr.name));
839 
840  /* Get HDF5-specific group and var info. */
841  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
842  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
843 
844  /* Scalar or not, we need a creation property list. */
845  if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
846  BAIL(NC_EHDFERR);
847  if ((access_plistid = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
848  BAIL(NC_EHDFERR);
849 
850  /* Turn off object tracking times in HDF5. */
851  if (H5Pset_obj_track_times(plistid, 0) < 0)
852  BAIL(NC_EHDFERR);
853 
854  /* Find the HDF5 type of the dataset. */
855  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &typeid,
856  var->type_info->endianness)))
857  BAIL(retval);
858 
859  /* Figure out what fill value to set, if any. */
860  if (var->no_fill)
861  {
862  /* Required to truly turn HDF5 fill values off */
863  if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0)
864  BAIL(NC_EHDFERR);
865  }
866  else
867  {
868  if ((retval = nc4_get_fill_value(grp->nc4_info, var, &fillp)))
869  BAIL(retval);
870 
871  /* If there is a fill value, set it. */
872  if (fillp)
873  {
874  if (var->type_info->nc_type_class == NC_STRING)
875  {
876  if (H5Pset_fill_value(plistid, typeid, fillp) < 0)
877  BAIL(NC_EHDFERR);
878  }
879  else
880  {
881  /* The fill value set in HDF5 must always be presented as
882  * a native type, even if the endianness for this dataset
883  * is non-native. HDF5 will translate the fill value to
884  * the target endiannesss. */
885  hid_t fill_typeid = 0;
886 
887  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &fill_typeid,
889  BAIL(retval);
890  if (H5Pset_fill_value(plistid, fill_typeid, fillp) < 0)
891  {
892  if (H5Tclose(fill_typeid) < 0)
893  BAIL(NC_EHDFERR);
894  BAIL(NC_EHDFERR);
895  }
896  if (H5Tclose(fill_typeid) < 0)
897  BAIL(NC_EHDFERR);
898  }
899  }
900  }
901 
902  /* If the user wants to shuffle the data, set that up now. */
903  if (var->shuffle) {
904  if (H5Pset_shuffle(plistid) < 0)
905  BAIL(NC_EHDFERR);
906  }
907 
908  /* If the user wants to compress the data, using either zlib
909  * (a.k.a deflate) or szip, or another filter, set that up now.
910  * Szip and zip can be turned on
911  * either directly with nc_def_var_szip/deflate(), or using
912  * nc_def_var_filter(). If the user
913  * has specified a filter, it will be applied here. */
914  if(var->filters != NULL) {
915  int j;
916  for(j=0;j<nclistlength(var->filters);j++) {
917  NC_FILTER_SPEC_HDF5* fi = (NC_FILTER_SPEC_HDF5*)nclistget(var->filters,j);
918  size_t nparams;
919  unsigned int* params;
920  nparams = fi->nparams;
921  params = fi->params;
922  if(fi->filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */
923  unsigned level;
924  if(nparams != 1)
925  BAIL(NC_EFILTER);
926  level = (int)params[0];
927  if(H5Pset_deflate(plistid, level) < 0)
928  BAIL(NC_EFILTER);
929  } else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */
930  int options_mask;
931  int bits_per_pixel;
932  if(nparams != 2)
933  BAIL(NC_EFILTER);
934  options_mask = (int)params[0];
935  bits_per_pixel = (int)params[1];
936  if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
937  BAIL(NC_EFILTER);
938  } else {
939  herr_t code = H5Pset_filter(plistid, (unsigned int)fi->filterid, H5Z_FLAG_MANDATORY, nparams, params);
940  if(code < 0) {
941  BAIL(NC_EFILTER);
942  }
943  }
944  }
945  }
946 
947  /* If the user wants to fletcher error correction, set that up now. */
948  if (var->fletcher32)
949  if (H5Pset_fletcher32(plistid) < 0)
950  BAIL(NC_EHDFERR);
951 
952  /* If ndims non-zero, get info for all dimensions. We look up the
953  dimids and get the len of each dimension. We need this to create
954  the space for the dataset. In netCDF a dimension length of zero
955  means an unlimited dimension. */
956  if (var->ndims)
957  {
958  int unlimdim = 0;
959 
960  /* Check to see if any unlimited dimensions are used in this var. */
961  for (d = 0; d < var->ndims; d++) {
962  dim = var->dim[d];
963  assert(dim && dim->hdr.id == var->dimids[d]);
964  if (dim->unlimited)
965  unlimdim++;
966  }
967 
968  /* If there are no unlimited dims, and no filters, and the user
969  * has not specified chunksizes, use contiguous variable for
970  * better performance. */
971  if (!var->shuffle && !var->fletcher32 && nclistlength(var->filters) == 0 &&
972  (var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim)
973  var->storage = NC_CONTIGUOUS;
974 
975  /* Gather current & maximum dimension sizes, along with chunk
976  * sizes. */
977  for (d = 0; d < var->ndims; d++)
978  {
979  dim = var->dim[d];
980  assert(dim && dim->hdr.id == var->dimids[d]);
981  dimsize[d] = dim->unlimited ? NC_HDF5_UNLIMITED_DIMSIZE : dim->len;
982  maxdimsize[d] = dim->unlimited ? H5S_UNLIMITED : (hsize_t)dim->len;
983  if (var->storage == NC_CHUNKED)
984  {
985  if (var->chunksizes[d])
986  chunksize[d] = var->chunksizes[d];
987  else
988  {
989  size_t type_size;
990  if (var->type_info->nc_type_class == NC_STRING)
991  type_size = sizeof(char *);
992  else
993  type_size = var->type_info->size;
994 
995  /* Unlimited dim always gets chunksize of 1. */
996  if (dim->unlimited)
997  chunksize[d] = 1;
998  else
999  chunksize[d] = pow((double)DEFAULT_CHUNK_SIZE/type_size,
1000  1/(double)(var->ndims - unlimdim));
1001 
1002  /* If the chunksize is greater than the dim
1003  * length, make it the dim length. */
1004  if (!dim->unlimited && chunksize[d] > dim->len)
1005  chunksize[d] = dim->len;
1006 
1007  /* Remember the computed chunksize */
1008  var->chunksizes[d] = chunksize[d];
1009  }
1010  }
1011  }
1012 
1013  /* Create the dataspace. */
1014  if ((spaceid = H5Screate_simple(var->ndims, dimsize, maxdimsize)) < 0)
1015  BAIL(NC_EHDFERR);
1016  }
1017  else
1018  {
1019  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1020  BAIL(NC_EHDFERR);
1021  }
1022 
1023  /* Set the var storage to contiguous, compact, or chunked. Don't
1024  * try to set chunking for scalar vars, they will default to
1025  * contiguous if not set to compact. */
1026  if (var->storage == NC_CONTIGUOUS)
1027  {
1028  if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
1029  BAIL(NC_EHDFERR);
1030  }
1031  else if (var->storage == NC_COMPACT)
1032  {
1033  if (H5Pset_layout(plistid, H5D_COMPACT) < 0)
1034  BAIL(NC_EHDFERR);
1035  }
1036  else if (var->ndims)
1037  {
1038  if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0)
1039  BAIL(NC_EHDFERR);
1040  }
1041 
1042  /* Turn on creation order tracking. */
1043  if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED|
1044  H5P_CRT_ORDER_INDEXED) < 0)
1045  BAIL(NC_EHDFERR);
1046 
1047  /* Set per-var chunk cache, for chunked datasets. */
1048  if (var->storage == NC_CHUNKED && var->chunk_cache_size)
1049  if (H5Pset_chunk_cache(access_plistid, var->chunk_cache_nelems,
1050  var->chunk_cache_size, var->chunk_cache_preemption) < 0)
1051  BAIL(NC_EHDFERR);
1052 
1053  /* At long last, create the dataset. */
1054  name_to_use = var->hdf5_name ? var->hdf5_name : var->hdr.name;
1055  LOG((4, "%s: about to H5Dcreate2 dataset %s of type 0x%x", __func__,
1056  name_to_use, typeid));
1057  if ((hdf5_var->hdf_datasetid = H5Dcreate2(hdf5_grp->hdf_grpid, name_to_use, typeid,
1058  spaceid, H5P_DEFAULT, plistid, access_plistid)) < 0)
1059  BAIL(NC_EHDFERR);
1060  var->created = NC_TRUE;
1061  var->is_new_var = NC_FALSE;
1062 
1063  /* Always write the hidden coordinates attribute, which lists the
1064  * dimids of this var. When present, this speeds opens. When no
1065  * present, dimscale matching is used. */
1066  if (var->ndims > 1)
1067  if ((retval = write_coord_dimids(var)))
1068  BAIL(retval);
1069 
1070  /* If this is a dimscale, mark it as such in the HDF5 file. Also
1071  * find the dimension info and store the dataset id of the dimscale
1072  * dataset. */
1073  if (var->dimscale)
1074  {
1075  if (H5DSset_scale(hdf5_var->hdf_datasetid, var->hdr.name) < 0)
1076  BAIL(NC_EHDFERR);
1077 
1078  /* If this is a multidimensional coordinate variable, write a
1079  * coordinates attribute. */
1080  /* if (var->ndims > 1) */
1081  /* if ((retval = write_coord_dimids(var))) */
1082  /* BAIL(retval); */
1083 
1084  /* If desired, write the netCDF dimid. */
1085  if (write_dimid)
1086  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid, var->dimids[0])))
1087  BAIL(retval);
1088  }
1089 
1090  /* Write attributes for this var. */
1091  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1092  BAIL(retval);
1093  var->attr_dirty = NC_FALSE;
1094 
1095 exit:
1096  if (typeid > 0 && H5Tclose(typeid) < 0)
1097  BAIL2(NC_EHDFERR);
1098  if (plistid > 0 && H5Pclose(plistid) < 0)
1099  BAIL2(NC_EHDFERR);
1100  if (access_plistid > 0 && H5Pclose(access_plistid) < 0)
1101  BAIL2(NC_EHDFERR);
1102  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1103  BAIL2(NC_EHDFERR);
1104  if (fillp)
1105  {
1106  if (var->type_info->nc_type_class == NC_VLEN)
1107  nc_free_vlen((nc_vlen_t *)fillp);
1108  else if (var->type_info->nc_type_class == NC_STRING && *(char **)fillp)
1109  free(*(char **)fillp);
1110  free(fillp);
1111  }
1112 
1113  return retval;
1114 }
1115 
1129 int
1130 nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
1131 {
1132  size_t chunk_size_bytes = 1;
1133  int d;
1134  int retval;
1135 
1136  /* Nothing to be done for contiguous or compact data. */
1137  if (var->storage != NC_CHUNKED)
1138  return NC_NOERR;
1139 
1140 #ifdef USE_PARALLEL4
1141  /* Don't set cache for files using parallel I/O. */
1142  if (grp->nc4_info->parallel)
1143  return NC_NOERR;
1144 #endif
1145 
1146  /* How many bytes in the chunk? */
1147  for (d = 0; d < var->ndims; d++)
1148  chunk_size_bytes *= var->chunksizes[d];
1149  if (var->type_info->size)
1150  chunk_size_bytes *= var->type_info->size;
1151  else
1152  chunk_size_bytes *= sizeof(char *);
1153 
1154  /* If the chunk cache is too small, and the user has not changed
1155  * the default value of the chunk cache size, then increase the
1156  * size of the cache. */
1157  if (var->chunk_cache_size == CHUNK_CACHE_SIZE)
1158  if (chunk_size_bytes > var->chunk_cache_size)
1159  {
1160  var->chunk_cache_size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
1161  if (var->chunk_cache_size > MAX_DEFAULT_CACHE_SIZE)
1162  var->chunk_cache_size = MAX_DEFAULT_CACHE_SIZE;
1163  if ((retval = nc4_reopen_dataset(grp, var)))
1164  return retval;
1165  }
1166 
1167  return NC_NOERR;
1168 }
1169 
1185 static int
1186 commit_type(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type)
1187 {
1188  NC_HDF5_GRP_INFO_T *hdf5_grp;
1189  NC_HDF5_TYPE_INFO_T *hdf5_type;
1190  hid_t base_hdf_typeid;
1191  int retval;
1192 
1193  assert(grp && grp->format_grp_info && type && type->format_type_info);
1194 
1195  /* Get HDF5-specific group and type info. */
1196  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1197  hdf5_type = (NC_HDF5_TYPE_INFO_T *)type->format_type_info;
1198 
1199  /* Did we already record this type? */
1200  if (type->committed)
1201  return NC_NOERR;
1202 
1203  /* Is this a compound type? */
1204  if (type->nc_type_class == NC_COMPOUND)
1205  {
1206  NC_FIELD_INFO_T *field;
1207  hid_t hdf_base_typeid, hdf_typeid;
1208  int i;
1209 
1210  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_COMPOUND, type->size)) < 0)
1211  return NC_EHDFERR;
1212  LOG((4, "creating compound type %s hdf_typeid 0x%x", type->hdr.name,
1213  hdf5_type->hdf_typeid));
1214 
1215  for(i=0;i<nclistlength(type->u.c.field);i++)
1216  {
1217  field = (NC_FIELD_INFO_T *)nclistget(type->u.c.field, i);
1218  assert(field);
1219  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, field->nc_typeid,
1220  &hdf_base_typeid, type->endianness)))
1221  return retval;
1222 
1223  /* If this is an array, create a special array type. */
1224  if (field->ndims)
1225  {
1226  int d;
1227  hsize_t dims[NC_MAX_VAR_DIMS];
1228 
1229  for (d = 0; d < field->ndims; d++)
1230  dims[d] = field->dim_size[d];
1231  if ((hdf_typeid = H5Tarray_create(hdf_base_typeid, field->ndims,
1232  dims, NULL)) < 0)
1233  {
1234  if (H5Tclose(hdf_base_typeid) < 0)
1235  return NC_EHDFERR;
1236  return NC_EHDFERR;
1237  }
1238  if (H5Tclose(hdf_base_typeid) < 0)
1239  return NC_EHDFERR;
1240  }
1241  else
1242  hdf_typeid = hdf_base_typeid;
1243  LOG((4, "inserting field %s offset %d hdf_typeid 0x%x", field->hdr.name,
1244  field->offset, hdf_typeid));
1245  if (H5Tinsert(hdf5_type->hdf_typeid, field->hdr.name, field->offset,
1246  hdf_typeid) < 0)
1247  return NC_EHDFERR;
1248  if (H5Tclose(hdf_typeid) < 0)
1249  return NC_EHDFERR;
1250  }
1251  }
1252  else if (type->nc_type_class == NC_VLEN)
1253  {
1254  /* Find the HDF typeid of the base type of this vlen. */
1255  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.v.base_nc_typeid,
1256  &base_hdf_typeid, type->endianness)))
1257  return retval;
1258 
1259  /* Create a vlen type. */
1260  if ((hdf5_type->hdf_typeid = H5Tvlen_create(base_hdf_typeid)) < 0)
1261  return NC_EHDFERR;
1262  }
1263  else if (type->nc_type_class == NC_OPAQUE)
1264  {
1265  /* Create the opaque type. */
1266  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_OPAQUE, type->size)) < 0)
1267  return NC_EHDFERR;
1268  }
1269  else if (type->nc_type_class == NC_ENUM)
1270  {
1271  NC_ENUM_MEMBER_INFO_T *enum_m;
1272  int i;
1273 
1274  if (nclistlength(type->u.e.enum_member) == 0)
1275  return NC_EINVAL;
1276 
1277  /* Find the HDF typeid of the base type of this enum. */
1278  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.e.base_nc_typeid,
1279  &base_hdf_typeid, type->endianness)))
1280  return retval;
1281 
1282  /* Create an enum type. */
1283  if ((hdf5_type->hdf_typeid = H5Tenum_create(base_hdf_typeid)) < 0)
1284  return NC_EHDFERR;
1285 
1286  /* Add all the members to the HDF5 type. */
1287  for(i=0;i<nclistlength(type->u.e.enum_member);i++) {
1288  enum_m = (NC_ENUM_MEMBER_INFO_T*)nclistget(type->u.e.enum_member,i);
1289  if (H5Tenum_insert(hdf5_type->hdf_typeid, enum_m->name, enum_m->value) < 0)
1290  return NC_EHDFERR;
1291  }
1292  }
1293  else
1294  {
1295  LOG((0, "Unknown class: %d", type->nc_type_class));
1296  return NC_EBADTYPE;
1297  }
1298 
1299  /* Commit the type. */
1300  if (H5Tcommit(hdf5_grp->hdf_grpid, type->hdr.name, hdf5_type->hdf_typeid) < 0)
1301  return NC_EHDFERR;
1302  type->committed = NC_TRUE;
1303  LOG((4, "just committed type %s, HDF typeid: 0x%x", type->hdr.name,
1304  hdf5_type->hdf_typeid));
1305 
1306  /* Later we will always use the native typeid. In this case, it is
1307  * a copy of the same type pointed to by hdf_typeid, but it's
1308  * easier to maintain a copy. */
1309  if ((hdf5_type->native_hdf_typeid = H5Tget_native_type(hdf5_type->hdf_typeid,
1310  H5T_DIR_DEFAULT)) < 0)
1311  return NC_EHDFERR;
1312 
1313  return NC_NOERR;
1314 }
1315 
1326 static int
1327 write_nc3_strict_att(hid_t hdf_grpid)
1328 {
1329  hid_t attid = 0, spaceid = 0;
1330  int one = 1;
1331  int retval = NC_NOERR;
1332  htri_t attr_exists;
1333 
1334  /* If the attribute already exists, call that a success and return
1335  * NC_NOERR. */
1336  if ((attr_exists = H5Aexists(hdf_grpid, NC3_STRICT_ATT_NAME)) < 0)
1337  return NC_EHDFERR;
1338  if (attr_exists)
1339  return NC_NOERR;
1340 
1341  /* Create the attribute to mark this as a file that needs to obey
1342  * strict netcdf-3 rules. */
1343  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1344  BAIL(NC_EFILEMETA);
1345  if ((attid = H5Acreate(hdf_grpid, NC3_STRICT_ATT_NAME,
1346  H5T_NATIVE_INT, spaceid, H5P_DEFAULT)) < 0)
1347  BAIL(NC_EFILEMETA);
1348  if (H5Awrite(attid, H5T_NATIVE_INT, &one) < 0)
1349  BAIL(NC_EFILEMETA);
1350 
1351 exit:
1352  if (spaceid > 0 && (H5Sclose(spaceid) < 0))
1353  BAIL2(NC_EFILEMETA);
1354  if (attid > 0 && (H5Aclose(attid) < 0))
1355  BAIL2(NC_EFILEMETA);
1356  return retval;
1357 }
1358 
1371 static int
1372 create_group(NC_GRP_INFO_T *grp)
1373 {
1374  NC_HDF5_GRP_INFO_T *hdf5_grp, *parent_hdf5_grp;
1375  hid_t gcpl_id = -1;
1376  int retval = NC_NOERR;;
1377 
1378  assert(grp && grp->format_grp_info && grp->parent &&
1379  grp->parent->format_grp_info);
1380 
1381  /* Get HDF5 specific group info for group and parent. */
1382  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1383  parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info;
1384  assert(parent_hdf5_grp->hdf_grpid);
1385 
1386  /* Create group, with link_creation_order set in the group
1387  * creation property list. */
1388  if ((gcpl_id = H5Pcreate(H5P_GROUP_CREATE)) < 0)
1389  BAIL(NC_EHDFERR);
1390 
1391  /* Set track_times to be FALSE. */
1392  if (H5Pset_obj_track_times(gcpl_id, 0) < 0)
1393  BAIL(NC_EHDFERR);
1394 
1395  /* Tell HDF5 to keep track of objects in creation order. */
1396  if (H5Pset_link_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1397  BAIL(NC_EHDFERR);
1398 
1399  /* Tell HDF5 to keep track of attributes in creation order. */
1400  if (H5Pset_attr_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1401  BAIL(NC_EHDFERR);
1402 
1403  /* Create the group. */
1404  if ((hdf5_grp->hdf_grpid = H5Gcreate2(parent_hdf5_grp->hdf_grpid, grp->hdr.name,
1405  H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0)
1406  BAIL(NC_EHDFERR);
1407 
1408 exit:
1409  if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0)
1410  BAIL2(NC_EHDFERR);
1411  if (retval)
1412  if (hdf5_grp->hdf_grpid > 0 && H5Gclose(hdf5_grp->hdf_grpid) < 0)
1413  BAIL2(NC_EHDFERR);
1414  return retval;
1415 }
1416 
1428 static int
1429 attach_dimscales(NC_GRP_INFO_T *grp)
1430 {
1431  NC_VAR_INFO_T *var;
1432  NC_HDF5_VAR_INFO_T *hdf5_var;
1433  int d, v;
1434 
1435  /* Attach dimension scales. */
1436  for (v = 0; v < ncindexsize(grp->vars); v++)
1437  {
1438  /* Get pointer to var and HDF5-specific var info. */
1439  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, v);
1440  assert(var && var->format_var_info);
1441  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1442 
1443  /* Scales themselves do not attach. But I really wish they
1444  * would. */
1445  if (var->dimscale)
1446  continue;
1447 
1448  /* Find the scale for each dimension, if any, and attach it. */
1449  for (d = 0; d < var->ndims; d++)
1450  {
1451  /* Is there a dimscale for this dimension? */
1452  if (var->dimscale_attached)
1453  {
1454  if (!var->dimscale_attached[d])
1455  {
1456  hid_t dsid; /* Dataset ID for dimension */
1457  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1458  var->dim[d]->format_dim_info);
1459 
1460  LOG((2, "%s: attaching scale for dimid %d to var %s",
1461  __func__, var->dimids[d], var->hdr.name));
1462 
1463  /* Find dataset ID for dimension */
1464  if (var->dim[d]->coord_var)
1465  dsid = ((NC_HDF5_VAR_INFO_T *)(var->dim[d]->coord_var->format_var_info))->hdf_datasetid;
1466  else
1467  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1468  assert(dsid > 0);
1469 
1470  /* Attach the scale. */
1471  if (H5DSattach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1472  return NC_EHDFERR;
1473  var->dimscale_attached[d] = NC_TRUE;
1474  }
1475  }
1476  }
1477  }
1478 
1479  return NC_NOERR;
1480 }
1481 
1492 static int
1493 var_exists(hid_t grpid, char *name, nc_bool_t *exists)
1494 {
1495  htri_t link_exists;
1496 
1497  /* Reset the boolean */
1498  *exists = NC_FALSE;
1499 
1500  /* Check if the object name exists in the group */
1501  if ((link_exists = H5Lexists(grpid, name, H5P_DEFAULT)) < 0)
1502  return NC_EHDFERR;
1503  if (link_exists)
1504  {
1505  H5G_stat_t statbuf;
1506 
1507  /* Get info about the object */
1508  if (H5Gget_objinfo(grpid, name, 1, &statbuf) < 0)
1509  return NC_EHDFERR;
1510 
1511  if (H5G_DATASET == statbuf.type)
1512  *exists = NC_TRUE;
1513  }
1514 
1515  return NC_NOERR;
1516 }
1517 
1533 static int
1534 remove_coord_atts(hid_t hdf_datasetid)
1535 {
1536  htri_t attr_exists;
1537 
1538  /* If the variable dataset has an optional NC_DIMID_ATT_NAME
1539  * attribute, delete it. */
1540  if ((attr_exists = H5Aexists(hdf_datasetid, NC_DIMID_ATT_NAME)) < 0)
1541  return NC_EHDFERR;
1542  if (attr_exists)
1543  {
1544  if (H5Adelete(hdf_datasetid, NC_DIMID_ATT_NAME) < 0)
1545  return NC_EHDFERR;
1546  }
1547 
1548  /* Remove the dimension scale 'CLASS' & 'NAME' attributes. */
1549  if ((attr_exists = H5Aexists(hdf_datasetid,
1550  HDF5_DIMSCALE_CLASS_ATT_NAME)) < 0)
1551  return NC_EHDFERR;
1552  if (attr_exists)
1553  {
1554  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_CLASS_ATT_NAME) < 0)
1555  return NC_EHDFERR;
1556  }
1557  if ((attr_exists = H5Aexists(hdf_datasetid,
1558  HDF5_DIMSCALE_NAME_ATT_NAME)) < 0)
1559  return NC_EHDFERR;
1560  if (attr_exists)
1561  {
1562  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_NAME_ATT_NAME) < 0)
1563  return NC_EHDFERR;
1564  }
1565  return NC_NOERR;
1566 }
1567 
1582 static int
1583 write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1584 {
1585  NC_HDF5_GRP_INFO_T *hdf5_grp;
1586  NC_HDF5_VAR_INFO_T *hdf5_var;
1587  nc_bool_t replace_existing_var = NC_FALSE;
1588  int retval;
1589 
1590  assert(var && var->format_var_info && grp && grp->format_grp_info);
1591 
1592  LOG((4, "%s: writing var %s", __func__, var->hdr.name));
1593 
1594  /* Get HDF5-specific group and var info. */
1595  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1596  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1597 
1598  /* If the variable has already been created & the fill value changed,
1599  * indicate that the existing variable should be replaced. */
1600  if (var->created && var->fill_val_changed)
1601  {
1602  replace_existing_var = NC_TRUE;
1603  var->fill_val_changed = NC_FALSE;
1604  /* If the variable is going to be replaced, we need to flag any
1605  other attributes associated with the variable as 'dirty', or
1606  else *only* the fill value attribute will be copied over and
1607  the rest will be lost. See
1608  https://github.com/Unidata/netcdf-c/issues/239 */
1609  flag_atts_dirty(var->att);
1610  }
1611 
1612  /* Is this a coordinate var that has already been created in
1613  * the HDF5 file as a dimscale dataset? Check for dims with the
1614  * same name in this group. If there is one, check to see if
1615  * this object exists in the HDF group. */
1616  if (var->became_coord_var)
1617  {
1618  if ((NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name))
1619  {
1620  nc_bool_t exists;
1621 
1622  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1623  return retval;
1624  if (exists)
1625  {
1626  /* Indicate that the variable already exists, and should
1627  * be replaced. */
1628  replace_existing_var = NC_TRUE;
1629  flag_atts_dirty(var->att);
1630  }
1631  }
1632  }
1633 
1634  /* Check dims if the variable will be replaced, so that the
1635  * dimensions will be de-attached and re-attached correctly. */
1636  if (replace_existing_var)
1637  {
1638  NC_DIM_INFO_T *d1;
1639 
1640  /* Is there a dim with this var's name? */
1641  if ((d1 = (NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name)))
1642  {
1643  nc_bool_t exists;
1644  assert(d1->format_dim_info && d1->hdr.name);
1645 
1646  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1647  return retval;
1648  if (exists)
1649  {
1650  hid_t dsid;
1651 
1652  /* Find dataset ID for dimension */
1653  if (d1->coord_var)
1654  dsid = ((NC_HDF5_VAR_INFO_T *)d1->coord_var->format_var_info)->hdf_datasetid;
1655  else
1656  dsid = ((NC_HDF5_DIM_INFO_T *)d1->format_dim_info)->hdf_dimscaleid;
1657  assert(dsid > 0);
1658 
1659  /* If we're replacing an existing dimscale dataset, go to
1660  * every var in the file and detach this dimension scale,
1661  * because we have to delete it. */
1662  if ((retval = rec_detach_scales(grp->nc4_info->root_grp,
1663  var->dimids[0], dsid)))
1664  return retval;
1665  }
1666  }
1667  }
1668 
1669  /* If this is not a dimension scale, remove any attached scales,
1670  * and delete dimscale attributes from the var. */
1671  if (var->was_coord_var && var->dimscale_attached)
1672  {
1673  int d;
1674 
1675  /* If the variable already exists in the file, Remove any dimension scale
1676  * attributes from it, if they exist. */
1677  if (var->created)
1678  if ((retval = remove_coord_atts(hdf5_var->hdf_datasetid)))
1679  return retval;
1680 
1681  /* If this is a regular var, detach all its dim scales. */
1682  for (d = 0; d < var->ndims; d++)
1683  {
1684  if (var->dimscale_attached[d])
1685  {
1686  hid_t dsid; /* Dataset ID for dimension */
1687  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1688  var->dim[d]->format_dim_info);
1689 
1690  /* Find dataset ID for dimension */
1691  if (var->dim[d]->coord_var)
1692  dsid = ((NC_HDF5_VAR_INFO_T *)var->dim[d]->coord_var->format_var_info)->hdf_datasetid;
1693  else
1694  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1695  assert(dsid > 0);
1696 
1697  /* Detach this dim scale. */
1698  if (H5DSdetach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1699  return NC_EHDFERR;
1700  var->dimscale_attached[d] = NC_FALSE;
1701  }
1702  }
1703  }
1704 
1705  /* Delete the HDF5 dataset that is to be replaced. */
1706  if (replace_existing_var)
1707  {
1708  /* Free the HDF5 dataset id. */
1709  if (hdf5_var->hdf_datasetid && H5Dclose(hdf5_var->hdf_datasetid) < 0)
1710  return NC_EHDFERR;
1711  hdf5_var->hdf_datasetid = 0;
1712 
1713  /* Now delete the variable. */
1714  if (H5Gunlink(hdf5_grp->hdf_grpid, var->hdr.name) < 0)
1715  return NC_EDIMMETA;
1716  }
1717 
1718  /* Create the dataset. */
1719  if (var->is_new_var || replace_existing_var)
1720  {
1721  if ((retval = var_create_dataset(grp, var, write_dimid)))
1722  return retval;
1723  }
1724  else
1725  {
1726  if (write_dimid && var->ndims)
1727  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid,
1728  var->dimids[0])))
1729  return retval;
1730  }
1731 
1732  if (replace_existing_var)
1733  {
1734  /* If this is a dimension scale, reattach the scale everywhere it
1735  * is used. (Recall that netCDF dimscales are always 1-D). */
1736  if(var->dimscale)
1737  {
1738  if ((retval = rec_reattach_scales(grp->nc4_info->root_grp,
1739  var->dimids[0], hdf5_var->hdf_datasetid)))
1740  return retval;
1741  }
1742  /* If it's not a dimension scale, clear the dimscale attached flags,
1743  * so the dimensions are re-attached. */
1744  else
1745  {
1746  if (var->dimscale_attached)
1747  memset(var->dimscale_attached, 0, sizeof(nc_bool_t) * var->ndims);
1748  }
1749  }
1750 
1751  /* Clear coord. var state transition flags */
1752  var->was_coord_var = NC_FALSE;
1753  var->became_coord_var = NC_FALSE;
1754 
1755  /* Now check the attributes for this var. */
1756  if (var->attr_dirty)
1757  {
1758  /* Write attributes for this var. */
1759  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1760  return retval;
1761  var->attr_dirty = NC_FALSE;
1762  }
1763 
1764  return NC_NOERR;
1765 }
1766 
1780 int
1781 nc4_create_dim_wo_var(NC_DIM_INFO_T *dim)
1782 {
1783  NC_HDF5_DIM_INFO_T *hdf5_dim;
1784  NC_HDF5_GRP_INFO_T *hdf5_grp;
1785  hid_t spaceid = -1, create_propid = -1;
1786  hsize_t dims[1], max_dims[1], chunk_dims[1] = {1};
1787  char dimscale_wo_var[NC_MAX_NAME];
1788  int retval = NC_NOERR;
1789 
1790  LOG((4, "%s: creating dim %s", __func__, dim->hdr.name));
1791 
1792  /* Sanity check */
1793  assert(!dim->coord_var);
1794 
1795  /* Get HDF5-specific dim and group info. */
1796  hdf5_grp = (NC_HDF5_GRP_INFO_T *)dim->container->format_grp_info;
1797  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1798 
1799  /* Create a property list. */
1800  if ((create_propid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
1801  BAIL(NC_EHDFERR);
1802 
1803  /* Turn off recording of times associated with this object. */
1804  if (H5Pset_obj_track_times(create_propid, 0) < 0)
1805  BAIL(NC_EHDFERR);
1806 
1807  /* Set size of dataset to size of dimension. */
1808  dims[0] = dim->len;
1809  max_dims[0] = dim->len;
1810 
1811  /* If this dimension scale is unlimited (i.e. it's an unlimited
1812  * dimension), then set up chunking, with a chunksize of 1. */
1813  if (dim->unlimited)
1814  {
1815  max_dims[0] = H5S_UNLIMITED;
1816  if (H5Pset_chunk(create_propid, 1, chunk_dims) < 0)
1817  BAIL(NC_EHDFERR);
1818  }
1819 
1820  /* Set up space. */
1821  if ((spaceid = H5Screate_simple(1, dims, max_dims)) < 0)
1822  BAIL(NC_EHDFERR);
1823 
1824  /* Turn on creation-order tracking. */
1825  if (H5Pset_attr_creation_order(create_propid, H5P_CRT_ORDER_TRACKED|
1826  H5P_CRT_ORDER_INDEXED) < 0)
1827  BAIL(NC_EHDFERR);
1828 
1829  /* Create the dataset that will be the dimension scale. */
1830  LOG((4, "%s: about to H5Dcreate1 a dimscale dataset %s", __func__,
1831  dim->hdr.name));
1832  if ((hdf5_dim->hdf_dimscaleid = H5Dcreate2(hdf5_grp->hdf_grpid, dim->hdr.name,
1833  H5T_IEEE_F32BE, spaceid,
1834  H5P_DEFAULT, create_propid,
1835  H5P_DEFAULT)) < 0)
1836  BAIL(NC_EHDFERR);
1837 
1838  /* Indicate that this is a scale. Also indicate that not
1839  * be shown to the user as a variable. It is hidden. It is
1840  * a DIM WITHOUT A VARIABLE! */
1841  sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len);
1842  if (H5DSset_scale(hdf5_dim->hdf_dimscaleid, dimscale_wo_var) < 0)
1843  BAIL(NC_EHDFERR);
1844 
1845  /* Since this dimension was created out of order, we cannot rely on
1846  * it getting the correct dimid on file open. We must assign it
1847  * explicitly. */
1848  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1849  BAIL(retval);
1850 
1851 exit:
1852  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1853  BAIL2(NC_EHDFERR);
1854  if (create_propid > 0 && H5Pclose(create_propid) < 0)
1855  BAIL2(NC_EHDFERR);
1856  return retval;
1857 }
1858 
1871 static int
1872 write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1873 {
1874  NC_HDF5_DIM_INFO_T *hdf5_dim;
1875  int retval = NC_NOERR;
1876 
1877  assert(dim && dim->format_dim_info && grp && grp->format_grp_info);
1878 
1879  /* Get HDF5-specific dim and group info. */
1880  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1881 
1882  /* If there's no dimscale dataset for this dim, create one,
1883  * and mark that it should be hidden from netCDF as a
1884  * variable. (That is, it should appear as a dimension
1885  * without an associated variable.) */
1886  if (!hdf5_dim->hdf_dimscaleid)
1887  if ((retval = nc4_create_dim_wo_var(dim)))
1888  BAIL(retval);
1889 
1890  /* Did we extend an unlimited dimension? */
1891  if (dim->extended)
1892  {
1893  NC_VAR_INFO_T *v1 = NULL;
1894 
1895  assert(dim->unlimited);
1896 
1897  /* If this is a dimension with an associated coordinate var,
1898  * then update the length of that coord var. */
1899  v1 = dim->coord_var;
1900  if (v1)
1901  {
1902  NC_HDF5_VAR_INFO_T *hdf5_v1;
1903  hsize_t *new_size;
1904  int d1;
1905 
1906  hdf5_v1 = (NC_HDF5_VAR_INFO_T *)v1->format_var_info;
1907 
1908  /* Extend the dimension scale dataset to reflect the new
1909  * length of the dimension. */
1910  if (!(new_size = malloc(v1->ndims * sizeof(hsize_t))))
1911  BAIL(NC_ENOMEM);
1912  for (d1 = 0; d1 < v1->ndims; d1++)
1913  {
1914  assert(v1->dim[d1] && v1->dim[d1]->hdr.id == v1->dimids[d1]);
1915  new_size[d1] = v1->dim[d1]->len;
1916  }
1917  if (H5Dset_extent(hdf5_v1->hdf_datasetid, new_size) < 0)
1918  BAIL(NC_EHDFERR);
1919  free(new_size);
1920  }
1921  }
1922 
1923  /* If desired, write the secret dimid. This will be used instead of
1924  * the dimid that the dimension would otherwise receive based on
1925  * creation order. This can be necessary when dims and their
1926  * coordinate variables were created in different order. */
1927  if (write_dimid && hdf5_dim->hdf_dimscaleid)
1928  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1929  BAIL(retval);
1930 
1931 exit:
1932 
1933  return retval;
1934 }
1935 
1948 int
1949 nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order)
1950 {
1951  NC_DIM_INFO_T *dim = NULL;
1952  NC_VAR_INFO_T *var = NULL;
1953  NC_GRP_INFO_T *child_grp = NULL;
1954  int coord_varid = -1;
1955  int var_index = 0;
1956  int dim_index = 0;
1957  int retval;
1958  int i;
1959 
1960  assert(grp && grp->hdr.name &&
1961  ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid);
1962  LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name,
1963  bad_coord_order));
1964 
1965  /* Write global attributes for this group. */
1966  if ((retval = write_attlist(grp->att, NC_GLOBAL, grp)))
1967  return retval;
1968 
1969  /* Set the pointers to the beginning of the list of dims & vars in this
1970  * group. */
1971  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, dim_index);
1972  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, var_index);
1973 
1974  /* Because of HDF5 ordering the dims and vars have to be stored in
1975  * this way to ensure that the dims and coordinate vars come out in
1976  * the correct order. */
1977  while (dim || var)
1978  {
1979  nc_bool_t found_coord, wrote_coord;
1980 
1981  /* Write non-coord dims in order, stopping at the first one that
1982  * has an associated coord var. */
1983  for (found_coord = NC_FALSE; dim && !found_coord; )
1984  {
1985  if (!dim->coord_var)
1986  {
1987  if ((retval = write_dim(dim, grp, bad_coord_order)))
1988  return retval;
1989  }
1990  else
1991  {
1992  coord_varid = dim->coord_var->hdr.id;
1993  found_coord = NC_TRUE;
1994  }
1995  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, ++dim_index);
1996  }
1997 
1998  /* Write each var. When we get to the coord var we are waiting
1999  * for (if any), then we break after writing it. */
2000  for (wrote_coord = NC_FALSE; var && !wrote_coord; )
2001  {
2002  if ((retval = write_var(var, grp, bad_coord_order)))
2003  return retval;
2004  if (found_coord && var->hdr.id == coord_varid)
2005  wrote_coord = NC_TRUE;
2006  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, ++var_index);
2007  }
2008  } /* end while */
2009 
2010  /* Attach dimscales to vars in this group. */
2011  if ((retval = attach_dimscales(grp)))
2012  return retval;
2013 
2014  /* If there are any child groups, write their metadata. */
2015  for (i = 0; i < ncindexsize(grp->children); i++)
2016  {
2017  child_grp = (NC_GRP_INFO_T *)ncindexith(grp->children, i);
2018  assert(child_grp);
2019  if ((retval = nc4_rec_write_metadata(child_grp, bad_coord_order)))
2020  return retval;
2021  }
2022  return NC_NOERR;
2023 }
2024 
2034 int
2035 nc4_rec_write_groups_types(NC_GRP_INFO_T *grp)
2036 {
2037  NC_GRP_INFO_T *child_grp;
2038  NC_HDF5_GRP_INFO_T *hdf5_grp;
2039  NC_TYPE_INFO_T *type;
2040  int retval;
2041  int i;
2042 
2043  assert(grp && grp->hdr.name && grp->format_grp_info);
2044  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2045 
2046  /* Get HDF5-specific group info. */
2047  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
2048 
2049  /* Create the group in the HDF5 file if it doesn't exist. */
2050  if (!hdf5_grp->hdf_grpid)
2051  if ((retval = create_group(grp)))
2052  return retval;
2053 
2054  /* If this is the root group of a file with strict NC3 rules, write
2055  * an attribute. But don't leave the attribute open. */
2056  if (!grp->parent && (grp->nc4_info->cmode & NC_CLASSIC_MODEL))
2057  if ((retval = write_nc3_strict_att(hdf5_grp->hdf_grpid)))
2058  return retval;
2059 
2060  /* If there are any user-defined types, write them now. */
2061  for(i=0;i<ncindexsize(grp->type);i++) {
2062  type = (NC_TYPE_INFO_T *)ncindexith(grp->type, i);
2063  assert(type);
2064  if ((retval = commit_type(grp, type)))
2065  return retval;
2066  }
2067 
2068  /* If there are any child groups, write their groups and types. */
2069  for(i=0;i<ncindexsize(grp->children);i++) {
2070  if((child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i)) == NULL) continue;
2071  if ((retval = nc4_rec_write_groups_types(child_grp)))
2072  return retval;
2073  }
2074  return NC_NOERR;
2075 }
2076 
2090 int
2091 nc4_rec_match_dimscales(NC_GRP_INFO_T *grp)
2092 {
2093  NC_GRP_INFO_T *g;
2094  NC_VAR_INFO_T *var;
2095  NC_DIM_INFO_T *dim;
2096  int retval = NC_NOERR;
2097  int i;
2098 
2099  assert(grp && grp->hdr.name);
2100  LOG((4, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2101 
2102  /* Perform var dimscale match for child groups. */
2103  for (i = 0; i < ncindexsize(grp->children); i++)
2104  {
2105  g = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
2106  assert(g);
2107  if ((retval = nc4_rec_match_dimscales(g)))
2108  return retval;
2109  }
2110 
2111  /* Check all the vars in this group. If they have dimscale info,
2112  * try and find a dimension for them. */
2113  for (i = 0; i < ncindexsize(grp->vars); i++)
2114  {
2115  NC_HDF5_VAR_INFO_T *hdf5_var;
2116  int ndims;
2117  int d;
2118 
2119  /* Get pointer to var and to the HDF5-specific var info. */
2120  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
2121  assert(var && var->format_var_info);
2122  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
2123 
2124  /* Check all vars and see if dim[i] != NULL if dimids[i] valid. */
2125  /* This loop is very odd. Under normal circumstances, var->dimid[d] is zero
2126  (from the initial calloc) which is a legitimate dimid. The code does not
2127  distinquish this case from the dimscale case where the id might actually
2128  be defined.
2129  The original nc4_find_dim searched up the group tree looking for the given
2130  dimid in one of the dim lists associated with each ancestor group.
2131  I changed nc4_fnd_dim to use the dimid directly using h5->alldims.
2132  However, here that is incorrect because it will find the dimid 0 always
2133  (if any dimensions were defined). Except that when dimscale dimids have
2134  been defined, one or more of the values in var->dimids will have a
2135  legitimate value.
2136  The solution I choose is to modify nc4_var_list_add to initialize dimids to
2137  illegal values (-1). This is another example of the problems with dimscales.
2138  */
2139  ndims = var->ndims;
2140  for (d = 0; d < ndims; d++)
2141  {
2142  if (var->dim[d] == NULL) {
2143  nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL);
2144  }
2145  /* assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d]); */
2146  }
2147 
2148  /* Skip dimension scale variables */
2149  if (!var->dimscale)
2150  {
2151  int d;
2152  int j;
2153 
2154  /* Are there dimscales for this variable? */
2155  if (hdf5_var->dimscale_hdf5_objids)
2156  {
2157  for (d = 0; d < var->ndims; d++)
2158  {
2159  nc_bool_t finished = NC_FALSE;
2160  LOG((5, "%s: var %s has dimscale info...", __func__, var->hdr.name));
2161 
2162  /* Check this and parent groups. */
2163  for (g = grp; g && !finished; g = g->parent)
2164  {
2165  /* Check all dims in this group. */
2166  for (j = 0; j < ncindexsize(g->dim); j++)
2167  {
2168  /* Get the HDF5 specific dim info. */
2169  NC_HDF5_DIM_INFO_T *hdf5_dim;
2170  dim = (NC_DIM_INFO_T *)ncindexith(g->dim, j);
2171  assert(dim && dim->format_dim_info);
2172  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
2173 
2174  /* Check for exact match of fileno/objid arrays
2175  * to find identical objects in HDF5 file. */
2176  if (hdf5_var->dimscale_hdf5_objids[d].fileno[0] == hdf5_dim->hdf5_objid.fileno[0] &&
2177  hdf5_var->dimscale_hdf5_objids[d].objno[0] == hdf5_dim->hdf5_objid.objno[0] &&
2178  hdf5_var->dimscale_hdf5_objids[d].fileno[1] == hdf5_dim->hdf5_objid.fileno[1] &&
2179  hdf5_var->dimscale_hdf5_objids[d].objno[1] == hdf5_dim->hdf5_objid.objno[1])
2180  {
2181  LOG((4, "%s: for dimension %d, found dim %s", __func__,
2182  d, dim->hdr.name));
2183  var->dimids[d] = dim->hdr.id;
2184  var->dim[d] = dim;
2185  finished = NC_TRUE;
2186  break;
2187  }
2188  } /* next dim */
2189  } /* next grp */
2190  LOG((5, "%s: dimid for this dimscale is %d", __func__,
2191  var->type_info->hdr.id));
2192  } /* next var->dim */
2193  }
2194  /* No dimscales for this var! Invent phony dimensions. */
2195  else
2196  {
2197  hid_t spaceid = 0;
2198  hsize_t *h5dimlen = NULL, *h5dimlenmax = NULL;
2199  int dataset_ndims;
2200 
2201  /* Find the space information for this dimension. */
2202  if ((spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0)
2203  return NC_EHDFERR;
2204 
2205  /* Get the len of each dim in the space. */
2206  if (var->ndims)
2207  {
2208  if (!(h5dimlen = malloc(var->ndims * sizeof(hsize_t))))
2209  return NC_ENOMEM;
2210  if (!(h5dimlenmax = malloc(var->ndims * sizeof(hsize_t))))
2211  {
2212  free(h5dimlen);
2213  return NC_ENOMEM;
2214  }
2215  if ((dataset_ndims = H5Sget_simple_extent_dims(spaceid, h5dimlen,
2216  h5dimlenmax)) < 0) {
2217  free(h5dimlenmax);
2218  free(h5dimlen);
2219  return NC_EHDFERR;
2220  }
2221  if (dataset_ndims != var->ndims) {
2222  free(h5dimlenmax);
2223  free(h5dimlen);
2224  return NC_EHDFERR;
2225  }
2226  }
2227  else
2228  {
2229  /* Make sure it's scalar. */
2230  if (H5Sget_simple_extent_type(spaceid) != H5S_SCALAR)
2231  return NC_EHDFERR;
2232  }
2233 
2234  /* Release the space object. */
2235  if (H5Sclose(spaceid) < 0) {
2236  free(h5dimlen);
2237  free(h5dimlenmax);
2238  return NC_EHDFERR;
2239  }
2240 
2241  /* Create a phony dimension for each dimension in the
2242  * dataset, unless there already is one the correct
2243  * size. */
2244  for (d = 0; d < var->ndims; d++)
2245  {
2246  int k;
2247  int match;
2248  /* Is there already a phony dimension of the correct size? */
2249  for(match=-1,k=0;k<ncindexsize(grp->dim);k++) {
2250  if((dim = (NC_DIM_INFO_T*)ncindexith(grp->dim,k)) == NULL) continue;
2251  if ((dim->len == h5dimlen[d]) &&
2252  ((h5dimlenmax[d] == H5S_UNLIMITED && dim->unlimited) ||
2253  (h5dimlenmax[d] != H5S_UNLIMITED && !dim->unlimited)))
2254  {match = k; break;}
2255  }
2256 
2257  /* Didn't find a phony dim? Then create one. */
2258  if (match < 0)
2259  {
2260  char phony_dim_name[NC_MAX_NAME + 1];
2261  sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid);
2262  LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name));
2263  if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim)))
2264  {
2265  free(h5dimlenmax);
2266  free(h5dimlen);
2267  return retval;
2268  }
2269  /* Create struct for HDF5-specific dim info. */
2270  if (!(dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T))))
2271  return NC_ENOMEM;
2272  if (h5dimlenmax[d] == H5S_UNLIMITED)
2273  dim->unlimited = NC_TRUE;
2274  }
2275 
2276  /* The variable must remember the dimid. */
2277  var->dimids[d] = dim->hdr.id;
2278  var->dim[d] = dim;
2279  } /* next dim */
2280 
2281  /* Free the memory we malloced. */
2282  free(h5dimlen);
2283  free(h5dimlenmax);
2284  }
2285  }
2286  }
2287 
2288  return retval;
2289 }
2290 
2303 int
2304 nc4_get_typeclass(const NC_FILE_INFO_T *h5, nc_type xtype, int *type_class)
2305 {
2306  int retval = NC_NOERR;
2307 
2308  LOG((4, "%s xtype: %d", __func__, xtype));
2309  assert(type_class);
2310 
2311  /* If this is an atomic type, the answer is easy. */
2312  if (xtype <= NC_STRING)
2313  {
2314  switch (xtype)
2315  {
2316  case NC_BYTE:
2317  case NC_UBYTE:
2318  case NC_SHORT:
2319  case NC_USHORT:
2320  case NC_INT:
2321  case NC_UINT:
2322  case NC_INT64:
2323  case NC_UINT64:
2324  /* NC_INT is class used for all integral types */
2325  *type_class = NC_INT;
2326  break;
2327 
2328  case NC_FLOAT:
2329  case NC_DOUBLE:
2330  /* NC_FLOAT is class used for all floating-point types */
2331  *type_class = NC_FLOAT;
2332  break;
2333 
2334  case NC_CHAR:
2335  *type_class = NC_CHAR;
2336  break;
2337 
2338  case NC_STRING:
2339  *type_class = NC_STRING;
2340  break;
2341 
2342  default:
2343  BAIL(NC_EBADTYPE);
2344  }
2345  }
2346  else
2347  {
2348  NC_TYPE_INFO_T *type;
2349 
2350  /* See if it's a used-defined type */
2351  if ((retval = nc4_find_type(h5, xtype, &type)))
2352  BAIL(retval);
2353  if (!type)
2354  BAIL(NC_EBADTYPE);
2355 
2356  *type_class = type->nc_type_class;
2357  }
2358 
2359 exit:
2360  return retval;
2361 }
2362 
2374 void
2375 reportobject(int uselog, hid_t id, unsigned int type)
2376 {
2377  char name[NC_HDF5_MAX_NAME];
2378  ssize_t len;
2379  const char* typename = NULL;
2380  long long printid = (long long)id;
2381 
2382  len = H5Iget_name(id, name, NC_HDF5_MAX_NAME);
2383  if(len < 0) return;
2384  name[len] = '\0';
2385 
2386  switch (type) {
2387  case H5F_OBJ_FILE: typename = "File"; break;
2388  case H5F_OBJ_DATASET: typename = "Dataset"; break;
2389  case H5F_OBJ_GROUP: typename = "Group"; break;
2390  case H5F_OBJ_DATATYPE: typename = "Datatype"; break;
2391  case H5F_OBJ_ATTR:
2392  typename = "Attribute";
2393  len = H5Aget_name(id, NC_HDF5_MAX_NAME, name);
2394  if(len < 0) len = 0;
2395  name[len] = '\0';
2396  break;
2397  default: typename = "<unknown>"; break;
2398  }
2399 #ifdef LOGGING
2400  if(uselog) {
2401  LOG((0,"Type = %s(%lld) name='%s'",typename,printid,name));
2402  } else
2403 #endif
2404  {
2405  fprintf(stderr,"Type = %s(%lld) name='%s'",typename,printid,name);
2406  }
2407 
2408 }
2409 
2420 static void
2421 reportopenobjectsT(int uselog, hid_t fid, int ntypes, unsigned int* otypes)
2422 {
2423  int t,i;
2424  ssize_t ocount;
2425  size_t maxobjs = -1;
2426  hid_t* idlist = NULL;
2427 
2428  /* Always report somehow */
2429 #ifdef LOGGING
2430  if(uselog)
2431  LOG((0,"\nReport: open objects on %lld",(long long)fid));
2432  else
2433 #endif
2434  fprintf(stdout,"\nReport: open objects on %lld\n",(long long)fid);
2435  maxobjs = H5Fget_obj_count(fid,H5F_OBJ_ALL);
2436  if(idlist != NULL) free(idlist);
2437  idlist = (hid_t*)malloc(sizeof(hid_t)*maxobjs);
2438  for(t=0;t<ntypes;t++) {
2439  unsigned int ot = otypes[t];
2440  ocount = H5Fget_obj_ids(fid,ot,maxobjs,idlist);
2441  for(i=0;i<ocount;i++) {
2442  hid_t o = idlist[i];
2443  reportobject(uselog,o,ot);
2444  }
2445  }
2446  if(idlist != NULL) free(idlist);
2447 }
2448 
2457 void
2458 reportopenobjects(int uselog, hid_t fid)
2459 {
2460  unsigned int OTYPES[5] = {H5F_OBJ_FILE, H5F_OBJ_DATASET, H5F_OBJ_GROUP,
2461  H5F_OBJ_DATATYPE, H5F_OBJ_ATTR};
2462 
2463  reportopenobjectsT(uselog, fid ,5, OTYPES);
2464 }
2465 
2473 void
2474 showopenobjects5(NC_FILE_INFO_T* h5)
2475 {
2476  NC_HDF5_FILE_INFO_T *hdf5_info;
2477 
2478  assert(h5 && h5->format_file_info);
2479  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2480 
2481  fprintf(stderr,"===== begin showopenobjects =====\n");
2482  reportopenobjects(0,hdf5_info->hdfid);
2483  fprintf(stderr,"===== end showopenobjects =====\n");
2484  fflush(stderr);
2485 }
2486 
2495 void
2496 showopenobjects(int ncid)
2497 {
2498  NC_FILE_INFO_T* h5 = NULL;
2499 
2500  /* Find our metadata for this file. */
2501  if (nc4_find_nc_grp_h5(ncid, NULL, NULL, &h5) != NC_NOERR)
2502  fprintf(stderr,"failed\n");
2503  else
2504  showopenobjects5(h5);
2505  fflush(stderr);
2506 }
2507 
2519 int
2520 NC4_hdf5get_libversion(unsigned* major,unsigned* minor,unsigned* release)
2521 {
2522  if(H5get_libversion(major,minor,release) < 0)
2523  return NC_EHDFERR;
2524  return NC_NOERR;
2525 }
2526 
2537 int
2538 NC4_hdf5get_superblock(struct NC_FILE_INFO* h5, int* idp)
2539 {
2540  NC_HDF5_FILE_INFO_T *hdf5_info;
2541  int stat = NC_NOERR;
2542  unsigned super;
2543  hid_t plist = -1;
2544 
2545  assert(h5 && h5->format_file_info);
2546  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2547 
2548  if((plist = H5Fget_create_plist(hdf5_info->hdfid)) < 0)
2549  {stat = NC_EHDFERR; goto done;}
2550  if(H5Pget_version(plist, &super, NULL, NULL, NULL) < 0)
2551  {stat = NC_EHDFERR; goto done;}
2552  if(idp) *idp = (int)super;
2553 done:
2554  if(plist >= 0) H5Pclose(plist);
2555  return stat;
2556 }
2557 
2558 static int NC4_strict_att_exists(NC_FILE_INFO_T*);
2559 static int NC4_walk(hid_t, int*);
2560 
2585 int
2586 NC4_isnetcdf4(struct NC_FILE_INFO* h5)
2587 {
2588  int stat;
2589  int isnc4 = 0;
2590  int exists;
2591  int count;
2592 
2593  /* Look for NC3_STRICT_ATT_NAME */
2594  exists = NC4_strict_att_exists(h5);
2595  if(exists)
2596  goto done;
2597  /* attribute did not exist */
2598  /* => last resort: walk the HDF5 file looking for markers */
2599  count = 0;
2600  stat = NC4_walk(((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid,
2601  &count);
2602  if(stat != NC_NOERR)
2603  isnc4 = 0;
2604  else /* Threshold is at least two matches */
2605  isnc4 = (count >= 2);
2606 
2607 done:
2608  return isnc4;
2609 }
2610 
2619 static int
2620 NC4_strict_att_exists(NC_FILE_INFO_T *h5)
2621 {
2622  hid_t grpid = -1;
2623  htri_t attr_exists;
2624 
2625  /* Get root group ID. */
2626  grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid;
2627 
2628  /* See if the NC3_STRICT_ATT_NAME attribute exists */
2629  if ((attr_exists = H5Aexists(grpid, NC3_STRICT_ATT_NAME)) < 0)
2630  return 1;
2631  return (attr_exists?1:0);
2632 }
2633 
2643 static int
2644 NC4_walk(hid_t gid, int* countp)
2645 {
2646  int ncstat = NC_NOERR;
2647  int i,j,na;
2648  ssize_t len;
2649  hsize_t nobj;
2650  herr_t err;
2651  int otype;
2652  hid_t grpid, dsid;
2653  char name[NC_HDF5_MAX_NAME];
2654 
2655  /* walk group members of interest */
2656  err = H5Gget_num_objs(gid, &nobj);
2657  if(err < 0) return err;
2658 
2659  for(i = 0; i < nobj; i++) {
2660  /* Get name & kind of object in the group */
2661  len = H5Gget_objname_by_idx(gid,(hsize_t)i,name,(size_t)NC_HDF5_MAX_NAME);
2662  if(len < 0) return len;
2663 
2664  otype = H5Gget_objtype_by_idx(gid,(size_t)i);
2665  switch(otype) {
2666  case H5G_GROUP:
2667  grpid = H5Gopen(gid,name);
2668  NC4_walk(grpid,countp);
2669  H5Gclose(grpid);
2670  break;
2671  case H5G_DATASET: /* variables */
2672  /* Check for phony_dim */
2673  if(strcmp(name,"phony_dim")==0)
2674  *countp = *countp + 1;
2675  dsid = H5Dopen(gid,name);
2676  na = H5Aget_num_attrs(dsid);
2677  for(j = 0; j < na; j++) {
2678  hid_t aid = H5Aopen_idx(dsid,(unsigned int) j);
2679  if(aid >= 0) {
2680  const NC_reservedatt* ra;
2681  ssize_t len = H5Aget_name(aid, NC_HDF5_MAX_NAME, name);
2682  if(len < 0) return len;
2683  /* Is this a netcdf-4 marker attribute */
2684  /* Is this a netcdf-4 marker attribute */
2685  ra = NC_findreserved(name);
2686  if(ra != NULL)
2687  *countp = *countp + 1;
2688  }
2689  H5Aclose(aid);
2690  }
2691  H5Dclose(dsid);
2692  break;
2693  default:/* ignore */
2694  break;
2695  }
2696  }
2697  return ncstat;
2698 }
2699 
2700 int
2701 NC4_hdf5_remove_filter(NC_VAR_INFO_T* var, unsigned int filterid)
2702 {
2703  int stat = NC_NOERR;
2704  NC_HDF5_VAR_INFO_T *hdf5_var;
2705  hid_t propid = -1;
2706  herr_t herr = -1;
2707  H5Z_filter_t hft;
2708 
2709  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
2710  if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0)
2711  {stat = NC_EHDFERR; goto done;}
2712 
2713  hft = filterid;
2714  if((herr = H5Premove_filter(propid,hft)) < 0)
2715  {stat = NC_EHDFERR; goto done;}
2716 done:
2717  return stat;
2718 }
NC_USHORT
#define NC_USHORT
unsigned 2-byte int
Definition: netcdf.h:43
NC_NOERR
#define NC_NOERR
No Error.
Definition: netcdf.h:329
NC_EINVAL
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:339
nc_vlen_t
This is the type of arrays of vlens.
Definition: netcdf.h:693
NC_FLOAT
#define NC_FLOAT
single precision floating point number
Definition: netcdf.h:40
NC_DOUBLE
#define NC_DOUBLE
double precision floating point number
Definition: netcdf.h:41
NC_ENDIAN_NATIVE
#define NC_ENDIAN_NATIVE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:288
NC_BYTE
#define NC_BYTE
signed 1 byte integer
Definition: netcdf.h:35
NC_VLEN
#define NC_VLEN
vlen (variable-length) types
Definition: netcdf.h:53
NC_ENDIAN_BIG
#define NC_ENDIAN_BIG
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:290
NC_CONTIGUOUS
#define NC_CONTIGUOUS
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:298
NC_EFILTER
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:474
NC_CHUNKED
#define NC_CHUNKED
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:297
NC_INT
#define NC_INT
signed 4 byte integer
Definition: netcdf.h:38
NC_MAX_NAME
#define NC_MAX_NAME
Maximum for classic library.
Definition: netcdf.h:275
NC_UBYTE
#define NC_UBYTE
unsigned 1 byte int
Definition: netcdf.h:42
NC_EHDFERR
#define NC_EHDFERR
Error at HDF5 layer.
Definition: netcdf.h:442
NC_GLOBAL
#define NC_GLOBAL
Attribute id to put/get a global attribute.
Definition: netcdf.h:248
NC_MAX_VAR_DIMS
#define NC_MAX_VAR_DIMS
max per variable dimensions
Definition: netcdf.h:276
nc_vlen_t::p
void * p
Pointer to VL data.
Definition: netcdf.h:695
NC_EPERM
#define NC_EPERM
Write to read only.
Definition: netcdf.h:340
NC_EVARMETA
#define NC_EVARMETA
Problem with variable metadata.
Definition: netcdf.h:449
NC_COMPACT
#define NC_COMPACT
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:299
NC_COMPOUND
#define NC_COMPOUND
compound types
Definition: netcdf.h:56
netcdf.h
NC_SHORT
#define NC_SHORT
signed 2 byte integer
Definition: netcdf.h:37
nc_vlen_t::len
size_t len
Length of VL data (in base type units)
Definition: netcdf.h:694
NC_OPAQUE
#define NC_OPAQUE
opaque types
Definition: netcdf.h:54
NC_ENDIAN_LITTLE
#define NC_ENDIAN_LITTLE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:289
nc_type
int nc_type
The nc_type type is just an int.
Definition: netcdf.h:25
NC_EBADTYPE
#define NC_EBADTYPE
Not a netcdf data type.
Definition: netcdf.h:371
NC_ENUM
#define NC_ENUM
enum types
Definition: netcdf.h:55
nc_free_vlen
EXTERNL int nc_free_vlen(nc_vlen_t *vl)
Definition: dvlen.c:41
NC_CLASSIC_MODEL
#define NC_CLASSIC_MODEL
Enforce classic model on netCDF-4.
Definition: netcdf.h:138
NC_ENOMEM
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:409
NC_NAT
#define NC_NAT
Not A Type.
Definition: netcdf.h:34
NC_EATTMETA
#define NC_EATTMETA
Problem with attribute metadata.
Definition: netcdf.h:448
NC_EDIMMETA
#define NC_EDIMMETA
Problem with dimension metadata.
Definition: netcdf.h:447
NC_UINT64
#define NC_UINT64
unsigned 8-byte int
Definition: netcdf.h:46
NC_UINT
#define NC_UINT
unsigned 4-byte int
Definition: netcdf.h:44
NC_STRING
#define NC_STRING
string
Definition: netcdf.h:47
NC_CHAR
#define NC_CHAR
ISO/ASCII character.
Definition: netcdf.h:36
NC_EFILEMETA
#define NC_EFILEMETA
Problem with file metadata.
Definition: netcdf.h:446
NC_ENOTVAR
#define NC_ENOTVAR
Variable not found.
Definition: netcdf.h:383
NC_INT64
#define NC_INT64
signed 8-byte int
Definition: netcdf.h:45

Return to the Main Unidata NetCDF page.
Generated on Tue Apr 21 2020 19:56:21 for NetCDF. NetCDF is a Unidata library.