bes  Updated for version 3.20.5
HDF5RequestHandler.cc
Go to the documentation of this file.
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of hdf5_handler, a data handler for the OPeNDAP data
5 // server.
6 
7 // Copyright (c) 2002,2003 OPeNDAP, Inc.
8 // Copyright (c) 2007-2016 The HDF Group, Inc. and OPeNDAP, Inc.
9 // Author: James Gallagher <jgallagher@opendap.org>
10 //
11 // This is free software; you can redistribute it and/or modify it under the
12 // terms of the GNU Lesser General Public License as published by the Free
13 // Software Foundation; either version 2.1 of the License, or (at your
14 // option) any later version.
15 //
16 // This software is distributed in the hope that it will be useful, but
17 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
19 // License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 
27 // HDF5RequestHandler.cc
33 
34 
35 #include <iostream>
36 #include <fstream>
37 #include <sstream>
38 #include <string>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <fcntl.h>
42 #include <unistd.h>
43 
44 #include <DMR.h>
45 #include <D4BaseTypeFactory.h>
46 #include <BESDMRResponse.h>
47 #include <ObjMemCache.h>
48 #include "HDF5_DMR.h"
49 
50 #include <mime_util.h>
51 #include "hdf5_handler.h"
52 #include "HDF5RequestHandler.h"
53 #include "HDF5_DDS.h"
54 
55 #include <BESDASResponse.h>
56 #include <BESDDSResponse.h>
57 #include <BESDataDDSResponse.h>
58 #include <Ancillary.h>
59 #include <BESInfo.h>
60 #include <BESDapNames.h>
61 #include <BESResponseNames.h>
62 #include <BESContainer.h>
63 #include <BESResponseHandler.h>
64 #include <BESVersionInfo.h>
65 #include <BESServiceRegistry.h>
66 #include <BESUtil.h>
67 #include <BESDapError.h>
68 #include <BESInternalFatalError.h>
69 #include <TheBESKeys.h>
70 #include <BESDebug.h>
71 #include "h5get.h"
72 #include "config_hdf5.h"
73 
74 #define HDF5_NAME "h5"
75 #include "h5cfdaputil.h"
76 
77 using namespace std;
78 using namespace libdap;
79 
80 // The debug function to dump all the contents of a DAS table.
81 void get_attr_contents(AttrTable* temp_table);
82 
83 // Five functions to generate a DAS cache file.
84 // 1. The wrapper function to call write_das_table_to_file to generate the cache.
85 void write_das_to_file(DAS*das_ptr,FILE* das_file);
86 
87 // 2. The main function to generate the DAS cache.
88 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
89 
90 // 3. The function to write the DAS container/table name to the cache
91 void write_container_name_to_file(const string&,FILE* das_file);
92 
93 // 4. The function to write the DAS attribute name,type and values to the cache
94 void write_das_attr_info(AttrTable* dtp,const string&,const string&,FILE * das_file);
95 
96 // 5. The function to copy a string to a memory buffer.
97 char* copy_str(char *temp_ptr,const string & str);
98 
99 // These two functions are for reading the DAS from the DAS cache file
100 // 1. Obtain the string from a memory buffer.
101 char* obtain_str(char*temp_ptr,string & str);
102 
103 // 2. The main function to obtain DAS info from the cache.
104 char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at);
105 
106 // Check if the BES key is set.
107 bool check_beskeys(const string);
108 
109 // Obtain the BES key as an integer
110 static unsigned int get_uint_key(const string &key,unsigned int def_val);
111 static unsigned long get_ulong_key(const string &key,unsigned long def_val);
112 
113 // Obtain the BES key as a floating-pointer number.
114 static float get_float_key(const string &key, float def_val);
115 
116 // Obtain the BES key as a string.
117 static string get_beskeys(const string&);
118 
119 // For the CF option
120 extern void read_cfdas(DAS &das, const string & filename,hid_t fileid);
121 extern void read_cfdds(DDS &dds, const string & filename,hid_t fileid);
122 
123 
124 // Check the description of cache_entries and cache_purge_level at h5.conf.in.
125 unsigned int HDF5RequestHandler::_mdcache_entries = 500;
126 unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
127 unsigned int HDF5RequestHandler::_srdcache_entries = 0;
128 float HDF5RequestHandler::_cache_purge_level = 0.2;
129 
130 // Metadata object cache at DAS,DDS and DMR.
131 ObjMemCache *HDF5RequestHandler::das_cache = 0;
132 ObjMemCache *HDF5RequestHandler::dds_cache = 0;
133 ObjMemCache *HDF5RequestHandler::dmr_cache = 0;
134 
135 ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = 0;
136 ObjMemCache *HDF5RequestHandler::srdata_mem_cache = 0;
137 
138 // Set default values of all BES keys be false.
139 bool HDF5RequestHandler::_usecf = false;
140 bool HDF5RequestHandler::_pass_fileid = false;
141 bool HDF5RequestHandler::_disable_structmeta = false;
142 bool HDF5RequestHandler::_disable_ecsmeta = false;
143 bool HDF5RequestHandler::_keep_var_leading_underscore = false;
144 bool HDF5RequestHandler::_check_name_clashing = false;
145 bool HDF5RequestHandler::_add_path_attrs = false;
146 bool HDF5RequestHandler::_drop_long_string = false;
147 bool HDF5RequestHandler::_fillvalue_check = false;
148 bool HDF5RequestHandler::_check_ignore_obj = false;
149 bool HDF5RequestHandler::_flatten_coor_attr = false;
150 bool HDF5RequestHandler::_default_handle_dimension = false;
151 
152 bool HDF5RequestHandler::_common_cache_dirs = false;
153 
154 bool HDF5RequestHandler::_use_disk_cache =false;
155 bool HDF5RequestHandler::_use_disk_dds_cache =false;
156 string HDF5RequestHandler::_disk_cache_dir ="";
157 string HDF5RequestHandler::_disk_cachefile_prefix ="";
158 unsigned long long HDF5RequestHandler::_disk_cache_size =0;
159 
160 
161 bool HDF5RequestHandler::_disk_cache_comp_data =false;
162 bool HDF5RequestHandler::_disk_cache_float_only_comp_data =false;
163 float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
164 unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
165 
166 bool HDF5RequestHandler::_use_disk_meta_cache = false;
167 string HDF5RequestHandler::_disk_meta_cache_path ="";
168 
169 bool HDF5RequestHandler::_use_latlon_disk_cache = false;
170 long HDF5RequestHandler::_latlon_disk_cache_size =0;
171 string HDF5RequestHandler::_latlon_disk_cache_dir ="";
172 string HDF5RequestHandler::_latlon_disk_cachefile_prefix="";
173 
174 DMR* HDF5RequestHandler::dmr_int64 = 0;
175 
176 
177 #if 0
178 //BaseTypeFactory factory;
179 //libdap::DDS HDF5RequestHandler::hd_dds(&factory,"");
180 #endif
181 string HDF5RequestHandler::_stp_east_filename;
182 string HDF5RequestHandler::_stp_north_filename;
183 vector<string> HDF5RequestHandler::lrd_cache_dir_list;
184 vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
185 vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
186 
187 #if 0
188 //libdap::DDS*cache_dds;
189 #endif
190 
191 
192 HDF5RequestHandler::HDF5RequestHandler(const string & name)
193  :BESRequestHandler(name)
194 {
195 
196  BESDEBUG(HDF5_NAME, "In HDF5RequestHandler::HDF5RequestHandler" << endl);
197 
198  add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
199  add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
200  add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
201  add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
202  add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
203 
204  add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
205  add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
206 
207  // Obtain the metadata cache entries and purge level.
208  HDF5RequestHandler::_mdcache_entries = get_uint_key("H5.MetaDataMemCacheEntries", 0);
209  HDF5RequestHandler::_lrdcache_entries = get_uint_key("H5.LargeDataMemCacheEntries", 0);
210  HDF5RequestHandler::_srdcache_entries = get_uint_key("H5.SmallDataMemCacheEntries", 0);
211  HDF5RequestHandler::_cache_purge_level = get_float_key("H5.CachePurgeLevel", 0.2);
212 
213  if (get_mdcache_entries()) { // else it stays at its default of null
214  das_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
215  dds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
216  dmr_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
217  }
218 
219  // Check if the EnableCF key is set.
220  _usecf = check_beskeys("H5.EnableCF");
221 
222  // The following keys are only effective when usecf is true.
223  _pass_fileid = check_beskeys("H5.EnablePassFileID");
224  _disable_structmeta = check_beskeys("H5.DisableStructMetaAttr");
225  _disable_ecsmeta = check_beskeys("H5.DisableECSMetaAttr");
226  _keep_var_leading_underscore = check_beskeys("H5.KeepVarLeadingUnderscore");
227  _check_name_clashing = check_beskeys("H5.EnableCheckNameClashing");
228  _add_path_attrs = check_beskeys("H5.EnableAddPathAttrs");
229  _drop_long_string = check_beskeys("H5.EnableDropLongString");
230  _fillvalue_check = check_beskeys("H5.EnableFillValueCheck");
231  _check_ignore_obj = check_beskeys("H5.CheckIgnoreObj");
232  _flatten_coor_attr = check_beskeys("H5.ForceFlattenNDCoorAttr");
233  _default_handle_dimension = check_beskeys("H5.DefaultHandleDimension");
234 
235  _use_disk_cache = check_beskeys("H5.EnableDiskDataCache");
236  _disk_cache_dir = get_beskeys("H5.DiskCacheDataPath");
237  _disk_cachefile_prefix = get_beskeys("H5.DiskCacheFilePrefix");
238  _disk_cache_size = get_ulong_key("H5.DiskCacheSize",0);
239 
240  _disk_cache_comp_data = check_beskeys("H5.DiskCacheComp");
241  _disk_cache_float_only_comp_data = check_beskeys("H5.DiskCacheFloatOnlyComp");
242  _disk_cache_comp_threshold = get_float_key("H5.DiskCacheCompThreshold",1.0);
243  _disk_cache_var_size = 1024*get_uint_key("H5.DiskCacheCompVarSize",0);
244 
245  _use_disk_meta_cache = check_beskeys("H5.EnableDiskMetaDataCache");
246  _use_disk_dds_cache = check_beskeys("H5.EnableDiskDDSCache");
247  _disk_meta_cache_path = get_beskeys("H5.DiskMetaDataCachePath");
248 
249  _use_latlon_disk_cache = check_beskeys("H5.EnableEOSGeoCacheFile");
250  _latlon_disk_cache_size = get_uint_key("H5.Cache.latlon.size",0);
251  _latlon_disk_cache_dir = get_beskeys("H5.Cache.latlon.path");
252  _latlon_disk_cachefile_prefix= get_beskeys("H5.Cache.latlon.prefix");
253 
254  if(get_usecf()) {
255  if(get_lrdcache_entries()) {
256  lrdata_mem_cache = new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
257  if(true == check_beskeys("H5.LargeDataMemCacheConfig")) {
258  _common_cache_dirs =obtain_lrd_common_cache_dirs();
259 #if 0
260 if(false == _common_cache_dirs)
261 cerr<<"No specific cache info"<<endl;
262 #endif
263 
264  }
265  }
266  if(get_srdcache_entries()) {
267 
268  BESDEBUG(HDF5_NAME, "Generate memory cache for smaller coordinate variables" << endl);
269  srdata_mem_cache = new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
270 
271  }
272 
273  if(_disk_cache_comp_data == true && _use_disk_cache == true) {
274  if(_disk_cache_comp_threshold < 1.0) {
275  ostringstream ss;
276  ss<< _disk_cache_comp_threshold;
277  string _comp_threshold_str(ss.str());
278  string invalid_comp_threshold ="The Compression Threshold is the total size of the variable array";
279  invalid_comp_threshold+=" divided by the storage size of compressed array. It should always be >1";
280  invalid_comp_threshold+=" The current threhold set at h5.conf is ";
281  invalid_comp_threshold+=_comp_threshold_str;
282  invalid_comp_threshold+=" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
283  throw BESInternalError(invalid_comp_threshold,__FILE__,__LINE__);
284  }
285  }
286  _stp_east_filename = get_beskeys("H5.STPEastFileName");
287  _stp_north_filename = get_beskeys("H5.STPNorthFileName");
288  }
289 
290 
291  BESDEBUG(HDF5_NAME, "Exiting HDF5RequestHandler::HDF5RequestHandler" << endl);
292 }
293 
294 HDF5RequestHandler::~HDF5RequestHandler()
295 {
296  // delete the cache.
297  delete das_cache;
298  delete dds_cache;
299  delete dmr_cache;
300  delete lrdata_mem_cache;
301  delete srdata_mem_cache;
302 
303 }
304 
305 // Build DAS
306 bool HDF5RequestHandler::hdf5_build_das(BESDataHandlerInterface & dhi)
307 {
308 
309  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
310  hid_t cf_fileid = -1;
311 
312  // Obtain the HDF5 file name.
313  string filename = dhi.container->access();
314 
315  // Obtain the BES object from the client
316  BESResponseObject *response = dhi.response_handler->get_response_object() ;
317 
318  // Convert to the BES DAS response
319  BESDASResponse *bdas = dynamic_cast < BESDASResponse * >(response) ;
320  if( !bdas )
321  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
322 
323  try {
324  bdas->set_container( dhi.container->get_symbolic_name() ) ;
325  DAS *das = bdas->get_das();
326 
327  // Look inside the memory cache to see if it's initialized
328  DAS *cached_das_ptr = 0;
329  bool use_das_cache = false;
330  if (das_cache)
331  cached_das_ptr = static_cast<DAS*>(das_cache->get(filename));
332  if (cached_das_ptr)
333  use_das_cache = true;
334 
335  if (true == use_das_cache) {
336 
337  // copy the cached DAS into the BES response object
338  BESDEBUG(HDF5_NAME, "DAS Cached hit for : " << filename << endl);
339  *das = *cached_das_ptr;
340  }
341  else {
342 
343  bool das_from_dc = false;
344  string das_cache_fname;
345 
346  // If the use_disk_meta_cache is set, check if the cache file exists and sets the flag.
347  if(_use_disk_meta_cache == true) {
348 
349  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
350  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
351 
352  if(access(das_cache_fname.c_str(),F_OK) !=-1)
353  das_from_dc = true;
354 
355  }
356 
357  // If reading DAS from the disk cache, call the corresponding function
358  if(true == das_from_dc) {
359  read_das_from_disk_cache(das_cache_fname,das);
360 
361  // If the memory cache is set, adding the DAS copy to the memory cache
362  if (das_cache) {
363  // add a copy
364  BESDEBUG(HDF5_NAME, "HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
365  das_cache->add(new DAS(*das), filename);
366  }
367  }
368 
369  else {// Need to build from the HDF5 file
370  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
371  if (true == _usecf) {//CF option
372 
373  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
374  if (cf_fileid < 0){
375  string invalid_file_msg="Could not open this HDF5 file ";
376  invalid_file_msg +=filename;
377  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
378  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
379  invalid_file_msg +=" distributor.";
380  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
381  }
382  // Need to check if DAP4 DMR CF 64-bit integer mapping is on.
383  if(HDF5RequestHandler::get_dmr_64bit_int()!=NULL)
384  HDF5RequestHandler::set_dmr_64bit_int(NULL);
385  read_cfdas( *das,filename,cf_fileid);
386  H5Fclose(cf_fileid);
387  }
388  else {// Default option
389  hid_t fileid = get_fileid(filename.c_str());
390  if (fileid < 0) {
391  string invalid_file_msg="Could not open this HDF5 file ";
392  invalid_file_msg +=filename;
393  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
394  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
395  invalid_file_msg +=" distributor.";
396  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
397 
398  }
399 
400  find_gloattr(fileid, *das);
401  depth_first(fileid, "/", *das);
402  close_fileid(fileid);
403  }
404 
405  Ancillary::read_ancillary_das( *das, filename ) ;
406 
407 #if 0
408 // Dump all attribute contents
409 AttrTable* top_table = das->get_top_level_attributes();
410 get_attr_contents(top_table);
411 
412 // Dump all variable contents
413 AttrTable::Attr_iter start_aiter=das->var_begin();
414 AttrTable::Attr_iter it = start_aiter;
415 AttrTable::Attr_iter end_aiter = das->var_end();
416 while(it != end_aiter) {
417 AttrTable* temp_table = das->get_table(it);
418 if(temp_table!=0){
419 cerr<<"var_begin"<<endl;
420 temp_table->print(cerr);
421 }
422 ++it;
423 }
424 #endif
425  // If the memory cache is turned on
426  if(das_cache) {
427  // add a copy
428  BESDEBUG(HDF5_NAME, "DAS added to the cache for : " << filename << endl);
429  das_cache->add(new DAS(*das), filename);
430  }
431 
432  // DAS disk cache fname will be set only when the metadata disk cache is turned on
433  // So if it comes here, the das cache should be generated.
434  if(das_cache_fname!="") {
435  BESDEBUG(HDF5_NAME, "HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
436  write_das_to_disk_cache(das_cache_fname,das);
437  }
438  }
439  }
440 
441  bdas->clear_container() ;
442  }
443  catch(InternalErr & e) {
444 
445  if(cf_fileid !=-1)
446  H5Fclose(cf_fileid);
447  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
448  __FILE__, __LINE__);
449  }
450  catch(Error & e) {
451 
452  if(cf_fileid !=-1)
453  H5Fclose(cf_fileid);
454  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
455  __FILE__, __LINE__);
456  }
457  catch(...) {
458 
459  if(cf_fileid !=-1)
460  H5Fclose(cf_fileid);
461  string s = "unknown exception caught building HDF5 DAS";
462  throw BESInternalFatalError(s, __FILE__, __LINE__);
463  }
464 
465  return true;
466 }
467 
468 // Convenient function that helps build DDS and Data
469 // Since this function will be used by both the DDS and Data services, we need to pass both BESDDSResponse and BESDataDDSResponse.
470 // This two parameters are necessary for the future DDS disk cache feature.
471 void HDF5RequestHandler::get_dds_with_attributes( BESDDSResponse*bdds,BESDataDDSResponse*data_bdds,const string &container_name, const string& filename,const string &dds_cache_fname, const string &das_cache_fname,bool dds_from_dc,bool das_from_dc, bool build_data)
472 {
473  DDS *dds;
474  if(true == build_data)
475  dds = data_bdds->get_dds();
476  else dds = bdds->get_dds();
477 
478  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
479  hid_t fileid = -1;
480  hid_t cf_fileid = -1;
481 
482  try {
483 
484  // Look in memory cache to see if it's initialized
485  DDS* cached_dds_ptr = 0;
486  bool use_dds_cache = false;
487  if (dds_cache)
488  cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename));
489  if (cached_dds_ptr)
490  use_dds_cache = true;
491  if (true == use_dds_cache) {
492  // copy the cached DDS into the BES response object. Assume that any cached DDS
493  // includes the DAS information.
494  BESDEBUG(HDF5_NAME, "DDS Metadata Cached hit for : " << filename << endl);
495  *dds = *cached_dds_ptr; // Copy the referenced object
496  }
497  else if (true ==dds_from_dc) {//Currently the dds_from_ds is always false by default.
498  read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
499  }
500  else {
501  BESDEBUG(HDF5_NAME, "Build DDS from the HDF5 file. " << filename << endl);
502  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
503  dds->filename(filename);
504 
505  // For the time being, not mess up CF's fileID with Default's fileID
506  if(true == _usecf) {
507 
508  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
509  if (cf_fileid < 0){
510  string invalid_file_msg="Could not open this HDF5 file ";
511  invalid_file_msg +=filename;
512  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
513  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
514  invalid_file_msg +=" distributor.";
515  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
516  }
517  // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
518  // to let the handler map the 64-bit integer.
519  if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
520  HDF5RequestHandler::set_dmr_64bit_int(NULL);
521  read_cfdds(*dds,filename,cf_fileid);
522  }
523  else {
524 
525  fileid = get_fileid(filename.c_str());
526  if (fileid < 0) {
527  string invalid_file_msg="Could not open this HDF5 file ";
528  invalid_file_msg +=filename;
529  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
530  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
531  invalid_file_msg +=" distributor.";
532  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
533  }
534 
535  depth_first(fileid, (char*)"/", *dds, filename.c_str());
536 
537  }
538  // Check semantics
539  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
540  dds->print(cerr);
541  throw InternalErr(__FILE__, __LINE__,
542  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
543  }
544 
545  Ancillary::read_ancillary_dds( *dds, filename ) ;
546 
547  // Generate the DDS cached file if needed,currently this if is always false by default
548  if(dds_cache_fname!="" && dds_from_dc == false)
549  write_dds_to_disk_cache(dds_cache_fname,dds);
550 
551  // Add attributes
552  {
553  hid_t h5_fd = -1;
554  if(_usecf == true)
555  h5_fd = cf_fileid;
556  else
557  h5_fd = fileid;
558  add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
559  }
560 
561  // Add memory cache if possible
562  if (dds_cache) {
563  // add a copy
564  BESDEBUG(HDF5_NAME, "DDS added to the cache for : " << filename << endl);
565  dds_cache->add(new DDS(*dds), filename);
566  }
567 
568  if(cf_fileid != -1)
569  H5Fclose(cf_fileid);
570  if(fileid != -1)
571  H5Fclose(fileid);
572 
573  }
574 
575  }
576  catch(InternalErr & e) {
577 
578  if(cf_fileid !=-1)
579  H5Fclose(cf_fileid);
580 
581  if(fileid != -1)
582  H5Fclose(fileid);
583 
584  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
585  __FILE__, __LINE__);
586  }
587  catch(Error & e) {
588 
589  if(cf_fileid !=-1)
590  H5Fclose(cf_fileid);
591  if(fileid !=-1)
592  H5Fclose(fileid);
593 
594  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
595  __FILE__, __LINE__);
596  }
597  catch(...) {
598 
599  if(cf_fileid !=-1)
600  H5Fclose(cf_fileid);
601  if(fileid !=-1)
602  H5Fclose(fileid);
603 
604  string s = "unknown exception caught building HDF5 DDS";
605  throw BESInternalFatalError(s, __FILE__, __LINE__);
606  }
607 
608 }
609 
610 #if 0
611 // OLD function: Keep it for a while
612 // Convenient function that helps build DDS and Data
613 void HDF5RequestHandler::get_dds_with_attributes(const string &filename, const string &container_name, DDS*dds) {
614 
615  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
616  hid_t fileid = -1;
617  hid_t cf_fileid = -1;
618 
619  try {
620 
621  // Look in memory cache to see if it's initialized
622  DDS* cached_dds_ptr = 0;
623  if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename)))) {
624  // copy the cached DDS into the BES response object. Assume that any cached DDS
625  // includes the DAS information.
626  BESDEBUG(HDF5_NAME, "DDS Cached hit for : " << filename << endl);
627  *dds = *cached_dds_ptr; // Copy the referenced object
628  }
629 
630  else {
631 
632  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
633  if (!container_name.empty())
634  dds->container_name(container_name);
635  dds->filename(filename);
636 
637  // For the time being, not mess up CF's fileID with Default's fileID
638  if(true == _usecf) {
639 // This block cannot be used to cache the data
640 #if 0
641  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
642  string dds_filename = "/tmp/"+base_filename+"_dds";
643  FILE *dds_file = fopen(dds_filename.c_str(),"r");
644 cerr<<"before parsing "<<endl;
645 BaseTypeFactory tf;
646 DDS tdds(&tf,name_path(filename),"3.2");
647 tdds.filename(filename);
648  //dds->parse(dds_file);
649  tdds.parse(dds_file);
650  //DDS *cache_dds = new DDS(tdds);
651  cache_dds = new DDS(tdds);
652 if(dds!=NULL)
653  delete dds;
654 dds = cache_dds;
655 tdds.print(cout);
656 dds->print(cout);
657 cerr<<"after parsing "<<endl;
658 //dds->print(cout);
659  // fclose(dds_file);
660 //#endif
661 
662 #endif
663 // end of this block
664 
665 
666  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
667  if (cf_fileid < 0){
668  string invalid_file_msg="Could not open this HDF5 file ";
669  invalid_file_msg +=filename;
670  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
671  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
672  invalid_file_msg +=" distributor.";
673  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
674  }
675 //#if 0
676  read_cfdds(*dds,filename,cf_fileid);
677 //#endif
678  // Generate the DDS cached file
679  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
680  string dds_filename = "/tmp/"+base_filename+"_dds";
681  FILE *dds_file = fopen(dds_filename.c_str(),"w");
682  dds->print(dds_file);
683  fclose(dds_file);
684 
685  }
686  else {
687 
688  fileid = get_fileid(filename.c_str());
689  if (fileid < 0) {
690  string invalid_file_msg="Could not open this HDF5 file ";
691  invalid_file_msg +=filename;
692  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
693  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
694  invalid_file_msg +=" distributor.";
695  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
696  }
697 
698  depth_first(fileid, (char*)"/", *dds, filename.c_str());
699 
700  }
701 
702 
703  // Check semantics
704  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
705  dds->print(cerr);
706  throw InternalErr(__FILE__, __LINE__,
707  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
708  }
709 
710  Ancillary::read_ancillary_dds( *dds, filename ) ;
711 
712 
713  // Check DAS cache
714  DAS *das = 0 ;
715 
716  if (das_cache && (das = static_cast<DAS*>(das_cache->get(filename)))) {
717  BESDEBUG(HDF5_NAME, "DAS Cached hit for : " << filename << endl);
718  dds->transfer_attributes(das); // no need to copy the cached DAS
719  }
720 
721  else {
722 
723  das = new DAS ;
724 
725  if (!container_name.empty())
726  das->container_name(container_name);
727 
728  if (true == _usecf) {
729 
730  // go to the CF option
731  read_cfdas( *das,filename,cf_fileid);
732 
733  }
734  else {
735 
736  find_gloattr(fileid, *das);
737  depth_first(fileid, "/", *das);
738  close_fileid(fileid);
739  }
740 
741  if(cf_fileid != -1)
742  H5Fclose(cf_fileid);
743 
744  Ancillary::read_ancillary_das( *das, filename ) ;
745 
746  dds->transfer_attributes(das);
747 
748 
749  // Only free the DAS if it's not added to the cache
750  if (das_cache) {
751  // add a copy
752  BESDEBUG(HDF5_NAME, "DAS added to the cache for : " << filename << endl);
753  //das_cache->add(new DAS(*das), filename);
754  das_cache->add(das, filename);
755  }
756  else {
757  delete das;
758  }
759  }
760 
761  if (dds_cache) {
762  // add a copy
763  BESDEBUG(HDF5_NAME, "DDS added to the cache for : " << filename << endl);
764  dds_cache->add(new DDS(*dds), filename);
765  }
766 
767  }
768 
769 //dds->print(cout);
770 
771  }
772  catch(InternalErr & e) {
773 
774  if(cf_fileid !=-1)
775  H5Fclose(cf_fileid);
776 
777  if(fileid != -1)
778  H5Fclose(fileid);
779 
780  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
781  __FILE__, __LINE__);
782  }
783  catch(Error & e) {
784 
785  if(cf_fileid !=-1)
786  H5Fclose(cf_fileid);
787  if(fileid !=-1)
788  H5Fclose(fileid);
789 
790  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
791  __FILE__, __LINE__);
792  }
793  catch(...) {
794 
795  if(cf_fileid !=-1)
796  H5Fclose(cf_fileid);
797  if(fileid !=-1)
798  H5Fclose(fileid);
799 
800  string s = "unknown exception caught building HDF5 DDS";
801  throw BESInternalFatalError(s, __FILE__, __LINE__);
802  }
803 
804 }
805 #endif
806 
807 // Build DDS
808 bool HDF5RequestHandler::hdf5_build_dds(BESDataHandlerInterface & dhi)
809 {
810 
811  // Obtain the HDF5 file name.
812  string filename = dhi.container->access();
813 
814  string container_name = dhi.container->get_symbolic_name();
815  BESResponseObject *response = dhi.response_handler->get_response_object();
816  BESDDSResponse *bdds = dynamic_cast < BESDDSResponse * >(response);
817  if( !bdds )
818  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
819  bdds->set_container(container_name);
820 
821  try {
822 
823  bool dds_from_dc = false;
824  bool das_from_dc = false;
825  bool build_data = false;
826  string dds_cache_fname;
827  string das_cache_fname;
828 
829  if(_use_disk_meta_cache == true) {
830 
831  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
832 
833  // The _use_disk_dds_cache is always set to false by default
834  if(_use_disk_dds_cache == true) {
835  dds_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_dds";
836  if(access(dds_cache_fname.c_str(),F_OK) !=-1)
837  dds_from_dc = true;
838  }
839 
840  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
841  // Check if das files exist
842  if(access(das_cache_fname.c_str(),F_OK) !=-1)
843  das_from_dc = true;
844 
845  }
846 
847  get_dds_with_attributes(bdds, NULL,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
848 
849  // The following block reads dds from a dds cache file.
850 #if 0
851  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
852  string dds_filename = "/tmp/"+base_filename+"_dds";
853 
854  BaseTypeFactory tf;
855  DDS tdds(&tf,name_path(filename),"3.2");
856  tdds.filename(filename);
857 
858 
859  FILE *dds_file = fopen(dds_filename.c_str(),"r");
860  tdds.parse(dds_file);
861 //cerr<<"before parsing "<<endl;
862  DDS* cache_dds = new DDS(tdds);
863  if(dds != NULL)
864  delete dds;
865  bdds->set_dds(cache_dds);
866  fclose(dds_file);
867 #endif
868 
869  bdds->set_constraint( dhi ) ;
870  bdds->clear_container() ;
871 
872  }
873  catch(InternalErr & e) {
874 
875  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
876  __FILE__, __LINE__);
877  }
878  catch(Error & e) {
879 
880  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
881  __FILE__, __LINE__);
882  }
883  catch(...) {
884 
885  string s = "unknown exception caught building HDF5 DDS";
886  throw BESInternalFatalError(s, __FILE__, __LINE__);
887  }
888 
889  return true;
890 }
891 
892 bool HDF5RequestHandler::hdf5_build_data(BESDataHandlerInterface & dhi)
893 {
894 
895  if(true ==_usecf) {
896 
897  if(true == _pass_fileid)
898  return hdf5_build_data_with_IDs(dhi);
899  }
900 
901  string filename = dhi.container->access();
902 
903  string container_name = dhi.container->get_symbolic_name();
904  BESResponseObject *response = dhi.response_handler->get_response_object();
905  BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
906  if( !bdds )
907  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
908  bdds->set_container(container_name);
909 
910  try {
911 
912  bool dds_from_dc = false;
913  bool das_from_dc = false;
914  bool build_data = true;
915  string dds_cache_fname;
916  string das_cache_fname;
917 
918  // Only DAS is read from the cache. dds_from_dc is always false.
919  if(_use_disk_meta_cache == true) {
920 
921  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
922  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
923 
924  if(access(das_cache_fname.c_str(),F_OK) !=-1)
925  das_from_dc = true;
926 
927  }
928 
929  get_dds_with_attributes(NULL,bdds, container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
930 
931  bdds->set_constraint( dhi ) ;
932  bdds->clear_container() ;
933 
934  }
935  catch(InternalErr & e) {
936 
937  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
938  __FILE__, __LINE__);
939  }
940  catch(Error & e) {
941 
942  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
943  __FILE__, __LINE__);
944  }
945  catch(...) {
946 
947  string s = "unknown exception caught building HDF5 DDS";
948  throw BESInternalFatalError(s, __FILE__, __LINE__);
949  }
950 
951  return true;
952 }
953 
954 // Obtain data when turning on the pass fileID key.The memory cache is not used.
955 bool HDF5RequestHandler::hdf5_build_data_with_IDs(BESDataHandlerInterface & dhi)
956 {
957 
958  BESDEBUG("h5","Building DataDDS by passing file IDs. "<<endl);
959  hid_t cf_fileid = -1;
960 
961  string filename = dhi.container->access();
962 
963  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
964  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
965  if (cf_fileid < 0){
966  string invalid_file_msg="Could not open this HDF5 file ";
967  invalid_file_msg +=filename;
968  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
969  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
970  invalid_file_msg +=" distributor.";
971  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
972  }
973 
974  BESResponseObject *response = dhi.response_handler->get_response_object();
975  BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
976  if( !bdds )
977  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
978 
979  try {
980 
981  bdds->set_container( dhi.container->get_symbolic_name() ) ;
982 
983  HDF5DDS *hdds = new HDF5DDS(bdds->get_dds());
984  delete bdds->get_dds();
985 
986  bdds->set_dds(hdds);
987 
988  hdds->setHDF5Dataset(cf_fileid);
989 
990  read_cfdds( *hdds,filename,cf_fileid);
991 
992  if (!hdds->check_semantics()) { // DDS didn't comply with the DAP semantics
993  hdds->print(cerr);
994  throw InternalErr(__FILE__, __LINE__,
995  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
996  }
997 
998  Ancillary::read_ancillary_dds( *hdds, filename ) ;
999 
1000  DAS *das = new DAS ;
1001  BESDASResponse bdas( das ) ;
1002  bdas.set_container( dhi.container->get_symbolic_name() ) ;
1003  read_cfdas( *das,filename,cf_fileid);
1004  Ancillary::read_ancillary_das( *das, filename ) ;
1005 
1006  hdds->transfer_attributes(das);
1007  bdds->set_constraint( dhi ) ;
1008  bdds->clear_container() ;
1009 
1010  }
1011 
1012  catch(InternalErr & e) {
1013  if(cf_fileid !=-1)
1014  H5Fclose(cf_fileid);
1015  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1016  __FILE__, __LINE__);
1017  }
1018  catch(Error & e) {
1019  if(cf_fileid !=-1)
1020  H5Fclose(cf_fileid);
1021  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1022  __FILE__, __LINE__);
1023  }
1024  catch(...) {
1025  if(cf_fileid !=-1)
1026  H5Fclose(cf_fileid);
1027  string s = "unknown exception caught building HDF5 DataDDS";
1028  throw BESInternalFatalError(s, __FILE__, __LINE__);
1029  }
1030 
1031  return true;
1032 }
1033 
1034 bool HDF5RequestHandler::hdf5_build_dmr(BESDataHandlerInterface & dhi)
1035 {
1036 
1037 
1038  // Extract the DMR Response object - this holds the DMR used by the
1039  // other parts of the framework.
1040  BESResponseObject *response = dhi.response_handler->get_response_object();
1041  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1042 
1043  string filename = dhi.container->access();
1044 
1045  DMR *dmr = bes_dmr.get_dmr();
1046 
1047  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
1048  hid_t fileid = -1;
1049  hid_t cf_fileid = -1;
1050 
1051  try {
1052 
1053  DMR* cached_dmr_ptr = 0;
1054  bool use_dmr_cache = false;
1055  if (dmr_cache)
1056  cached_dmr_ptr = static_cast<DMR*>(dmr_cache->get(filename));
1057  if (cached_dmr_ptr)
1058  use_dmr_cache = true;
1059  if (true == use_dmr_cache) {
1060  // copy the cached DMR into the BES response object
1061  BESDEBUG(HDF5_NAME, "DMR Cached hit for : " << filename << endl);
1062  *dmr = *cached_dmr_ptr; // Copy the referenced object
1063  }
1064  else {// No cache
1065 
1066  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1067  D4BaseTypeFactory MyD4TypeFactory;
1068  dmr->set_factory(&MyD4TypeFactory);
1069 
1070  if(true ==_usecf) {// CF option
1071 
1072  if(true == _pass_fileid)
1073  return hdf5_build_dmr_with_IDs(dhi);
1074 
1075  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1076  if (cf_fileid < 0){
1077  string invalid_file_msg="Could not open this HDF5 file ";
1078  invalid_file_msg +=filename;
1079  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1080  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1081  invalid_file_msg +=" distributor.";
1082  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1083  }
1084 
1085  BaseTypeFactory factory;
1086  DDS dds(&factory, name_path(filename), "3.2");
1087  dds.filename(filename);
1088 
1089  DAS das;
1090 
1091  // For the CF option dmr response, we need to map 64-bit integer separately
1092  // So set the flag to map 64-bit integer.
1093  HDF5RequestHandler::set_dmr_64bit_int(dmr);
1094  read_cfdds( dds,filename,cf_fileid);
1095  if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1096  dds.print(cerr);
1097  throw InternalErr(__FILE__, __LINE__,
1098  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1099  }
1100 
1101  read_cfdas(das,filename,cf_fileid);
1102  Ancillary::read_ancillary_das( das, filename ) ;
1103 
1104  dds.transfer_attributes(&das);
1105 
1107  if(cf_fileid !=-1)
1108  H5Fclose(cf_fileid);
1109 
1110  dmr->build_using_dds(dds);
1111 
1112  }// "if(true == _usecf)"
1113  else {// default option
1114 
1115  // Obtain the HDF5 file ID.
1116  fileid = get_fileid(filename.c_str());
1117  if (fileid < 0) {
1118  string invalid_file_msg="Could not open this HDF5 file ";
1119  invalid_file_msg +=filename;
1120  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1121  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1122  invalid_file_msg +=" distributor.";
1123  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1124  }
1125 
1126  bool use_dimscale = false;
1127  if(true == _default_handle_dimension)
1128  use_dimscale = check_dimscale(fileid);
1129  dmr->set_name(name_path(filename));
1130  dmr->set_filename(name_path(filename));
1131 
1132  D4Group* root_grp = dmr->root();
1133  breadth_first(fileid,(char*)"/",root_grp,filename.c_str(),use_dimscale);
1134 
1135 #if 0
1136  if(true == use_dimscale)
1137  //breadth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str(),true);
1138  breadth_first(fileid,(char*)"/",root_grp,filename.c_str(),true);
1139  else
1140  depth_first(fileid,(char*)"/",root_grp,filename.c_str());
1141  //depth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str());
1142 #endif
1143 
1144  close_fileid(fileid);
1145 
1146  }// else (default option)
1147 
1148  // If the cache is turned on, add the memory cache.
1149  if (dmr_cache) {
1150  // add a copy
1151  BESDEBUG(HDF5_NAME, "DMR added to the cache for : " << filename << endl);
1152  dmr_cache->add(new DMR(*dmr), filename);
1153  }
1154  }// else no cache
1155  }// try
1156 
1157  catch(InternalErr & e) {
1158 
1159  if(cf_fileid !=-1)
1160  H5Fclose(cf_fileid);
1161  if(fileid !=-1)
1162  H5Fclose(fileid);
1163 
1164  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1165  __FILE__, __LINE__);
1166  }
1167  catch(Error & e) {
1168 
1169  if(cf_fileid !=-1)
1170  H5Fclose(cf_fileid);
1171  if(fileid !=-1)
1172  H5Fclose(fileid);
1173  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1174  __FILE__, __LINE__);
1175  }
1176  catch(...) {
1177 
1178  if(cf_fileid !=-1)
1179  H5Fclose(cf_fileid);
1180  if(fileid !=-1)
1181  H5Fclose(fileid);
1182  string s = "unknown exception caught building HDF5 DMR";
1183  throw BESInternalFatalError(s, __FILE__, __LINE__);
1184  }
1185 
1186  //dmr->print(cout);
1187 
1188  // Instead of fiddling with the internal storage of the DHI object,
1189  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1190  // methods to set the constraints. But, why? Ans: from Patrick is that
1191  // in the 'container' mode of BES each container can have a different
1192  // CE.
1193  bes_dmr.set_dap4_constraint(dhi);
1194  bes_dmr.set_dap4_function(dhi);
1195  dmr->set_factory(0);
1196 
1197  return true;
1198 }
1199 
1200 // This function is only used when EnableCF is true.
1201 bool HDF5RequestHandler::hdf5_build_dmr_with_IDs(BESDataHandlerInterface & dhi)
1202 {
1203 
1204  BESDEBUG("h5","Building DMR with passing file IDs. "<<endl);
1205  string filename = dhi.container->access();
1206  hid_t cf_fileid = -1;
1207 
1208  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1209  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1210  if (cf_fileid < 0){
1211  string invalid_file_msg="Could not open this HDF5 file ";
1212  invalid_file_msg +=filename;
1213  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1214  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1215  invalid_file_msg +=" distributor.";
1216  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1217  }
1218 
1219  BaseTypeFactory factory;
1220  DDS dds(&factory, name_path(filename), "3.2");
1221  dds.filename(filename);
1222 
1223  DAS das;
1224 
1225  try {
1226 
1227 
1228  // This is the CF option
1229  read_cfdds( dds,filename,cf_fileid);
1230 
1231  if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1232  dds.print(cerr);
1233  throw InternalErr(__FILE__, __LINE__,
1234  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1235  }
1236 
1237  Ancillary::read_ancillary_dds( dds, filename ) ;
1238 
1239 
1240  read_cfdas(das,filename,cf_fileid);
1241 
1242  Ancillary::read_ancillary_das( das, filename ) ;
1243 
1244  dds.transfer_attributes(&das);
1245 
1247  //if(cf_fileid !=-1)
1248  // H5Fclose(cf_fileid);
1249 
1250  }
1251  catch(InternalErr & e) {
1252 
1253  if(cf_fileid !=-1)
1254  H5Fclose(cf_fileid);
1255 
1256  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1257  __FILE__, __LINE__);
1258  }
1259  catch(Error & e) {
1260 
1261  if(cf_fileid !=-1)
1262  H5Fclose(cf_fileid);
1263 
1264  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1265  __FILE__, __LINE__);
1266  }
1267  catch(...) {
1268 
1269  if(cf_fileid !=-1)
1270  H5Fclose(cf_fileid);
1271 
1272  string s = "unknown exception caught building HDF5 DataDDS";
1273  throw BESInternalFatalError(s, __FILE__, __LINE__);
1274  }
1275 
1276  // Extract the DMR Response object - this holds the DMR used by the
1277  // other parts of the framework.
1278  BESResponseObject *response = dhi.response_handler->get_response_object();
1279  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1280 
1281  // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1282  // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1283  // the BES built and add this one. The HDF4DMR object will close the open dataset when
1284  // the BES runs the DMR's destructor.
1285 
1286  DMR *dmr = bes_dmr.get_dmr();
1287  D4BaseTypeFactory MyD4TypeFactory;
1288  dmr->set_factory(&MyD4TypeFactory);
1289  dmr->build_using_dds(dds);
1290 
1291  HDF5DMR *hdf5_dmr = new HDF5DMR(dmr);
1292  hdf5_dmr->setHDF5Dataset(cf_fileid);
1293  delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1294  bes_dmr.set_dmr(hdf5_dmr); // BESDMRResponse will delete hdf5_dmr
1295 
1296  // Instead of fiddling with the internal storage of the DHI object,
1297  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1298  // methods to set the constraints. But, why? Ans: from Patrick is that
1299  // in the 'container' mode of BES each container can have a different
1300  // CE.
1301  bes_dmr.set_dap4_constraint(dhi);
1302  bes_dmr.set_dap4_function(dhi);
1303  hdf5_dmr->set_factory(0);
1304 
1305  return true;
1306 }
1307 
1308 bool HDF5RequestHandler::hdf5_build_help(BESDataHandlerInterface & dhi)
1309 {
1310  BESResponseObject *response = dhi.response_handler->get_response_object();
1311  BESInfo *info = dynamic_cast<BESInfo *>(response);
1312  if( !info )
1313  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1314 
1315  string add_info="Just for Test";
1316 
1317  map<string,string> attrs ;
1318  attrs["name"] = MODULE_NAME ;
1319  attrs["version"] = MODULE_VERSION ;
1320  list<string> services ;
1321  BESServiceRegistry::TheRegistry()->services_handled( HDF5_NAME, services );
1322  if( services.size() > 0 )
1323  {
1324  string handles = BESUtil::implode( services, ',' ) ;
1325  attrs["handles"] = handles ;
1326  }
1327  info->begin_tag( "module", &attrs ) ;
1328  info->end_tag( "module" ) ;
1329  info->add_data(add_info);
1330 
1331  return true;
1332 }
1333 
1334 bool HDF5RequestHandler::hdf5_build_version(BESDataHandlerInterface & dhi)
1335 {
1336  BESResponseObject *response = dhi.response_handler->get_response_object();
1337  BESVersionInfo *info = dynamic_cast < BESVersionInfo * >(response);
1338  if( !info )
1339  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1340 
1341  info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1342 
1343  return true;
1344 }
1345 
1346 
1347 bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1348 {
1349  string lrd_config_fpath;
1350  string lrd_config_fname;
1351 
1352  // Obtain DataCache path
1353  lrd_config_fpath = get_beskeys("H5.DataCachePath");
1354 
1355  // Obtain the configure file name that specifics the large file configuration
1356  lrd_config_fname = get_beskeys("H5.LargeDataMemCacheFileName");
1357 
1358  // If either the configure file path or fname is missing, won't add specific mem. cache dirs.
1359  if(lrd_config_fpath=="" || lrd_config_fname=="")
1360  return false;
1361 
1362  // temp_line for storing info of one line in the config. file
1363  string temp_line;
1364 
1365  // The full path of the configure file
1366  string mcache_config_fname = lrd_config_fpath+"/"+lrd_config_fname;
1367 
1368  //ifstream mcache_config_file("example.txt");
1369  // Open the configure file
1370  ifstream mcache_config_file(mcache_config_fname.c_str());
1371 
1372  // If the configuration file is not open, return false.
1373  if(mcache_config_file.is_open()==false){
1374  BESDEBUG(HDF5_NAME,"The large data memory cache configure file "<<mcache_config_fname );
1375  BESDEBUG(HDF5_NAME," cannot be opened."<<endl);
1376  return false;
1377  }
1378 
1379  // Read the configuration file line by line
1380  while(getline(mcache_config_file,temp_line)) {
1381 
1382  // Only consider lines that is no less than 2 characters and the 2nd character is space.
1383  if(temp_line.size()>1 && temp_line.at(1)==' ') {
1384  char sep=' ';
1385  string subline = temp_line.substr(2);
1386  vector<string> temp_name_list;
1387 
1388  // Include directories to store common latitude and longitude values
1389  if(temp_line.at(0)=='1') {
1390  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1391  //lrd_cache_dir_list +=temp_name_list;
1392  lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1393  }
1394  // Include directories not to store common latitude and longitude values
1395  else if(temp_line.at(0)=='0'){
1396  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1397  //lrd_non_cache_dir_list +=temp_name_list;
1398  lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1399  }
1400  // Include variable names that the server would like to store in the memory cache
1401  else if(temp_line.at(0)=='2') {
1402 
1403  // We need to handle the space case inside a variable path
1404  // either "" or '' needs to be used to identify a var path
1405  vector<int>dq_pos;
1406  vector<int>sq_pos;
1407  for(unsigned int i = 0; i<subline.size();i++){
1408  if(subline[i]=='"') {
1409  dq_pos.push_back(i);
1410  }
1411  else if(subline[i]=='\'')
1412  sq_pos.push_back(i);
1413  }
1414  if(dq_pos.size()==0 && sq_pos.size()==0)
1415  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1416  else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1417  unsigned int dq_index= 0;
1418  while(dq_index < dq_pos.size()){
1419  if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1420  temp_name_list.push_back
1421  (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1422  }
1423  dq_index = dq_index + 2;
1424  }
1425  }
1426  else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1427  unsigned int sq_index= 0;
1428  while(sq_index < sq_pos.size()){
1429  if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1430  temp_name_list.push_back
1431  (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1432  }
1433  sq_index = sq_index+2;
1434  }
1435  }
1436 
1437  lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1438  }
1439  }
1440  }
1441 
1442 
1443 #if 0
1444 
1445 for(int i =0; i<lrd_cache_dir_list.size();i++)
1446 cerr<<"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1447 for(int i =0; i<lrd_non_cache_dir_list.size();i++)
1448 cerr<<"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1449 for(int i =0; i<lrd_var_cache_file_list.size();i++)
1450 cerr<<"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1451 #endif
1452 
1453 
1454  mcache_config_file.close();
1455  if(lrd_cache_dir_list.size()==0 && lrd_non_cache_dir_list.size()==0 && lrd_var_cache_file_list.size()==0)
1456  return false;
1457  else
1458  return true;
1459 }
1460 
1461 
1462 bool HDF5RequestHandler::read_das_from_disk_cache(const string & cache_filename,DAS *das_ptr) {
1463 
1464  BESDEBUG(HDF5_NAME, "Coming to read_das_from_disk_cache() " << cache_filename << endl);
1465  bool ret_value = true;
1466  FILE *md_file = NULL;
1467  md_file = fopen(cache_filename.c_str(),"rb");
1468 
1469  if(NULL == md_file) {
1470  string bes_error = "An error occurred trying to open a metadata cache file " + cache_filename;
1471  throw BESInternalError( bes_error, __FILE__, __LINE__);
1472  }
1473  else {
1474 
1475  int fd_md = fileno(md_file);
1476  struct flock *l_md;
1477  l_md = lock(F_RDLCK);
1478 
1479  // hold a read(shared) lock to read metadata from a file.
1480  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1481  fclose(md_file);
1482  ostringstream oss;
1483  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1484  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1485  }
1486 
1487  try {
1488 
1489  struct stat sb;
1490  if(stat(cache_filename.c_str(),&sb) != 0) {
1491  string bes_error = "An error occurred trying to stat a metadata cache file size " + cache_filename;
1492  throw BESInternalError( bes_error, __FILE__, __LINE__);
1493 
1494  }
1495 
1496 
1497  size_t bytes_expected_read=(size_t)sb.st_size;
1498  BESDEBUG(HDF5_NAME, "DAS Disk cache file size is " << bytes_expected_read << endl);
1499 
1500  vector<char> buf;
1501  buf.resize(bytes_expected_read);
1502  size_t bytes_to_read =fread((void*)&buf[0],1,bytes_expected_read,md_file);
1503  if(bytes_to_read != bytes_expected_read)
1504  throw InternalErr(__FILE__,__LINE__,"Fail to read the data from the das cache file.");
1505 
1506  char* temp_pointer =&buf[0];
1507 
1508  AttrTable*at = NULL;
1509  // recursively build DAS
1510 //#if 0
1511  temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1512 //#endif
1513 
1514 
1515  }
1516  catch(...) {
1517  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1518  fclose(md_file);
1519  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1520  }
1521 
1522  fclose(md_file);
1523  throw InternalErr(__FILE__,__LINE__,"Fail to parse a das cache file.");
1524  }
1525 
1526  // Unlock the cache file
1527  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1528  fclose(md_file);
1529  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1530  }
1531  fclose(md_file);
1532  }
1533  return ret_value;
1534 
1535 }
1536 
1537 // This fucntion will NOT be used by default. Leave here for future improvement.
1538 bool HDF5RequestHandler::write_dds_to_disk_cache(const string& dds_cache_fname,DDS *dds_ptr) {
1539 
1540  BESDEBUG(HDF5_NAME, "Write DDS to disk cache " << dds_cache_fname << endl);
1541  FILE *dds_file = fopen(dds_cache_fname.c_str(),"w");
1542 
1543  if(NULL == dds_file) {
1544  string bes_error = "An error occurred trying to open a metadata cache file " + dds_cache_fname;
1545  throw BESInternalError( bes_error, __FILE__, __LINE__);
1546  }
1547  else {
1548 
1549  int fd_md = fileno(dds_file);
1550  struct flock *l_md;
1551  l_md = lock(F_WRLCK);
1552 
1553  // hold a read(shared) lock to read metadata from a file.
1554  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1555  fclose(dds_file);
1556  ostringstream oss;
1557  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1558  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1559  }
1560 
1561  try {
1562  dds_ptr->print(dds_file);
1563  }
1564  catch(...) {
1565  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1566  fclose(dds_file);
1567  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1568  }
1569 
1570  fclose(dds_file);
1571  throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
1572  }
1573 
1574  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1575  fclose(dds_file);
1576  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1577  }
1578 
1579  fclose(dds_file);
1580  }
1581  return true;
1582 
1583 }
1584 
1585 // Write DAS to a binary cached file on disk.
1586 bool HDF5RequestHandler::write_das_to_disk_cache(const string & das_cache_fname, DAS *das_ptr) {
1587 
1588  BESDEBUG(HDF5_NAME, "Write DAS to disk cache " << das_cache_fname << endl);
1589  FILE *das_file = fopen(das_cache_fname.c_str(),"wb");
1590  if(NULL == das_file) {
1591  string bes_error = "An error occurred trying to open a metadata cache file " + das_cache_fname;
1592  throw BESInternalError( bes_error, __FILE__, __LINE__);
1593  }
1594  else {
1595  int fd_md = fileno(das_file);
1596  struct flock *l_md;
1597  l_md = lock(F_WRLCK);
1598 
1599  // hold a write(exclusive) lock to write metadata to a file.
1600  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1601  fclose(das_file);
1602  ostringstream oss;
1603  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1604  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1605  }
1606 
1607  try {
1608  write_das_to_file(das_ptr,das_file);
1609  }
1610  catch(...) {
1611  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1612  fclose(das_file);
1613  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1614  }
1615 
1616  fclose(das_file);
1617  throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
1618  }
1619 
1620  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1621  fclose(das_file);
1622  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1623  }
1624 
1625  fclose(das_file);
1626 
1627  }
1628 
1629  return true;
1630 
1631 }
1632 
1633 // The wrapper function to call write_das_table_to_file to generate the cache.
1634 void write_das_to_file(DAS*das_ptr,FILE* das_file) {
1635 
1636  // When category_flag is 2, it marks the end of the file.
1637  uint8_t category_flag = 2;
1638  AttrTable* top_table = das_ptr->get_top_level_attributes();
1639  write_das_table_to_file(top_table,das_file);
1640 
1641  // Add the final ending flag for retrieving the info.
1642  fwrite((const void*)&category_flag,1,1,das_file);
1643  return;
1644 
1645 }
1646 
1647 // The main function to write DAS to a file
1648 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
1649 
1650  if(temp_table !=NULL) {
1651 
1652  // 2 is the end mark of an attribute table
1653  uint8_t category_flag = 2;
1654 
1655  // Loop through the whole DAS top table
1656  AttrTable::Attr_iter top_startit = temp_table->attr_begin();
1657  AttrTable::Attr_iter top_endit = temp_table->attr_end();
1658  AttrTable::Attr_iter top_it = top_startit;
1659  while(top_it !=top_endit) {
1660  AttrType atype = temp_table->get_attr_type(top_it);
1661  if(atype == Attr_unknown)
1662  throw InternalErr(__FILE__,__LINE__,"Unsupported DAS Attribute type");
1663  else if(atype!=Attr_container) {
1664  BESDEBUG(HDF5_NAME, "DAS to the disk cache, attr name is: "
1665  << temp_table->get_name(top_it) << endl);
1666  BESDEBUG(HDF5_NAME, "DAS to the disk cache, attr type is: "
1667  << temp_table->get_type(top_it) << endl);
1668  // For the debugging purpose
1669  //unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
1670  //cerr<<"Attribute values are "<<endl;
1671  //for (int i = 0; i <num_attrs;i++)
1672  // cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
1673  //cerr<<endl;
1674  //write_das_attr_info(temp_table,top_it,das_file);
1675  // Write DAS attribute info to the file
1676  write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
1677  }
1678  else {
1679  BESDEBUG(HDF5_NAME, "DAS to the disk cache, attr container name is: "
1680  << (*top_it)->name << endl);
1681  // Write the container and then write the info. in this container
1682  AttrTable* sub_table = temp_table->get_attr_table(top_it);
1683  write_container_name_to_file(sub_table->get_name(),das_file);
1684  write_das_table_to_file(sub_table,das_file);
1685 
1686  // Write the end flag
1687  fwrite((const void*)&category_flag,1,1,das_file);
1688 
1689  }
1690  ++top_it;
1691  }
1692 
1693  }
1694 }
1695 
1696 // Write container name to the disk file
1697 void write_container_name_to_file(const string& cont_name,FILE *das_file) {
1698 
1699  // 1 marks the starting of a container
1700  uint8_t category_flag = 1;
1701  vector<char> buf;
1702  size_t bytes_to_write = cont_name.size()+sizeof(size_t)+1;
1703  buf.resize(bytes_to_write);
1704  char*temp_pointer =&buf[0];
1705  memcpy((void*)temp_pointer,(void*)&category_flag,1);
1706  temp_pointer++;
1707  temp_pointer=copy_str(temp_pointer,cont_name);
1708 
1709  size_t bytes_to_be_written = fwrite((const void*)&buf[0],1,bytes_to_write,das_file);
1710  if(bytes_to_be_written != bytes_to_write)
1711  throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS container name to a cache");
1712  return;
1713 }
1714 
1715 
1716 // Write DAS attribute info. to the disk cache file
1717 void write_das_attr_info(AttrTable* dtp,const string& attr_name, const string & attr_type,FILE * das_file) {
1718 
1719  // 0 marks the starting of a DAS attribute
1720  uint8_t category_flag = 0;
1721 
1722  unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
1723  vector<string> attr_values;
1724  size_t total_attr_values_size = 0;
1725  for (unsigned int i = 0; i <num_attr_elems;i++){
1726  attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
1727  total_attr_values_size += attr_values[i].size();
1728  }
1729  // Need to add a flag, value as 0 to indicate the attribute.
1730  // DAS: category flag, sizeof attirubte name, attribute name, size of attribute type, attribute type
1731  size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2* sizeof(size_t);
1732 
1733  // One unsigned int to store the number of element elements i
1734  // + sizeof(size_t) * number of elements to store the number of characters for each attribute
1735  // (in DAP, every attribute is in string format)
1736  // +total size of all attribute values
1737  bytes_to_write_attr += sizeof(unsigned int) + num_attr_elems*sizeof(size_t)+total_attr_values_size;
1738 
1739  vector<char>attr_buf;
1740  attr_buf.resize(bytes_to_write_attr);
1741  char* temp_attrp =&attr_buf[0];
1742 
1743  // The attribute flag
1744  memcpy((void*)temp_attrp,(void*)&category_flag,1);
1745  temp_attrp++;
1746 
1747  // The attribute name and type
1748  temp_attrp=copy_str(temp_attrp,attr_name);
1749  temp_attrp=copy_str(temp_attrp,attr_type);
1750 
1751  // Number of elements
1752  memcpy((void*)temp_attrp,(void*)&num_attr_elems,sizeof(unsigned int));
1753  temp_attrp+=sizeof(unsigned int);
1754 
1755  // All attributes
1756  for (unsigned int i = 0; i <num_attr_elems;i++)
1757  temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
1758 
1759  size_t bytes_to_be_written = fwrite((const void*)&attr_buf[0],1,bytes_to_write_attr,das_file);
1760  if(bytes_to_be_written != bytes_to_write_attr)
1761  throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS attribute to a cache");
1762 
1763  return;
1764 
1765 }
1766 
1767 // Read DDS from a disk cache, this function is not used by default.
1768 void HDF5RequestHandler::read_dds_from_disk_cache(BESDDSResponse* bdds, BESDataDDSResponse* data_bdds,
1769  bool build_data,const string & container_name,const string & h5_fname,
1770  const string & dds_cache_fname,const string &das_cache_fname, hid_t h5_fd,
1771  bool das_from_dc) {
1772 
1773 
1774  BESDEBUG(HDF5_NAME, "Read DDS from disk cache " << dds_cache_fname << endl);
1775 
1776  DDS *dds;
1777  if(true == build_data)
1778  dds = data_bdds->get_dds();
1779  else
1780  dds = bdds->get_dds();
1781 
1782  // write a function to pass the following with the lock.
1783  BaseTypeFactory tf;
1784  DDS tdds(&tf,name_path(h5_fname),"3.2");
1785  tdds.filename(h5_fname);
1786 
1787  FILE *dds_file = fopen(dds_cache_fname.c_str(),"r");
1788  tdds.parse(dds_file);
1789  DDS* cache_dds = new DDS(tdds);
1790 #if 0
1791 cerr<<"before dds "<<endl;
1792 dds->dump(cerr);
1793 cerr<<"after dds "<<endl;
1794 cerr<<"before tdds "<<endl;
1795 cache_dds->dump(cerr);
1796 cerr<<"after tdds "<<endl;
1797 #endif
1798  if(dds != NULL)
1799  delete dds;
1800 
1801  Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
1802 
1803  add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
1804  if(true == build_data)
1805  data_bdds->set_dds(cache_dds);
1806  else
1807  bdds->set_dds(cache_dds);
1808  fclose(dds_file);
1809 
1810  if (dds_cache) {
1811  // add a copy
1812  BESDEBUG(HDF5_NAME, "Reading DDS from Disk Cache routine, For memory cache, DDS added to the cache for : " << h5_fname << endl);
1813  dds_cache->add(new DDS(*cache_dds), h5_fname);
1814  }
1815 
1816 }
1817 
1818 // Add DAS to DDS.
1819 void HDF5RequestHandler::add_das_to_dds(DDS *dds, const string &/*container_name*/, const string &filename,
1820  const string &das_cache_fname, hid_t h5_fd, bool das_from_dc) {
1821 
1822  BESDEBUG(HDF5_NAME, "Coming to add_das_to_dds() " << endl);
1823 
1824  // Check DAS memory cache
1825  DAS *das = 0 ;
1826  bool use_das_cache = false;
1827  if (das_cache)
1828  das = static_cast<DAS*>(das_cache->get(filename));
1829  if (das)
1830  use_das_cache = true;
1831 
1832  if (true == use_das_cache) {
1833  BESDEBUG(HDF5_NAME, "DAS Cached hit for : " << filename << endl);
1834  dds->transfer_attributes(das); // no need to copy the cached DAS
1835  }
1836 
1837  else {
1838 
1839  das = new DAS ;
1840 #if 0
1841  if (!container_name.empty())
1842  das->container_name(container_name);
1843 #endif
1844  if(das_from_dc == true)
1845  read_das_from_disk_cache(das_cache_fname,das);
1846  else {
1847  // This bool is for the case, when DDS is read from a cache then we need to open the HDF5 file.
1848  bool h5_file_open = true;
1849  if(h5_fd == -1)
1850  h5_file_open = false;
1851  if (true == _usecf) {
1852  // go to the CF option
1853  if(h5_file_open == false)
1854  h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1855 
1856  read_cfdas( *das,filename,h5_fd);
1857  if(h5_file_open == false)
1858  H5Fclose(h5_fd);
1859  }
1860  else {
1861  if(h5_file_open == false)
1862  h5_fd = get_fileid(filename.c_str());
1863  find_gloattr(h5_fd, *das);
1864  depth_first(h5_fd, "/", *das);
1865  if(h5_file_open == false)
1866  close_fileid(h5_fd);
1867  }
1868 
1869  Ancillary::read_ancillary_das( *das, filename ) ;
1870 
1871  if(das_cache_fname!="" && das_from_dc == false)
1872  write_das_to_disk_cache(das_cache_fname,das);
1873  }
1874 
1875  dds->transfer_attributes(das);
1876 
1877  if (das_cache) {
1878  // add a copy
1879  BESDEBUG(HDF5_NAME, "Reading DDS from Disk Cache routine, For memory cache, DAS added to the cache for : " << filename << endl);
1880  das_cache->add(new DAS(*das), filename);
1881  }
1882  delete das;
1883 
1884  }
1885 
1886 }
1887 
1888 bool check_beskeys(const string key) {
1889 
1890  bool found = false;
1891  string doset ="";
1892  const string dosettrue ="true";
1893  const string dosetyes = "yes";
1894 
1895  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
1896  if( true == found ) {
1897  doset = BESUtil::lowercase( doset ) ;
1898  if( dosettrue == doset || dosetyes == doset )
1899  return true;
1900  }
1901  return false;
1902 
1903 }
1904 
1905 // get_uint_key and get_float_key are copied from the netCDF handler.
1906 
1907 static unsigned int get_uint_key(const string &key, unsigned int def_val)
1908 {
1909  bool found = false;
1910  string doset = "";
1911 
1912  TheBESKeys::TheKeys()->get_value(key, doset, found);
1913  if (true == found) {
1914  // In C++11, stoi is better.
1915  return atoi(doset.c_str()); // use better code TODO
1916  }
1917  else {
1918  return def_val;
1919  }
1920 }
1921 
1922 static unsigned long get_ulong_key(const string &key, unsigned long def_val)
1923 {
1924  bool found = false;
1925  string doset = "";
1926 
1927  TheBESKeys::TheKeys()->get_value(key, doset, found);
1928  if (true == found) {
1929  // In C++11, stoull is better.
1930  return atol(doset.c_str()); // use better code TODO
1931  }
1932  else {
1933  return def_val;
1934  }
1935 }
1936 static float get_float_key(const string &key, float def_val)
1937 {
1938  bool found = false;
1939  string doset = "";
1940 
1941  TheBESKeys::TheKeys()->get_value(key, doset, found);
1942  if (true == found) {
1943  return atof(doset.c_str()); // use better code TODO
1944  }
1945  else {
1946  return def_val;
1947  }
1948 }
1949 
1950 static string get_beskeys(const string &key) {
1951 
1952  bool found = false;
1953  string ret_value ="";
1954 
1955  TheBESKeys::TheKeys()->get_value( key, ret_value, found ) ;
1956  return ret_value;
1957 
1958 }
1959 
1960 // The function to copy a string to a memory buffer.
1961 char* copy_str(char*temp_ptr,const string & str) {
1962 
1963  size_t str_size=str.size();
1964  memcpy((void*)temp_ptr,(void*)&str_size,sizeof(size_t));
1965  temp_ptr+=sizeof(size_t);
1966  vector<char>temp_vc2(str.begin(),str.end());
1967  memcpy((void*)temp_ptr,(void*)&temp_vc2[0],str.size());
1968  temp_ptr+=str.size();
1969  return temp_ptr;
1970 
1971 }
1972 
1973 
1974 // Obtain the string from a memory buffer.
1975 // Note: both char* and string(as a reference) will be returned
1976 // The attribute binary first stores the size of the string, then the string itself
1977 char* obtain_str(char*temp_ptr,string & str) {
1978 
1979  size_t oname_size = *((size_t *)temp_ptr);
1980  temp_ptr = temp_ptr + sizeof(size_t);
1981  string oname;
1982  for(unsigned int i =0; i<oname_size; i++){
1983  oname.push_back(*temp_ptr);
1984  ++temp_ptr;
1985  }
1986  str = oname;
1987  return temp_ptr;
1988 
1989 }
1990 
1991 // For our case, there are no global attributes for DAS.
1992 // The global attribures are always under HDF_GLOBAL.
1993 // The main function to obtain the DAS info. from the cache.
1994 char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at_par) {
1995 
1996  // 3 is only for the code to come into the loop.
1997  uint8_t flag =3;
1998  while(flag !=2) {
1999  flag = *((uint8_t*)(temp_pointer));
2000  BESDEBUG(HDF5_NAME, "Build DAS from the disk cache file flag: "
2001  <<" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2002  <<" flag = 3; the initial value to get the attribute retrieval process started."
2003  <<" The flag value is "
2004  << (int)flag <<endl);
2005  temp_pointer++;
2006 
2007  if(flag ==1) {
2008  string container_name;
2009  temp_pointer = obtain_str(temp_pointer,container_name);
2010  BESDEBUG(HDF5_NAME, "DAS from the disk cache, container name is " << container_name << endl);
2011 
2012  // Remember the current Attribute table state
2013  AttrTable*temp_at_par = at_par;
2014  if(at_par == NULL)
2015  at_par = das->add_table(container_name, new AttrTable);
2016  else
2017  at_par = at_par->append_container(container_name);
2018 
2019  temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2020  // MUST resume the original state
2021  at_par = temp_at_par;
2022 
2023  }
2024  else if(flag == 0) {
2025  // The attribute must have a table.
2026  if(at_par ==NULL)
2027  throw BESInternalError( "The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2028 
2029  // Attribute name
2030  string attr_name;
2031  temp_pointer = obtain_str(temp_pointer,attr_name);
2032  BESDEBUG(HDF5_NAME, "DAS from the disk cache, attr name is: " << attr_name << endl);
2033 
2034  // Attribute type
2035  string attr_type;
2036  temp_pointer = obtain_str(temp_pointer,attr_type);
2037  BESDEBUG(HDF5_NAME, "DAS from the disk cache, attr type is: " << attr_type << endl);
2038 
2039  // Attribute values
2040  unsigned int num_values = *((unsigned int*)(temp_pointer));
2041  BESDEBUG(HDF5_NAME, "DAS from the disk cache, number of attribute values is: " << num_values << endl);
2042  temp_pointer+=sizeof(unsigned int);
2043 
2044  vector <string> attr_values;
2045 
2046  for(unsigned int i = 0; i<num_values; i++) {
2047  string attr_value;
2048  temp_pointer = obtain_str(temp_pointer,attr_value);
2049  attr_values.push_back(attr_value);
2050  BESDEBUG(HDF5_NAME, "DAS from the disk cache, attribute value is: " << attr_value << endl);
2051  }
2052 
2053  at_par->append_attr(attr_name,attr_type,&attr_values);
2054  }
2055 
2056  }
2057  return temp_pointer;
2058 
2059 }
2060 
2061 // The debugging function to get attribute info.
2062 void get_attr_contents(AttrTable*temp_table) {
2063  if(temp_table !=NULL) {
2064  AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2065  AttrTable::Attr_iter top_endit = temp_table->attr_end();
2066  AttrTable::Attr_iter top_it = top_startit;
2067  while(top_it !=top_endit) {
2068  AttrType atype = temp_table->get_attr_type(top_it);
2069  if(atype == Attr_unknown)
2070  cerr<<"unsupported DAS attributes" <<endl;
2071  else if(atype!=Attr_container) {
2072 
2073  cerr<<"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2074  cerr<<"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2075  unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2076  cerr<<"Attribute values are "<<endl;
2077  for (unsigned int i = 0; i <num_attrs;i++)
2078  cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2079  cerr<<endl;
2080  }
2081  else {
2082  cerr<<"Coming to the attribute container. "<<endl;
2083  cerr<<"container name is "<<(*top_it)->name <<endl;
2084  AttrTable* sub_table = temp_table->get_attr_table(top_it);
2085  cerr<<"container table name is "<<sub_table->get_name() <<endl;
2086  get_attr_contents(sub_table);
2087  }
2088  ++top_it;
2089  }
2090 
2091  }
2092 }
2093 
2094 
BESRequestHandler
Represents a specific data type request handler.
Definition: BESRequestHandler.h:77
BESDataHandlerInterface::container
BESContainer * container
pointer to current container in this interface
Definition: BESDataHandlerInterface.h:79
breadth_first
bool breadth_first(hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale)
Definition: h5dmr.cc:318
BESDapResponse::set_dap4_constraint
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:133
BESInternalFatalError
exception thrown if an internal error is found and is fatal to the BES
Definition: BESInternalFatalError.h:43
BESContainer::get_symbolic_name
string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:224
ObjMemCache
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:86
BESDataDDSResponse::set_container
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Definition: BESDataDDSResponse.cc:50
BESContainer::access
virtual string access()=0
returns the true name of this container
close_fileid
void close_fileid(hid_t fid)
Definition: h5get.cc:414
BESDapResponse::set_dap4_function
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:150
BESDataDDSResponse::set_dds
void set_dds(libdap::DDS *ddsIn)
Definition: BESDataDDSResponse.h:68
BESDDSResponse::set_dds
void set_dds(libdap::DDS *ddsIn)
Definition: BESDDSResponse.h:71
BESDDSResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition: BESDDSResponse.cc:73
BESDDSResponse::get_dds
libdap::DDS * get_dds()
Definition: BESDDSResponse.h:80
BESDASResponse
Represents an OPeNDAP DAS DAP2 data object within the BES.
Definition: BESDASResponse.h:46
get_fileid
hid_t get_fileid(const char *filename)
Definition: h5get.cc:392
HDF5DDS
Definition: HDF5_DDS.h:37
BESInfo
informational response object
Definition: BESInfo.h:68
ObjMemCache::add
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
find_gloattr
void find_gloattr(hid_t file, DAS &das)
Definition: h5das.cc:482
libdap
Definition: BESDapFunctionResponseCache.h:35
BESDDSResponse::set_container
virtual void set_container(const string &cn)
set the container in the DAP response object
Definition: BESDDSResponse.cc:64
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:61
BESResponseHandler::get_response_object
virtual BESResponseObject * get_response_object()
return the current response object
Definition: BESResponseHandler.cc:78
BESVersionInfo
Definition: BESVersionInfo.h:47
HDF5RequestHandler.h
include the entry functions to execute the handlers
BESUtil::lowercase
static string lowercase(const string &s)
Definition: BESUtil.cc:197
BESDataDDSResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition: BESDataDDSResponse.cc:59
BESInternalError
exception thrown if inernal error encountered
Definition: BESInternalError.h:43
HDF5DMR
Definition: HDF5_DMR.h:40
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:420
BESDDSResponse
Holds a DDS object within the BES.
Definition: BESDDSResponse.h:50
BESDASResponse::set_container
virtual void set_container(const string &cn)
set the container in the DAP response object
Definition: BESDASResponse.cc:49
BESDASResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition: BESDASResponse.cc:58
h5cfdaputil.h
Helper functions for generating DAS attributes and a function to check BES Key.
BESDapResponse::set_constraint
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:111
BESDataDDSResponse
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Definition: BESDataDDSResponse.h:46
BESInfo::add_data
virtual void add_data(const string &s)
add data to this informational object. If buffering is not set then the information is output directl...
Definition: BESInfo.cc:160
h5get.h
Error
depth_first
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
BESDapError
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
BESDataHandlerInterface
Structure storing information used by the BES to handle the request.
Definition: BESDataHandlerInterface.h:60
BESServiceRegistry::services_handled
virtual void services_handled(const string &handler, list< string > &services)
returns the list of servies provided by the handler in question
Definition: BESServiceRegistry.cc:328
hdf5_handler.h
The main header of the HDF5 OPeNDAP handler.
BESUtil::implode
static string implode(const list< string > &values, char delim)
Definition: BESUtil.cc:635
BESResponseObject
Abstract base class representing a specific set of information in response to a request to the BES.
Definition: BESResponseObject.h:51
ObjMemCache::get
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
BESDMRResponse
Represents an OPeNDAP DMR DAP4 data object within the BES.
Definition: BESDMRResponse.h:39