|
moab
|
Read variable-length data from 1-D array dataset. More...
#include <ReadHDF5VarLen.hpp>
Public Member Functions | |
| ReadHDF5VarLen (DebugOutput &debug_output, void *buffer, size_t buffer_size) | |
| Constructor. | |
| virtual | ~ReadHDF5VarLen () |
| ErrorCode | read_data (ReadHDF5Dataset &data_set, const Range &offsets, EntityHandle start_offset, hid_t data_type, const Range &file_ids, const std::vector< unsigned > &vals_per_ent, const Range &ranged_file_ids) |
| Do actual read of data set. | |
| ErrorCode | read_offsets (ReadHDF5Dataset &data_set, const Range &file_ids, EntityHandle start_file_id, EntityHandle nudge, Range &offsets_out, std::vector< unsigned > &counts_out) |
| Read set description table or offset vector for var-len tags or old-format poly(gon|hedra) connectivity. | |
| ErrorCode | read (ReadHDF5Dataset &offset_data, ReadHDF5Dataset &value_data, const Range &file_ids, EntityHandle start_file_id, hid_t data_type, const Range *ranged=0) |
Protected Member Functions | |
| virtual ErrorCode | store_data (EntityHandle file_id, void *data, long num_data, bool ranged)=0 |
| Store data list for a single entity. | |
Protected Attributes | |
| DebugOutput & | dbgOut |
Static Private Member Functions | |
| static bool | is_ranged (EntityHandle file_id, Range::const_iterator &ranged_iter, Range::const_iterator ranged_end) |
| Test if passed file_id is value pointed to by ranged_iter, and if so, incremenet ranged_iter. | |
Private Attributes | |
| void *const | dataBuffer |
| const size_t | bufferSize |
Read variable-length data from 1-D array dataset.
Utility class for reading variable-length data from an HDF5 dataset. Used for reading set contents, set parents, set children, polygon and polyhedron connectivity, and variable-length tag data.
This is an abstract class. The pure virtual store_data method must be implemented to create a concrete instance.
Definition at line 30 of file ReadHDF5VarLen.hpp.
| moab::ReadHDF5VarLen::ReadHDF5VarLen | ( | DebugOutput & | debug_output, |
| void * | buffer, | ||
| size_t | buffer_size | ||
| ) | [inline] |
Constructor.
| buffer | A temporary buffer to use during read |
| buffer_size | Size of buffer, in bytes. |
Definition at line 69 of file ReadHDF5VarLen.hpp.
: dbgOut( debug_output ), dataBuffer( buffer ), bufferSize( buffer_size ) {}
| virtual moab::ReadHDF5VarLen::~ReadHDF5VarLen | ( | ) | [inline, virtual] |
Definition at line 77 of file ReadHDF5VarLen.hpp.
{}
| bool moab::ReadHDF5VarLen::is_ranged | ( | EntityHandle | file_id, |
| Range::const_iterator & | ranged_iter, | ||
| Range::const_iterator | ranged_end | ||
| ) | [static, private] |
Test if passed file_id is value pointed to by ranged_iter, and if so, incremenet ranged_iter.
Definition at line 13 of file ReadHDF5VarLen.cpp.
{
if (ranged_iter == range_end)
return false;
assert( file_id <= *ranged_iter );
if (*ranged_iter != file_id)
return false;
++ranged_iter;
return true;
}
| ErrorCode moab::ReadHDF5VarLen::read | ( | ReadHDF5Dataset & | offset_data, |
| ReadHDF5Dataset & | value_data, | ||
| const Range & | file_ids, | ||
| EntityHandle | start_file_id, | ||
| hid_t | data_type, | ||
| const Range * | ranged = 0 |
||
| ) | [inline] |
Definition at line 139 of file ReadHDF5VarLen.hpp.
{
ErrorCode rval;
const EntityHandle nudge = 1;
Range offsets;
std::vector<unsigned> counts;
rval = read_offsets( offset_data, file_ids,
start_file_id, nudge,
offsets, counts );
if (MB_SUCCESS != rval)
return rval;
Range empty;
rval = read_data( value_data,
offsets, nudge, data_type,
file_ids, counts, ranged ? *ranged : empty );
return rval;
}
| ErrorCode moab::ReadHDF5VarLen::read_data | ( | ReadHDF5Dataset & | data_set, |
| const Range & | offsets, | ||
| EntityHandle | start_offset, | ||
| hid_t | data_type, | ||
| const Range & | file_ids, | ||
| const std::vector< unsigned > & | vals_per_ent, | ||
| const Range & | ranged_file_ids | ||
| ) |
Do actual read of data set.
| data_set | The data set to read. |
| file_ids | The file ids of the entities to read. |
| start_file_id | The file id corresponding to the first row of the dataset |
| data_type | The desired, in-memory data type for values |
| vals_per_ent | The number of values for each entity |
| ranged_file_ids | Those file ids for which the 'ranged' argument to storedata should be passed as true. |
Definition at line 28 of file ReadHDF5VarLen.cpp.
{
ErrorCode rval;
const size_t value_size = H5Tget_size( data_type );
const size_t buffer_size = bufferSize / value_size;
unsigned char* const data_buffer = reinterpret_cast<unsigned char*>(dataBuffer);
std::vector<unsigned char> partial; // for when we read only part of the contents of a set/entity
Range::const_iterator fileid_iter = file_ids.begin();
Range::const_iterator ranged_iter = ranged_file_ids.begin();
std::vector<unsigned>::const_iterator count_iter = vals_per_ent.begin();
size_t count, offset;
bool ranged;
int nn = 0;
assert( file_ids.size() == vals_per_ent.size() );
try {
data_set.set_file_ids( offsets, start_offset, buffer_size, data_type );
}
catch (ReadHDF5Dataset::Exception ) {
return MB_FAILURE;
}
dbgOut.printf( 3, "Reading %s in %lu chunks\n", data_set.get_debug_desc(), data_set.get_read_count() );
while (!data_set.done()) {
dbgOut.printf( 3, "Reading chunk %d of %s\n", ++nn, data_set.get_debug_desc() );
try {
data_set.read( data_buffer, count );
}
catch (ReadHDF5Dataset::Exception ) {
return MB_FAILURE;
}
assert( 0 == count || fileid_iter != file_ids.end() );
// Handle 'special' case where we read some, but not all
// of the data for an entity during the last iteration.
offset = 0;
if (!partial.empty()) { // didn't read all of previous entity
assert( fileid_iter != file_ids.end() );
assert( 0 == (partial.size() % value_size) );
size_t num_prev = partial.size() / value_size;
offset = *count_iter - num_prev;
if (offset > count) { // still don't have all
partial.insert( partial.end(), data_buffer, data_buffer+count*value_size );
continue;
}
partial.insert( partial.end(), data_buffer, data_buffer+offset*value_size );
ranged = is_ranged( *fileid_iter, ranged_iter, ranged_file_ids.end() );
assert(partial.size() == *count_iter * value_size );
rval = store_data( *fileid_iter, &partial[0], *count_iter, ranged );
if (MB_SUCCESS != rval)
return rval;
++count_iter;
++fileid_iter;
partial.clear();
}
// Process contents for all entities for which we
// have read the complete list
while (count_iter != vals_per_ent.end() && offset + *count_iter <= count) {
assert( fileid_iter != file_ids.end() );
ranged = is_ranged( *fileid_iter, ranged_iter, ranged_file_ids.end() );
rval = store_data( *fileid_iter, data_buffer + offset*value_size, *count_iter, ranged );
if (MB_SUCCESS != rval)
return rval;
offset += *count_iter;
++count_iter;
++fileid_iter;
}
// If we did not read all of the final entity,
// store what we did read to be processed in the
// next iteration
if (offset < count) {
assert(partial.empty());
partial.insert( partial.end(),
data_buffer + offset*value_size,
data_buffer + count*value_size );
}
}
// NOTE: If the last set is empty, we will not process it here
// assert(fileid_iter == file_ids.end());
#ifndef NDEBUG
for (;fileid_iter != file_ids.end(); ++fileid_iter)
assert(0 == *count_iter++);
#endif
return MB_SUCCESS;
}
| ErrorCode moab::ReadHDF5VarLen::read_offsets | ( | ReadHDF5Dataset & | data_set, |
| const Range & | file_ids, | ||
| EntityHandle | start_file_id, | ||
| EntityHandle | nudge, | ||
| Range & | offsets_out, | ||
| std::vector< unsigned > & | counts_out | ||
| ) |
Read set description table or offset vector for var-len tags or old-format poly(gon|hedra) connectivity.
| data_set | The data set to read. |
| file_ids | The file ids of the entities to read. |
| start_file_id | The file id corresponding to the first row of the dataset |
| num_columns | The number of columns of offsets in the dataset |
| indices | Array of length num_columns contaning the indices of the columns to read. |
| nudge | Amount by which to offset values in offset_out to avoid putting zeros in Range. Must be greater than 0. Probably 1. |
| offsets_out | An array of length num_columns which will be populated with the resulting list of offsets into the contents list calculated from reading the offsets from the passed data set. |
| counts_out | An array of length num_columns of std::vectors, where each vector will be filled with one value per file ID indicating the length of the data for the corresponding file ID. |
| ranged_file_ids | If non-null, the last column of the table will be read and tested for the ranged bit. For all file_ids for which the range bit is set, the file ID will be added to this list. |
Definition at line 285 of file ReadHDF5VarLen.cpp.
{
// Use hints to make sure insertion into ranges is O(1)
offsets_out.clear();
counts_out.clear();
counts_out.reserve( file_ids.size() );
Range::iterator hint;
// Calculate which rows we need to read from the offsets table
Range rows;
hint = rows.begin();
Range::const_pair_iterator pair = file_ids.const_pair_begin();
// special case if reading first entity in dataset, because
// there is no previous end value.
if (pair != file_ids.const_pair_end() && pair->first == start_file_id) {
hint = rows.insert( nudge, pair->second - start_file_id + nudge );
++pair;
}
while (pair != file_ids.const_pair_end()) {
hint = rows.insert( hint,
pair->first - start_file_id + nudge - 1,
pair->second - start_file_id + nudge );
++pair;
}
// set up read of offsets dataset
hsize_t buffer_size = bufferSize / sizeof(hssize_t);
hssize_t* buffer = reinterpret_cast<hssize_t*>(dataBuffer);
data_set.set_file_ids( rows, nudge, buffer_size, H5T_NATIVE_HSSIZE );
hssize_t prev_end;
bool have_prev_end = false;
// If we're reading the first row of the table, then the
// previous end is implicitly -1.
if (!file_ids.empty() && file_ids.front() == start_file_id) {
prev_end = -1;
have_prev_end = true;
}
dbgOut.printf( 3, "Reading %s in %lu chunks\n", data_set.get_debug_desc(), data_set.get_read_count() );
// read offset table
size_t count, offset;
Range::const_iterator fiter = file_ids.begin();
hint = offsets_out.begin();
int nn = 0;
while (!data_set.done()) {
dbgOut.printf( 3, "Reading chunk %d of %s\n", ++nn, data_set.get_debug_desc() );
try {
data_set.read( buffer, count );
}
catch (ReadHDF5Dataset::Exception ) {
return MB_FAILURE;
}
if (!count) // might have been NULL read for collectve IO
continue;
// If the previous end values were read in the previous iteration,
// then they're stored in prev_end.
offset = 0;
if (have_prev_end) {
counts_out.push_back( buffer[0] - prev_end );
hint = offsets_out.insert( hint,
prev_end + 1 + nudge,
buffer[0] + nudge );
++fiter;
offset = 1;
have_prev_end = false;
}
while (offset < count) {
assert(fiter != file_ids.end());
// whenever we get to a gap between blocks we need to
// advance one step because we read an extra end id
// preceeding teah block
if (fiter == fiter.start_of_block()) {
if (offset == count-1)
break;
++offset;
}
size_t s = buffer[offset-1] + 1;
size_t e = buffer[offset];
counts_out.push_back( e - s + 1 );
hint = offsets_out.insert( hint, s + nudge, e + nudge );
++fiter;
++offset;
}
// If we did not end on the boundary between two blocks,
// then we need to save the end indices for the final entry
// for use in the next iteration. Similarly, if we ended
// with extra values that were read with the express intention
// of getting the previus end values for a block, we need to
// save them. This case only arises if we hit the break in
// the above loop.
if (fiter != fiter.start_of_block() || offset < count) {
assert(!have_prev_end);
if (offset == count) {
--offset;
assert(fiter != fiter.start_of_block());
}
else {
assert(offset+1 == count);
assert(fiter == fiter.start_of_block());
}
have_prev_end = true;
prev_end = buffer[offset];
}
}
assert(!have_prev_end);
assert(fiter == file_ids.end());
return MB_SUCCESS;
}
| virtual ErrorCode moab::ReadHDF5VarLen::store_data | ( | EntityHandle | file_id, |
| void * | data, | ||
| long | num_data, | ||
| bool | ranged | ||
| ) | [protected, pure virtual] |
Store data list for a single entity.
The is the pure virtual method that must be provided. It is responsible for storing the data read for a single entity.
This function will always be called in the order of the file_ids in the range passed to the read method.
| file_id | The file ID for the entity |
| data | A pointer to the data for the entity |
| num_data | Number of values for the entity |
| ranged | For set contents, true if in ranged format. |
const size_t moab::ReadHDF5VarLen::bufferSize [private] |
Definition at line 35 of file ReadHDF5VarLen.hpp.
void* const moab::ReadHDF5VarLen::dataBuffer [private] |
Definition at line 34 of file ReadHDF5VarLen.hpp.
DebugOutput& moab::ReadHDF5VarLen::dbgOut [protected] |
Definition at line 32 of file ReadHDF5VarLen.hpp.
1.7.6.1