From a23a30ae5b90cd4632877c6d8f47a1fe4540a058 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 4 May 2018 10:27:37 -0400 Subject: [PATCH 1/2] Added bpls to utils It can - dump data from global variables - dump from BP and HDF5 files It cannot yet - print values and min/max - print decomposition - dump data per block - print mesh info --- source/utils/CMakeLists.txt | 11 +- source/utils/bpls/bpls.cpp | 2675 ++++++++++++++++++++++++++++++++++ source/utils/bpls/bpls.h | 116 ++ source/utils/bpls2/BPLS2.cpp | 335 ----- source/utils/bpls2/BPLS2.h | 48 - source/utils/bpls2/main.cpp | 32 - 6 files changed, 2795 insertions(+), 422 deletions(-) create mode 100644 source/utils/bpls/bpls.cpp create mode 100644 source/utils/bpls/bpls.h delete mode 100644 source/utils/bpls2/BPLS2.cpp delete mode 100644 source/utils/bpls2/BPLS2.h delete mode 100644 source/utils/bpls2/main.cpp diff --git a/source/utils/CMakeLists.txt b/source/utils/CMakeLists.txt index fb385d910a..c8abc1e4e7 100644 --- a/source/utils/CMakeLists.txt +++ b/source/utils/CMakeLists.txt @@ -3,16 +3,13 @@ # accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -# BPLS2 -add_executable(bpls2 ./bpls2/main.cpp ./bpls2/BPLS2.cpp Utils.cpp) - -target_link_libraries(bpls2 adios2) - -install(TARGETS bpls2 EXPORT adios2 +# BPLS +add_executable(bpls ./bpls/bpls.cpp) +target_link_libraries(bpls adios2) +install(TARGETS bpls EXPORT adios2 RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) - # ADIOS2_REORGANIZE add_executable(adios2_reorganize ./adios2_reorganize/main.cpp diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp new file mode 100644 index 0000000000..1c2e6f8af7 --- /dev/null +++ b/source/utils/bpls/bpls.cpp @@ -0,0 +1,2675 @@ +/* + * ADIOS is freely available under the terms of the BSD license described + * in the COPYING file in the top level directory of this source distribution. + * + * Copyright (c) 2008 - 2009. UT-BATTELLE, LLC. All rights reserved. + */ + +/* + * List the content of a BP file. Rewritten from adios 1.x bpls C code with + *adios 2.x C++ API + * + * Author: Norbert Podhorszki, pnorbert@ornl.gov + * + **/ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +//#include +//#include +#include +#include +#include + +#include "./bpls.h" + +#include +#include // shell pattern matching +#include +#include // regular expression matching + +namespace adios2 +{ +namespace utils +{ + +using EntryMap = std::map; + +// global variables +// Values from the arguments or defaults +char *outpath; // output files' starting path (can be extended with subdirs, + // names, indexes) +char *varmask[MAX_MASKS]; // can have many -var masks (either shell patterns or + // extended regular expressions) +int nmasks; // number of masks specified +char *vfile; // file name to bpls +char *start; // dimension spec starting points +char *count; // dimension spec counts +char format[32]; // format string for one data element (e.g. %6.2f) +bool formatgiven; // true if format string is provided as argument + +// Flags from arguments or defaults +bool dump; // dump data not just list info +bool output_xml; +bool use_regexp; // use varmasks as regular expressions +bool sortnames; // sort names before listing +bool listattrs; // do list attributes too +bool listmeshes; // do list meshes too +bool attrsonly; // do list attributes only +bool readattrs; // also read all attributes and print +bool longopt; // -l is turned on +bool timestep; +bool noindex; // do no print array indices with data +bool printByteAsChar; // print 8 bit integer arrays as string +bool plot; // dump histogram related information +bool hidden_attrs; // show hidden attrs in BP file +int hidden_attrs_flag; // to be passed on in option struct +bool show_decomp; // show decomposition of arrays + +// other global variables +char *prgname; /* argv[0] */ +// long timefrom, timeto; +int64_t istart[MAX_DIMS], icount[MAX_DIMS]; // negative values are allowed +int ndimsspecified = 0; +regex_t varregex[MAX_MASKS]; // compiled regular expressions of varmask +int ncols = 6; // how many values to print in one row (only for -p) +int verbose = 0; +FILE *outf; // file to print to or stdout +char commentchar; + +struct option options[] = { + {"help", no_argument, NULL, 'h'}, + {"verbose", no_argument, NULL, 'v'}, + {"dump", no_argument, NULL, 'd'}, + {"group", no_argument, NULL, 'g'}, + {"regexp", no_argument, NULL, 'e'}, + {"plot", no_argument, NULL, 'p'}, + {"output", required_argument, NULL, 'o'}, + {"xml", no_argument, NULL, 'x'}, + {"start", required_argument, NULL, 's'}, + {"count", required_argument, NULL, 'c'}, + {"noindex", no_argument, NULL, 'y'}, + {"timestep", no_argument, NULL, 't'}, + {"attrs", no_argument, NULL, 'a'}, + {"attrsonly", no_argument, NULL, 'A'}, + {"meshes", no_argument, NULL, 'm'}, + {"long", no_argument, NULL, 'l'}, + {"string", no_argument, NULL, 'S'}, + {"columns", required_argument, NULL, 'n'}, + {"format", required_argument, NULL, 'f'}, + {"hidden_attrs", no_argument, &hidden_attrs_flag, true}, + {"decomp", no_argument, NULL, 'D'}, + // {"sort", no_argument, NULL, 'r'}, + // {"time", required_argument, NULL, 't'}, + {NULL, 0, NULL, 0}}; + +static const char *optstring = "hvepyrtaAmldSDg:o:x:s:c:n:f:"; + +// help function +void display_help() +{ + // printf( "Usage: %s \n", prgname); + printf( + "usage: bpls [OPTIONS] file [mask1 mask2 ...]\n" + "\nList/dump content of a BP file. \n" + "A mask can be a shell pattern like with 'ls' e.g. \"*/x?\".\n" + "Variables with multiple timesteps are reported with an extra " + "dimensions.\n" + "The time dimension is the first dimension then.\n" + "\n" + " --long | -l Print values of all scalars and " + "attributes and\n" + " min/max values of arrays (no overhead " + "to get them!)\n" + " --attrs | -a List/match attributes too\n" + " --attrsonly | -A List attributes only\n" + " --meshes | -m List meshes\n" + /* + " --sort | -r Sort names before listing\n" + */ + " --timestep | -t Print values of timestep elements\n" + " --group | -g List/dump groups matching the mask only\n" + " --dump | -d Dump matched variables/attributes\n" + " To match attributes too, add option " + "-a\n" + " --regexp | -e Treat masks as extended regular " + "expressions\n" + " --plot | -p Dumps the histogram information that can " + "be read by gnuplot\n" + " --output | -o Print to a file instead of stdout\n" + /* + " --xml | -x # print as xml instead of ascii text\n" + */ + " --start | -s \"spec\" Offset indices in each dimension \n" + " (default is 0 for all dimensions) \n" + " <0 is handled as in python (-1 is " + "last)\n" + " --count | -c \"spec\" Number of elements in each dimension\n" + " -1 denotes 'until end' of dimension\n" + " (default is -1 for all dimensions)\n" + " --noindex | -y Print data without array indices\n" + " --string | -S Print 8bit integer arrays as strings\n" + " --columns | -n \"cols\" Number of data elements per row to " + "print\n" + " --format | -f \"str\" Format string to use for one data item " + "in print\n" + " instead of the default. E.g. " + "\"%%6.3f\"\n" + " --hidden_attrs Show hidden ADIOS attributes in the " + "file\n" + " --decomp | -D Show decomposition of variables as layed " + "out in file\n" + /* + " --time | -t N [M] # print data for timesteps N..M only (or + only N)\n" + " default is to print all available + timesteps\n" + */ + "\n" + " Examples for slicing:\n" + " -s \"0,0,0\" -c \"1,99,1\": Print 100 elements (of the 2nd " + "dimension).\n" + " -s \"0,0\" -c \"1,-1\": Print the whole 2nd dimension " + "however large it is.\n" + " -s \"-1,-1\" -c \"1,1\": Print the very last element (of a 2D " + "array)\n" + "\n" + "Help options\n" + " --help | -h Print this help.\n" + " --verbose | -v Print log about what this program is " + "doing.\n" + " Use multiple -v to increase logging " + "level.\n" + "Typical use: bpls -lav \n"); +} + +/** Main */ +int bplsMain(int argc, char *argv[]) +{ + int retval = 0; + int i, timearg = false; + long int tmp; + + init_globals(); + + ////prgname = strdup(argv[0]); + + /* other variables */ + int c; + // int last_c = '_'; + /* Process the arguments */ + while ((c = getopt_long(argc, argv, optstring, options, NULL)) != -1) + { + switch (c) + { + case 'a': + listattrs = true; + break; + case 'A': + listattrs = true; + attrsonly = true; + break; + case 'c': + count = strndup(optarg, 256); + break; + case 'd': + dump = true; + break; + case 'e': + use_regexp = true; + break; + case 'f': + snprintf(format, sizeof(format), "%s", optarg); + formatgiven = true; + break; + case 'h': + display_help(); + return 0; + case 'r': + // sortnames = true; + break; + case 'l': + longopt = true; + readattrs = true; + break; + case 'm': + listmeshes = true; + break; + case 'n': + errno = 0; + tmp = strtol(optarg, (char **)NULL, 0); + if (errno) + { + fprintf(stderr, + "Error: could not convert --columns value: %s\n", + optarg); + return 1; + } + ncols = tmp; + break; + case 'o': + outpath = strndup(optarg, 256); + break; + case 'p': + plot = true; + break; + case 's': + start = strndup(optarg, 256); + break; + case 'S': + printByteAsChar = true; + break; + case 't': + timestep = true; + break; + case 'x': + output_xml = true; + break; + case 'y': + noindex = true; + break; + case 'v': + verbose++; + break; + case 'D': + show_decomp = true; + break; + case 1: + /* This means a field is unknown, or could be multiple arg or bad + * arg*/ + /* + if (last_c=='t') { // --time 2nd arg (or not if not a number) + errno = 0; + tmp = strtol(optarg, (char **)NULL, 0); + if (!errno) { + timeto = tmp; + printf("Time set to %d - %d\n", timefrom, timeto); + timearg=true; + } + } + */ + if (!timearg) + { + fprintf(stderr, "Unrecognized argument: %s\n", optarg); + return 1; + } + break; + + default: + printf("Processing default: %c\n", c); + break; + } /* end switch */ + // last_c = c; + } /* end while */ + + /* Check if we have a file defined */ + if (optind >= argc) + { + fprintf(stderr, "Missing file name\n"); + return 1; + } + vfile = strdup(argv[optind++]); + while (optind < argc) + { + varmask[nmasks] = strndup(argv[optind++], 256); + nmasks++; + } + + /* Process dimension specifications */ + if (start != NULL) + parseDimSpec(start, istart); + if (count != NULL) + parseDimSpec(count, icount); + + // process the regular expressions + if (use_regexp) + { + retval = compile_regexp_masks(); + if (retval) + return retval; + } + + if (noindex) + commentchar = ';'; + else + commentchar = ' '; + + if (hidden_attrs_flag) + hidden_attrs = true; + + if (verbose > 1) + printSettings(); + + retval = print_start(outpath); + if (retval) + return retval; + + /* Start working */ + retval = doList(vfile); + + print_stop(); + + /* Free allocated memories */ + // myfree(prgname); + myfree(outpath); + for (i = 0; i < nmasks; i++) + { + myfree(varmask[i]); + regfree(&(varregex[i])); + } + myfree(vfile); + + return retval; +} + +void init_globals() +{ + int i; + // variables for arguments + outpath = NULL; + for (i = 0; i < MAX_MASKS; i++) + varmask[i] = NULL; + nmasks = 0; + vfile = NULL; + start = NULL; + count = NULL; + verbose = 0; + ncols = 6; // by default when printing ascii, print "X Y", not X: Y1 Y2... + dump = false; + output_xml = false; + noindex = false; + timestep = false; + sortnames = false; + listattrs = false; + listmeshes = false; + attrsonly = false; + readattrs = false; + longopt = false; + // timefrom = 1; + // timeto = -1; + use_regexp = false; + plot = false; + hidden_attrs = false; + hidden_attrs_flag = 0; + formatgiven = false; + printByteAsChar = false; + show_decomp = false; + for (i = 0; i < MAX_DIMS; i++) + { + istart[i] = 0LL; + icount[i] = -1LL; // read full var by default + } + ndimsspecified = 0; +} + +#define PRINT_DIMS_INT(str, v, n, loopvar) \ + printf("%s = { ", str); \ + for (loopvar = 0; loopvar < n; loopvar++) \ + printf("%d ", v[loopvar]); \ + printf("}") + +#define PRINT_DIMS_UINT64(str, v, n, loopvar) \ + printf("%s = { ", str); \ + for (loopvar = 0; loopvar < n; loopvar++) \ + printf("%" PRIu64 " ", v[loopvar]); \ + printf("}") + +#define PRINT_DIMS_INT64(str, v, n, loopvar) \ + printf("%s = { ", str); \ + for (loopvar = 0; loopvar < n; loopvar++) \ + printf("%" PRId64 " ", v[loopvar]); \ + printf("}") + +#define PRINT_DIMS_SIZET(str, v, n, loopvar) \ + printf("%s = { ", str); \ + for (loopvar = 0; loopvar < n; loopvar++) \ + printf("%zu ", v[loopvar]); \ + printf("}") + +void printSettings(void) +{ + int i; + printf("Settings :\n"); + printf(" masks : %d ", nmasks); + for (i = 0; i < nmasks; i++) + printf("%s ", varmask[i]); + printf("\n"); + printf(" file : %s\n", vfile); + printf(" output : %s\n", (outpath ? outpath : "stdout")); + + if (start != NULL) + { + PRINT_DIMS_INT64(" start", istart, ndimsspecified, i); + printf("\n"); + } + if (count != NULL) + { + PRINT_DIMS_INT64(" count", icount, ndimsspecified, i); + printf("\n"); + } + + if (longopt) + printf(" -l : show scalar values and min/max/avg of arrays\n"); + if (sortnames) + printf(" -r : sort names before listing\n"); + if (attrsonly) + printf(" -A : list attributes only\n"); + else if (listattrs) + printf(" -a : list attributes too\n"); + else if (listmeshes) + printf(" -m : list meshes too\n"); + if (dump) + printf(" -d : dump matching variables and attributes\n"); + if (use_regexp) + printf(" -e : handle masks as regular expressions\n"); + if (formatgiven) + printf(" -f : dump using printf format \"%s\"\n", format); + if (output_xml) + printf(" -x : output data in XML format\n"); + if (show_decomp) + printf(" -D : show decomposition of variables in the file\n"); + if (hidden_attrs) + { + printf(" : show hidden attributes in the file\n"); + } +} + +void bpexit(int code, Engine *fp) +{ + if (fp > 0) + fp->Close(); + exit(code); +} + +void print_file_size(uint64_t size) +{ + static const char *sm[] = {"bytes", "KB", "MB", "GB", "TB", "PB", "EB"}; + uint64_t s = size, r = 0; + int idx = 0; + while (s / 1024 > 0) + { + r = s % 1024; + s = s / 1024; + idx++; + } + if (r > 511) + s++; + printf(" file size: %" PRIu64 " %s\n", s, sm[idx]); +} + +static inline int ndigits(int64_t n) +{ + static char digitstr[32]; + return snprintf(digitstr, 32, "%" PRId64, n); +} + +int nEntriesMatched = 0; + +int doList_vars(Engine *fp, IO *io) +{ + enum ADIOS_DATATYPES vartype; + int i, j, n; // loop vars + int attrsize; // info about one attribute + int len, maxlen, maxtypelen; + bool timed; // variable has multiple timesteps + + const DataMap &variables = io->GetVariablesDataMap(); + const DataMap &attributes = io->GetAttributesDataMap(); + + // make a sorted list of all variables and attributes + EntryMap entries; + if (!attrsonly) + { + for (const auto &vpair : variables) + { + Entry e(true, vpair.second.first, vpair.second.second); + entries.emplace(vpair.first, e); + } + } + if (listattrs) + { + for (const auto &apair : attributes) + { + Entry e(false, apair.second.first, apair.second.second); + entries.emplace(apair.first, e); + } + } + + int nNames = entries.size(); + + // calculate max length of variable names and type names in the first round + maxlen = 4; + maxtypelen = 7; + for (const auto &entrypair : entries) + { + len = entrypair.first.size(); + if (len > maxlen) + maxlen = len; + len = entrypair.second.typeName.size(); + if (len > maxtypelen) + maxtypelen = len; + } + + /* VARIABLES */ + for (const auto &entrypair : entries) + { + int retval = 0; + const std::string &name = entrypair.first; + const Entry &entry = entrypair.second; + bool matches = matchesAMask(name.c_str()); + if (matches) + { + nEntriesMatched++; + + // print definition of variable + fprintf(outf, "%c %-*s %-*s", commentchar, maxtypelen, + entry.typeName.c_str(), maxlen, name.c_str()); + if (!entry.isVar) + { + // list (and print) attribute + if (readattrs || dump) + { + fprintf(outf, " attr = "); + /* + int type_size = adios_type_size(vartype, value); + int nelems = attrsize / type_size; + char *p = (char *)value; + if (nelems > 1) + fprintf(outf, "{"); + for (i = 0; i < nelems; i++) + { + if (i > 0) + fprintf(outf, ", "); + print_data(p, 0, vartype, false); + p += type_size; + } + if (nelems > 1) + fprintf(outf, "}"); + */ + fprintf(outf, "\n"); + matches = false; // already printed + } + else + { + fprintf(outf, " attr\n"); + } + } + else + { + VariableBase *v = nullptr; + if (entry.typeName == "compound") + { + // not supported + } +#define declare_template_instantiation(T) \ + else if (entry.typeName == adios2::GetType()) \ + { \ + Variable *v = io->InquireVariable(name); \ + retval = printVariableInfo(fp, io, v); \ + } + ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) +#undef declare_template_instantiation + } + } + + if (retval && retval != 10) // do not return after unsupported type + return retval; + } + + entries.clear(); + return 0; +} + +template +int printVariableInfo(Engine *fp, IO *io, Variable *variable) +{ + size_t nsteps = variable->GetAvailableStepsCount(); + enum ADIOS_DATATYPES adiosvartype = type_to_enum(variable->m_Type); + int retval = 0; + + if (!variable->m_SingleValue || nsteps > 1) + { + fprintf(outf, " "); + if (nsteps > 1) + fprintf(outf, "%zu*", nsteps); + if (variable->m_Shape.size() > 0) + { + fprintf(outf, "{%zu", variable->m_Shape[0]); + for (int j = 1; j < variable->m_Shape.size(); j++) + { + fprintf(outf, ", %zu", variable->m_Shape[j]); + } + fprintf(outf, "}"); + } + else + { + fprintf(outf, "scalar"); + } +#if 0 + if (longopt || plot) + { + adios_inq_var_stat(fp, vi, timestep && timed, show_decomp); + } + + if (plot && vi->statistics && vi->statistics->histogram) + { + print_data_hist(vi, &names[n][1]); + } + + if (longopt && vi->statistics) + { + + if (timestep == false || timed == false) + { + + fprintf(outf, " = "); + if (vartype == adios_complex || vartype == adios_double_complex) + { + // force printing (double,double) here + print_data(vi->statistics->min, 0, adios_double_complex, + false); + fprintf(outf, " / "); + print_data(vi->statistics->max, 0, adios_double_complex, + false); + fprintf(outf, " / "); + print_data(vi->statistics->avg, 0, adios_double_complex, + false); + fprintf(outf, " / "); + print_data(vi->statistics->std_dev, 0, adios_double_complex, + false); + } + else + { + print_data(vi->statistics->min, 0, vartype, false); + fprintf(outf, " / "); + print_data(vi->statistics->max, 0, vartype, false); + fprintf(outf, " / "); + print_data(vi->statistics->avg, 0, adios_double, false); + fprintf(outf, " / "); + print_data(vi->statistics->std_dev, 0, adios_double, false); + } + + // fprintf(outf," {MIN / MAX / AVG / STD_DEV} "); + } + else + { + int time_start = 0, time_end = vi->nsteps; + + if (start != NULL) + { + if (istart[0] >= 0) + time_start = istart[0]; + else + time_start = vi->nsteps - 1 + (int)istart[0]; + } + + if (count != NULL) + { + if (icount[0] > 0) + time_end = time_start + (int)icount[0]; + else + time_end = vi->nsteps + (int)icount[0] + 1; + } + + if (time_start < 0 || time_start >= vi->nsteps) + { + fprintf(stderr, "Error when reading variable %s. " + "errno=%d : Variable (id=%d) has " + "no data at %d time step\n", + names[n], 15, vi->varid, time_start); + bpexit(15, fp); + } + + if (time_end < 0 || time_end > vi->nsteps) + { + fprintf(stderr, "Error when reading variable %s. " + "errno=%d : Variable (id=%d) has " + "no data at %d time step\n", + names[n], 15, vi->varid, time_end); + bpexit(16, fp); + } + + static char *indent_char = " "; + int indent_len = 11; + + /* Start - Print the headers of statistics first */ + fprintf(outf, "\n%-*s", indent_len + 7, indent_char); + fprintf(outf, "%10s ", "MIN"); + fprintf(outf, "%10s ", "MAX"); + fprintf(outf, "%10s ", "AVG"); + fprintf(outf, "%10s ", "STD DEV"); + + /* End - Print the headers of statistics first */ + + void *min, *max, *avg, *std_dev; + enum ADIOS_DATATYPES vt = vartype; + struct ADIOS_STAT_STEP *s = vi->statistics->steps; + if (vi->type == adios_complex || + vi->type == adios_double_complex) + vt = adios_double; + fprintf(outf, "\n%-*sglobal:", indent_len, indent_char); + print_data_characteristics( + vi->statistics->min, vi->statistics->max, + vi->statistics->avg, vi->statistics->std_dev, vt, false); + + for (i = time_start; i < time_end; i++) + { + min = max = avg = std_dev = 0; + if (s->maxs && s->maxs[i]) + max = s->maxs[i]; + if (s->mins && s->mins[i]) + min = s->mins[i]; + if (s->avgs && s->avgs[i]) + avg = s->avgs[i]; + if (s->std_devs && s->std_devs[i]) + std_dev = s->std_devs[i]; + + // Align the output, previous lines has atleast + // (maxlen + strlen(names[n])) characters + // Better way to printf N spaces? + fprintf(outf, "\n%-*st%-5d:", indent_len, indent_char, i); + print_data_characteristics(min, max, avg, std_dev, vt, + false); + } + fprintf(outf, "\n"); + } + } // longopt && vi->statistics +#endif + fprintf(outf, "\n"); + + if (show_decomp) + { + // adios_inq_var_blockinfo(fp, vi); + print_decomp(fp, io, variable); + } + } + else + { + // scalar + fprintf(outf, " scalar"); + if (longopt) + { + fprintf(outf, " = "); + print_data(&variable->m_Value, 0, adiosvartype, false); + } + fprintf(outf, "\n"); + + /*if (show_decomp) + { + adios_inq_var_blockinfo(fp, vi); + adios_inq_var_stat(fp, vi, false, show_decomp); + print_decomp(fp, vi, names[n], timed); + }*/ + } + + if (dump && !show_decomp) + { + // print variable content + retval = readVar(fp, io, variable); + fprintf(outf, "\n"); + } + return retval; +} + +#define PRINT_ARRAY(str, ndim, dims, loopvar, format) \ + fprintf(outf, "%s", str); \ + if (ndim > 0) \ + { \ + fprintf(outf, "{%" #format, dims[0]); \ + for (loopvar = 1; loopvar < ndim; loopvar++) \ + { \ + fprintf(outf, ", %" #format, dims[loopvar]); \ + } \ + fprintf(outf, "}\n"); \ + } \ + else \ + { \ + fprintf(outf, "empty\n"); \ + } + +#define PRINT_ARRAY64(str, ndim, dims, loopvar) \ + fprintf(outf, "%s", str); \ + if (ndim > 0) \ + { \ + fprintf(outf, "{%" PRIu64, dims[0]); \ + for (loopvar = 1; loopvar < ndim; loopvar++) \ + { \ + fprintf(outf, ", %" PRIu64, dims[loopvar]); \ + } \ + fprintf(outf, "}\n"); \ + } \ + else \ + { \ + fprintf(outf, "empty\n"); \ + } + +void printMeshes(Engine *fp) +{ + fprintf(outf, "Mesh info: is not implemented in adios 2.x at the moment\n"); + return; + /* + int meshid, i, j; // loop vars + int mpi_comm_dummy = 0; + if (fp->nmeshes == 0) + { + fprintf(outf, "Mesh info: There are no meshes defined in this file\n"); + return; + } + fprintf(outf, "Mesh info: \n"); + for (meshid = 0; meshid < fp->nmeshes; meshid++) + { + fprintf(outf, " %s\n", fp->mesh_namelist[meshid]); + ADIOS_MESH *mi = adios_inq_mesh_byid(fp, meshid); + if (mi) + { + if (meshid != mi->id) + fprintf( + outf, + " bpls warning: meshid (=%d) != inquired mesh id (%d)\n", + meshid, mi->id); + if (strcmp(fp->mesh_namelist[meshid], mi->name)) + fprintf(outf, " bpls warning: mesh name in list (=\"%s\") != " + "inquired mesh name (\"%s\")\n", + fp->mesh_namelist[meshid], mi->name); + if (mi->file_name) + { + IO &io = adios.DeclareIO("mesh"); + Engine &meshfp = io->Open(mi->file_name, Mode::Read); + adios_complete_meshinfo(fp, meshfp, mi); + meshfp.Close(); + } + fprintf(outf, " type: "); + + switch (mi->type) + { + case ADIOS_MESH_UNIFORM: + fprintf(outf, "uniform\n"); + PRINT_ARRAY64(" dimensions: ", mi->uniform->num_dimensions, + mi->uniform->dimensions, j) + if (mi->uniform->origins) + { + PRINT_ARRAY(" origins: ", + mi->uniform->num_dimensions, + mi->uniform->origins, j, g) + } + if (mi->uniform->spacings) + { + PRINT_ARRAY(" spacings: ", + mi->uniform->num_dimensions, + mi->uniform->spacings, j, g) + } + if (mi->uniform->maximums) + { + PRINT_ARRAY(" maximums: ", + mi->uniform->num_dimensions, + mi->uniform->maximums, j, g) + } + break; + + case ADIOS_MESH_RECTILINEAR: + fprintf(outf, "rectilinear\n"); + PRINT_ARRAY64(" dimensions: ", + mi->rectilinear->num_dimensions, + mi->rectilinear->dimensions, j) + if (mi->rectilinear->use_single_var) + { + fprintf(outf, " coordinates: single-var: \"%s\"\n", + mi->rectilinear->coordinates[0]); + } + else + { + fprintf(outf, " coordinates: multi-var: \"%s\"", + mi->rectilinear->coordinates[0]); + for (i = 1; i < mi->rectilinear->num_dimensions; i++) + { + fprintf(outf, ", \"%s\"", + mi->rectilinear->coordinates[i]); + } + fprintf(outf, "\n"); + } + break; + + case ADIOS_MESH_STRUCTURED: + fprintf(outf, "structured\n"); + PRINT_ARRAY64(" dimensions: ", + mi->structured->num_dimensions, + mi->structured->dimensions, j); + if (mi->structured->use_single_var) + { + fprintf(outf, " points: single-var: \"%s\"\n", + mi->structured->points[0]); + } + else + { + fprintf(outf, " points: multi-var: \"%s\"", + mi->structured->points[0]); + for (i = 1; i < mi->structured->num_dimensions; i++) + { + fprintf(outf, ", \"%s\"", mi->structured->points[i]); + } + fprintf(outf, "\n"); + } + fprintf(outf, " nspaces: %d\n", + mi->structured->nspaces); + break; + + case ADIOS_MESH_UNSTRUCTURED: + fprintf(outf, "unstructured\n"); + if (mi->unstructured->nvar_points <= 1) + { + fprintf(outf, " npoints: %" PRIu64 "\n", + mi->unstructured->npoints); + fprintf(outf, " points: single-var: \"%s\"\n", + mi->unstructured->points[0]); + } + else + { + fprintf(outf, " points: multi-var: \"%s\"", + mi->unstructured->points[0]); + for (i = 1; i < mi->unstructured->nvar_points; i++) + { + fprintf(outf, ", \"%s\"", mi->unstructured->points[i]); + } + fprintf(outf, "\n"); + } + fprintf(outf, " ncsets: %d\n", + mi->unstructured->ncsets); + for (i = 0; i < mi->unstructured->ncsets; i++) + { + fprintf(outf, " cell set %d:\n", i); + fprintf(outf, " cell type: %d\n", + mi->unstructured->ctypes[i]); + fprintf(outf, " ncells: %" PRIu64 "\n", + mi->unstructured->ccounts[i]); + fprintf(outf, " cells var: \"%s\"\n", + mi->unstructured->cdata[i]); + } + fprintf(outf, " nspaces: %d\n", + mi->unstructured->nspaces); + break; + + default: + fprintf(outf, "undefined\n"); + } + fprintf(outf, " time varying: %s\n", + (mi->time_varying ? "yes" : "no")); + adios_free_meshinfo(mi); + } + } + fprintf(outf, "\n"); + */ +} + +std::vector getEnginesList(const std::string path) +{ + std::vector list; + size_t slen = path.length(); +#ifdef ADIOS2_HAVE_HDF5 + if (slen >= 3 && path.compare(slen - 3, 3, ".h5") == 0) + { + list.push_back("HDF5"); + list.push_back("BPFile"); + } + else + { + list.push_back("BPFile"); + list.push_back("HDF5"); + } +#else + list.push_back("BPFile"); +#endif + return list; +} + +int doList(const char *path) +{ + int grpid; + int status; + int mpi_comm_dummy = 0; + int nGroupsMatched = 0; + int nGroups; // number of groups + char **group_namelist; + char init_params[128]; + int adios_verbose = 2; + + if (verbose > 1) + printf("\nADIOS Open: read header info from %s\n", path); + + // initialize BP reader + if (verbose > 1) + adios_verbose = 3; // print info lines + if (verbose > 2) + adios_verbose = 4; // print debug lines + sprintf(init_params, "verbose=%d", adios_verbose); + if (hidden_attrs) + strcat(init_params, ";show_hidden_attrs"); + + ADIOS adios(true); + IO &io = adios.DeclareIO("bpls"); + Engine *fp = nullptr; + std::vector engineList = getEnginesList(path); + for (auto &engineName : engineList) + { + if (verbose > 2) + printf("Try %s engine to open the file...\n", engineName.c_str()); + io.SetEngine(engineName); + try + { + fp = &io.Open(path, Mode::Read); + } + catch (std::exception &e) + { + if (verbose > 2) + printf("Failed to open with %s engine.\n", engineName.c_str()); + } + if (fp != nullptr) + break; + } + + if (fp != nullptr) + { + //, variables, timesteps, and attributes + // all parameters are integers, + // besides the last parameter, which is an array of strings for holding + // the + // list of group names + // ntsteps = fp->tidx_stop - fp->tidx_start + 1; + if (verbose) + { + const std::map &variablesInfo = + io.GetAvailableVariables(); + const std::map &attributesInfo = + io.GetAvailableAttributes(); + printf("File info:\n"); + printf(" of variables: %zu\n", variablesInfo.size()); + printf(" of attributes: %zu\n", attributesInfo.size()); + // printf(" of meshes: %d\n", fp->nmeshes); + // print_file_size(fp->file_size); + // printf(" bp version: %d\n", fp->version); + // printf(" endianness: %s\n", + // (fp->endianness ? "Big Endian" : "Little Endian")); + if (longopt) + printf(" statistics: Min / Max \n"); + printf("\n"); + } + + // Print out the meshes in the file + if (listmeshes) + { + printMeshes(fp); + } + + doList_vars(fp, &io); + + if (nmasks > 0 && nEntriesMatched == 0) + { + fprintf(stderr, + "\nError: None of the variables/attributes matched any " + "name/regexp you provided\n"); + return 4; + } + fp->Close(); + } + else + { + fprintf(stderr, "\nError: Could not open this file with any ADIOS2 " + "file reading engines\n"); + return 4; + } + return 0; +} + +/* +int print_data_hist(ADIOS_VARINFO *vi, char *varname) +{ + char hist_file[256], gnuplot_file[256]; + int i; + char xtics[512], str[512]; + FILE *out_hist, *out_plot; + struct ADIOS_HIST *h = vi->statistics->histogram; + + memcpy(hist_file, varname, strlen(varname) + 1); + strcat(hist_file, ".hist"); + + if ((out_hist = fopen(hist_file, "w")) == NULL) + { + fprintf(stderr, "Error at opening for writing file %s: %s\n", +hist_file, + strerror(errno)); + return 30; + } + + memcpy(gnuplot_file, varname, strlen(varname) + 1); + strcat(gnuplot_file, ".gpl"); + + if ((out_plot = fopen(gnuplot_file, "w")) == NULL) + { + fprintf(stderr, "Error at opening for writing file %s: %s\n", + gnuplot_file, strerror(errno)); + return 30; + } + + xtics[0] = '\0'; + strcat(xtics, "set xtics offset start axis ("); + for (i = 0; i <= h->num_breaks; i++) + { + if (i == 0) + { + fprintf(out_hist, "-Inf %.2lf %u\n", h->breaks[i], + h->gfrequencies[i]); + sprintf(str, "\"-Inf\" pos(%d)", i); + } + else if (i < h->num_breaks) + { + fprintf(out_hist, "%.2lf %.2lf %u\n", h->breaks[i - 1], + h->breaks[i], h->gfrequencies[i]); + sprintf(str, ", \"%.2lf\" pos(%d)", h->breaks[i - 1], i); + } + else + { + fprintf(out_hist, "%.2lf Inf %u\n", h->breaks[i], + h->gfrequencies[i]); + sprintf(str, ", \"Inf\" pos(%d)", i); + } + strcat(xtics, str); + } + strcat(xtics, ")\n"); + + fprintf(out_plot, "start = -0.5\npos(x) = start + x * 1\nset boxwidth " + "1\nset style fill solid border 5#5lt6#6\n"); + fputs(xtics, out_plot); + fprintf(out_plot, "plot '%s' using 3 smooth frequency w boxes\n", + hist_file); + fprintf(out_plot, "pause -1 'Press Enter to quit'\n"); + return 0; +} +*/ + +int cmpstringp(const void *p1, const void *p2) +{ + /* The actual arguments to this function are "pointers to + pointers to char", but strcmp() arguments are "pointers + to char", hence the following cast plus dereference */ + return strcmp(*(char *const *)p1, *(char *const *)p2); +} +/** Merge listV with listA if listattrs=true, return listA if + attrsonly=true, + otherwise just return listV. + If sortnames=true, quicksort the result list. + */ +void mergeLists(int nV, char **listV, int nA, char **listA, char **mlist, + bool *isVar) +{ + int v, a, idx; + if (sortnames && listattrs && !attrsonly) + { + // merge sort the two lists + v = 0; + a = 0; + while (v < nV || a < nA) + { + if (a < nA && (v >= nV || strcmp(listV[v], listA[a]) > 0)) + { + // fully consumed var list or + // next item in attr list is less than next item in var list + mlist[v + a] = listA[a]; + isVar[v + a] = false; + a++; + } + else + { + mlist[v + a] = listV[v]; + isVar[v + a] = true; + v++; + } + } + } + else + { + // first add vars then attrs (if ask ed) + idx = 0; + if (!attrsonly) + { + for (v = 0; v < nV; v++) + { + mlist[idx] = listV[v]; + isVar[idx] = true; + idx++; + } + } + if (listattrs) + { + for (a = 0; a < nA; a++) + { + mlist[idx] = listA[a]; + isVar[idx] = false; + idx++; + } + } + } +} + +int getTypeInfo(enum ADIOS_DATATYPES adiosvartype, int *elemsize) +{ + switch (adiosvartype) + { + case adios_unsigned_byte: + *elemsize = 1; + break; + case adios_byte: + *elemsize = 1; + break; + case adios_string: + *elemsize = 1; + break; + + case adios_unsigned_short: + *elemsize = 2; + break; + case adios_short: + *elemsize = 2; + break; + + case adios_unsigned_integer: + *elemsize = 4; + break; + case adios_integer: + *elemsize = 4; + break; + + case adios_unsigned_long: + *elemsize = 8; + break; + case adios_long: + *elemsize = 8; + break; + + case adios_real: + *elemsize = 4; + break; + + case adios_double: + *elemsize = 8; + break; + + case adios_complex: + *elemsize = 8; + break; + + case adios_double_complex: + *elemsize = 16; + break; + + case adios_long_double: // do not know how to print + //*elemsize = 16; + default: + return 1; + } + return 0; +} + +/** Read data of a variable and print + * Return: 0: ok, != 0 on error + */ +template +int readVar(Engine *fp, IO *io, Variable *variable) +{ + int i, j; + uint64_t start_t[MAX_DIMS], + count_t[MAX_DIMS]; // processed <0 values in start/count + uint64_t s[MAX_DIMS], c[MAX_DIMS]; // for block reading of smaller chunks + int tdims; // number of dimensions including time + int tidx; // 0 or 1 to account for time dimension + uint64_t nelems; // number of elements to read + // size_t elemsize; // size in bytes of one element + uint64_t st, ct; + T *data; + uint64_t sum; // working var to sum up things + int maxreadn; // max number of elements to read once up to a limit (10MB + // of + // data) + int actualreadn; // our decision how much to read at once + int readn[MAX_DIMS]; // how big chunk to read in in each dimension? + int status; + bool incdim; // used in incremental reading in + int ndigits_dims[32]; // # of digits (to print) of each dimension + + const size_t elemsize = variable->m_ElementSize; + const int ndim = variable->m_Shape.size(); + const int nsteps = variable->GetAvailableStepsCount(); + + // create the counter arrays with the appropriate lengths + // transfer start and count arrays to format dependent arrays + + nelems = 1; + tidx = 0; + + if (nsteps > 1) + { + if (istart[0] < 0) // negative index means last-|index| + st = nsteps + istart[0]; + else + st = istart[0]; + if (icount[0] < 0) // negative index means last-|index|+1-start + ct = nsteps + icount[0] + 1 - st; + else + ct = icount[0]; + + if (verbose > 2) + printf(" j=0, st=%" PRIu64 " ct=%" PRIu64 "\n", st, ct); + + start_t[0] = st; + count_t[0] = ct; + nelems *= ct; + if (verbose > 1) + printf(" s[0]=%" PRIu64 ", c[0]=%" PRIu64 ", n=%" PRIu64 "\n", + start_t[0], count_t[0], nelems); + + tidx = 1; + } + tdims = variable->m_Shape.size() + tidx; + + for (j = 0; j < variable->m_Shape.size(); j++) + { + if (istart[j + tidx] < 0) // negative index means last-|index| + st = variable->m_Shape[j] + istart[j + tidx]; + else + st = istart[j + tidx]; + if (icount[j + tidx] < 0) // negative index means last-|index|+1-start + ct = variable->m_Shape[j] + icount[j + tidx] + 1 - st; + else + ct = icount[j + tidx]; + + if (verbose > 2) + printf(" j=%d, st=%" PRIu64 " ct=%" PRIu64 "\n", j + tidx, st, + ct); + + start_t[j + tidx] = st; + count_t[j + tidx] = ct; + nelems *= ct; + if (verbose > 1) + printf(" s[%d]=%" PRIu64 ", c[%d]=%" PRIu64 ", n=%" PRIu64 "\n", + j + tidx, start_t[j + tidx], j + tidx, count_t[j + tidx], + nelems); + } + + if (verbose > 1) + { + printf(" total size of data to read = %" PRIu64 "\n", + nelems * elemsize); + } + + print_slice_info(variable, start_t, count_t); + + maxreadn = MAX_BUFFERSIZE / elemsize; + if (nelems < maxreadn) + maxreadn = nelems; + + // special case: string. Need to use different elemsize + /*if (vi->type == adios_string) + { + if (vi->value) + elemsize = strlen(vi->value) + 1; + maxreadn = elemsize; + }*/ + + // allocate data array + data = (T *)malloc(maxreadn * elemsize); + + // determine strategy how to read in: + // - at once + // - loop over 1st dimension + // - loop over 1st & 2nd dimension + // - etc + if (verbose > 1) + printf("Read size strategy:\n"); + sum = (uint64_t)1; + actualreadn = (uint64_t)1; + for (i = tdims - 1; i >= 0; i--) + { + if (sum >= (uint64_t)maxreadn) + { + readn[i] = 1; + } + else + { + readn[i] = maxreadn / (int)sum; // sum is small for 4 bytes here + // this may be over the max count for this dimension + if (readn[i] > count_t[i]) + readn[i] = count_t[i]; + } + if (verbose > 1) + printf(" dim %d: read %d elements\n", i, readn[i]); + sum = sum * (uint64_t)count_t[i]; + actualreadn = actualreadn * readn[i]; + } + if (verbose > 1) + printf(" read %d elements at once, %" PRIu64 + " in total (nelems=%" PRIu64 ")\n", + actualreadn, sum, nelems); + + // init s and c + // and calculate ndigits_dims + for (j = 0; j < tdims; j++) + { + s[j] = start_t[j]; + c[j] = readn[j]; + + ndigits_dims[j] = ndigits(start_t[j] + count_t[j] - + 1); // -1: dim=100 results in 2 digits (0..99) + } + + // read until read all 'nelems' elements + sum = 0; + while (sum < nelems) + { + + // how many elements do we read in next? + actualreadn = 1; + for (j = 0; j < tdims; j++) + actualreadn *= c[j]; + + if (verbose > 2) + { + printf("adios_read_var name=%s ", variable->m_Name.c_str()); + PRINT_DIMS_UINT64(" start", s, tdims, j); + PRINT_DIMS_UINT64(" count", c, tdims, j); + printf(" read %d elems\n", actualreadn); + } + + // read a slice finally + Dims startv = Uint64ArrayToSizetVector(tdims - tidx, s + tidx); + Dims countv = Uint64ArrayToSizetVector(tdims - tidx, c + tidx); + if (verbose > 2) + { + printf("set selection: "); + PRINT_DIMS_UINT64(" start", startv.data(), tdims - tidx, j); + PRINT_DIMS_UINT64(" count", countv.data(), tdims - tidx, j); + printf("\n"); + } + variable->SetSelection({startv, countv}); + + if (nsteps > 1) + { + if (verbose > 2) + { + printf("set Step selection: from %zu read %zu steps\n", s[0], + c[0]); + } + variable->SetStepSelection({s[0], c[0]}); + } + fp->GetSync(*variable, data); + + // print slice + print_dataset(data, variable->m_Type, s, c, tdims, ndigits_dims); + + // prepare for next read + sum += actualreadn; + incdim = true; // largest dim should be increased + for (j = tdims - 1; j >= 0; j--) + { + if (incdim) + { + if (s[j] + c[j] == start_t[j] + count_t[j]) + { + // reached the end of this dimension + s[j] = start_t[j]; + c[j] = readn[j]; + incdim = true; // next smaller dim can increase too + } + else + { + // move up in this dimension up to total count + s[j] += readn[j]; + if (s[j] + c[j] > start_t[j] + count_t[j]) + { + // do not reach over the limit + c[j] = start_t[j] + count_t[j] - s[j]; + } + incdim = false; + } + } + } + } // end while sum < nelems + print_endline(); + + free(data); + return 0; +} + +/** Read one writeblock of a variable and print + * Return: 0: ok, != 0 on error + */ +template +int readVarBlock(Engine *fp, IO *io, Variable *variable, int blockid) +{ + int i, j; + uint64_t start_t[MAX_DIMS], + count_t[MAX_DIMS]; // processed <0 values in start/count + uint64_t s[MAX_DIMS], c[MAX_DIMS]; // for block reading of smaller chunks + uint64_t nelems; // number of elements to read + int elemsize; // size in bytes of one element + int tidx; + uint64_t st, ct; + void *data; + uint64_t sum; // working var to sum up things + int maxreadn; // max number of elements to read once up to a limit (10MB + // of + // data) + int actualreadn; // our decision how much to read at once + int readn[MAX_DIMS]; // how big chunk to read in in each dimension? + int status; + bool incdim; // used in incremental reading in + int ndigits_dims[32]; // # of digits (to print) of each dimension + +#if 0 + if (getTypeInfo(vi->type, &elemsize)) + { + fprintf(stderr, "Adios type %d (%s) not supported in bpls. var=%s\n", + vi->type, adios_type_to_string(vi->type), name); + return 10; + } + + if (blockid < 0 || blockid > vi->sum_nblocks) + { + fprintf(stderr, + "Invalid block id for var=%s, id=%d, available %d blocks\n", + name, blockid, vi->sum_nblocks); + return 10; + } + + // create the counter arrays with the appropriate lengths + // transfer start and count arrays to format dependent arrays + + nelems = 1; + tidx = 0; + + if (timed) + { + if (istart[0] < 0) // negative index means last-|index| + st = nsteps + istart[0]; + else + st = istart[0]; + if (icount[0] < 0) // negative index means last-|index|+1-start + ct = nsteps + icount[0] + 1 - st; + else + ct = icount[0]; + + if (verbose > 2) + printf(" time: st=%" PRIu64 " ct=%" PRIu64 "\n", st, ct); + + // check if this block falls into the requested time + int idx = 0; + for (i = 0; i < st; i++) + { + idx += vi->nblocks[i]; + } + if (blockid < idx) + return 0; + for (i = st; i < st + ct; i++) + { + idx += vi->nblocks[i]; + } + if (blockid > idx) + return 0; + tidx = 1; + } + + int out_of_bound = 0; + for (j = 0; j < vi->ndim; j++) + { + if (istart[j + tidx] < 0) // negative index means last-|index| + st = vi->blockinfo[blockid].count[j] + istart[j + tidx]; + else + st = istart[j + tidx]; + if (icount[j + tidx] < 0) // negative index means last-|index|+1-start + ct = vi->blockinfo[blockid].count[j] + icount[j + tidx] + 1 - st; + else + ct = icount[j + tidx]; + + if (st > vi->blockinfo[blockid].count[j]) + { + out_of_bound = 1; + } + else if (ct > vi->blockinfo[blockid].count[j] - st) + { + ct = vi->blockinfo[blockid].count[j] - st; + } + if (verbose > 2) + printf(" j=%d, st=%" PRIu64 " ct=%" PRIu64 "\n", j, st, ct); + + start_t[j] = st; + count_t[j] = ct; + nelems *= ct; + if (verbose > 1) + printf(" s[%d]=%" PRIu64 ", c[%d]=%" PRIu64 ", n=%" PRIu64 "\n", + j, start_t[j], j, count_t[j], nelems); + } + + if (verbose > 1) + { + printf(" total size of data to read = %" PRIu64 "\n", + nelems * elemsize); + } + + print_slice_info(vi->ndim, vi->blockinfo[blockid].count, false, vi->nsteps, + start_t, count_t); + + if (out_of_bound) + return 0; + + maxreadn = MAX_BUFFERSIZE / elemsize; + if (nelems < maxreadn) + maxreadn = nelems; + + // allocate data array + data = (void *)malloc(maxreadn * elemsize + 8); // +8 for just to be sure + + // determine strategy how to read in: + // - at once + // - loop over 1st dimension + // - loop over 1st & 2nd dimension + // - etc + if (verbose > 1) + printf("Read size strategy:\n"); + sum = (uint64_t)1; + actualreadn = (uint64_t)1; + for (i = vi->ndim - 1; i >= 0; i--) + { + if (sum >= (uint64_t)maxreadn) + { + readn[i] = 1; + } + else + { + readn[i] = maxreadn / (int)sum; // sum is small for 4 bytes here + // this may be over the max count for this dimension + if (readn[i] > count_t[i]) + readn[i] = count_t[i]; + } + if (verbose > 1) + printf(" dim %d: read %d elements\n", i, readn[i]); + sum = sum * (uint64_t)count_t[i]; + actualreadn = actualreadn * readn[i]; + } + if (verbose > 1) + printf(" read %d elements at once, %" PRIu64 + " in total (nelems=%" PRIu64 ")\n", + actualreadn, sum, nelems); + + // init s and c + // and calculate ndigits_dims + for (j = 0; j < vi->ndim; j++) + { + s[j] = start_t[j]; + c[j] = readn[j]; + + ndigits_dims[j] = ndigits(start_t[j] + count_t[j] - + 1); // -1: dim=100 results in 2 digits (0..99) + } + + // read until read all 'nelems' elements + sum = 0; + while (sum < nelems) + { + + // how many elements do we read in next? + actualreadn = 1; + for (j = 0; j < vi->ndim; j++) + actualreadn *= c[j]; + + uint64_t startoffset = s[vi->ndim - 1]; + uint64_t tmpprod = c[vi->ndim - 1]; + for (i = vi->ndim - 2; i >= 0; i--) + { + startoffset += s[i] * tmpprod; + tmpprod *= c[i]; + } + + if (verbose > 2) + { + printf("adios_read_var name=%s ", name); + PRINT_DIMS_UINT64(" start", s, vi->ndim, j); + PRINT_DIMS_UINT64(" count", c, vi->ndim, j); + printf(" read %d elems\n", actualreadn); + } + if (verbose > 1) + printf(" read block %d from offset %" PRIu64 " nelems %d)\n", + blockid, startoffset, actualreadn); + + // read a slice finally + ADIOS_SELECTION *wb = adios_selection_writeblock(blockid); + wb->u.block.is_absolute_index = true; + wb->u.block.is_sub_pg_selection = 1; + wb->u.block.element_offset = startoffset; + wb->u.block.nelements = actualreadn; + status = adios_schedule_read_byid(fp, wb, vi->varid, 0, 1, data); + + if (status < 0) + { + fprintf(stderr, "Error when scheduling variable %s for reading. " + "errno=%d : %s \n", + name, adios_errno, adios_errmsg()); + free(data); + return 11; + } + + status = adios_perform_reads(fp, 1); // blocking read performed here + adios_selection_delete(wb); + if (status < 0) + { + fprintf(stderr, "Error when reading variable %s. errno=%d : %s \n", + name, adios_errno, adios_errmsg()); + free(data); + return 11; + } + + // print slice + print_dataset(data, vi->type, s, c, vi->ndim, ndigits_dims); + + // prepare for next read + sum += actualreadn; + incdim = true; // largest dim should be increased + for (j = vi->ndim - 1; j >= 0; j--) + { + if (incdim) + { + if (s[j] + c[j] == start_t[j] + count_t[j]) + { + // reached the end of this dimension + s[j] = start_t[j]; + c[j] = readn[j]; + incdim = true; // next smaller dim can increase too + } + else + { + // move up in this dimension up to total count + s[j] += readn[j]; + if (s[j] + c[j] > start_t[j] + count_t[j]) + { + // do not reach over the limit + c[j] = start_t[j] + count_t[j] - s[j]; + } + incdim = false; + } + } + } + } // end while sum < nelems + print_endline(); + + free(data); +#else + printf("Block reading is not implemented yet\n"); +#endif + return 0; +} + +bool matchesAMask(const char *name) +{ + int excode; + int i; + int startpos = 0; // to match with starting / or without + regmatch_t pmatch[1] = {{(regoff_t)-1, (regoff_t)-1}}; + + if (nmasks == 0) + return true; + + for (i = 0; i < nmasks; i++) + { + if (use_regexp) + { + excode = regexec(&(varregex[i]), name, 1, pmatch, 0); + if (name[0] == '/') // have to check if it matches from the second + // character too + startpos = 1; + if (excode == 0 && // matches + (pmatch[0].rm_so == 0 || + pmatch[0].rm_so == startpos) && // from the beginning + pmatch[0].rm_eo == strlen(name) // to the very end of the name + ) + { + if (verbose > 1) + printf("Name %s matches regexp %i %s\n", name, i, + varmask[i]); + // printf("Match from %d to %d\n", (int) pmatch[0].rm_so, + // (int) + // pmatch[0].rm_eo); + return true; + } + } + else + { + // use shell pattern matching + if (varmask[i][0] != '/' && name[0] == '/') + startpos = 1; + if (fnmatch(varmask[i], name + startpos, FNM_FILE_NAME) == 0) + { + if (verbose > 1) + printf("Name %s matches varmask %i %s\n", name, i, + varmask[i]); + return true; + } + } + } + return false; +} + +int print_start(const char *fname) +{ + if (fname == NULL) + { + outf = stdout; + } + else + { + if ((outf = fopen(fname, "w")) == NULL) + { + fprintf(stderr, "Error at opening for writing file %s: %s\n", fname, + strerror(errno)); + return 30; + } + } + return 0; +} + +void print_stop() { fclose(outf); } + +static int nextcol = + 0; // column index to start with (can have lines split in two calls) + +void print_slice_info(VariableBase *variable, uint64_t *s, uint64_t *c) +{ + // print the slice info in indexing is on and + // not the complete variable is read + size_t ndim = variable->m_Shape.size(); + size_t nsteps = variable->m_AvailableStepsCount; + int i; + bool isaslice = false; + int tidx = (nsteps > 1 ? 1 : 0); + size_t tdim = ndim + tidx; + if (nsteps > 1) + { + if (c[0] < nsteps) + isaslice = true; + } + for (i = 0; i < ndim; i++) + { + if (c[i + tidx] < variable->m_Shape[i]) + isaslice = true; + } + if (isaslice) + { + fprintf(outf, "%c slice (%" PRIu64 ":%" PRIu64, commentchar, s[0], + s[0] + c[0] - 1); + for (i = 1; i < tdim; i++) + { + fprintf(outf, ", %" PRIu64 ":%" PRIu64, s[i], s[i] + c[i] - 1); + } + fprintf(outf, ")\n"); + } +} + +const std::map adios_types_map = { + {"char", adios_unsigned_byte}, + {"int", adios_integer}, + {"float", adios_real}, + {"double", adios_double}, + {"float complex", adios_complex}, + {"double complex", adios_double_complex}, + {"signed char", adios_byte}, + {"short", adios_short}, + {"long int", adios_long}, + {"long long int", adios_long}, + {"string", adios_string}, + {"string array", adios_string_array}, + {"unsigned char", adios_unsigned_byte}, + {"unsigned short", adios_unsigned_short}, + {"unsigned int", adios_unsigned_integer}, + {"unsigned long int", adios_unsigned_long}, + {"unsigned long long int", adios_unsigned_long}}; + +enum ADIOS_DATATYPES type_to_enum(std::string type) +{ + enum ADIOS_DATATYPES t; + auto itType = adios_types_map.find(type); + if (itType == adios_types_map.end()) + { + return adios_unknown; + } + return itType->second; +} + +int print_data_as_string(const void *data, int maxlen, + enum ADIOS_DATATYPES adiosvartype) +{ + const char *str = (const char *)data; + int len = maxlen; + switch (adiosvartype) + { + case adios_unsigned_byte: + case adios_byte: + case adios_string: + while (str[len - 1] == 0) + { + len--; + } // go backwards on ascii 0s + if (len < maxlen) + { + // it's a C string with terminating \0 + fprintf(outf, "\"%s\"", str); + } + else + { + // fortran VARCHAR, lets trim from right padded zeros + while (str[len - 1] == ' ') + { + len--; + } + fprintf(outf, "\"%*.*s\"", len, len, (char *)data); + if (len < maxlen) + fprintf(outf, " + %d spaces", maxlen - len); + } + break; + default: + fprintf(stderr, "Error in bpls code: cannot use print_data_as_string() " + "for type \"%d\"\n", + adiosvartype); + return -1; + break; + } + return 0; +} + +int print_data_characteristics(void *min, void *max, double *avg, + double *std_dev, + enum ADIOS_DATATYPES adiosvartype, + bool allowformat) +{ + bool f = formatgiven && allowformat; + + switch (adiosvartype) + { + case adios_unsigned_byte: + if (min) + fprintf(outf, (f ? format : "%10hhu "), *((unsigned char *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10hhu "), *((unsigned char *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2f ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2f ", *std_dev); + else + fprintf(outf, " null "); + break; + case adios_byte: + if (min) + fprintf(outf, (f ? format : "%10hhd "), *((char *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10hhd "), *((char *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2f ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2f ", *std_dev); + else + fprintf(outf, " null "); + break; + case adios_string: + break; + + case adios_unsigned_short: + if (min) + fprintf(outf, (f ? format : "%10hu "), (*(unsigned short *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10hu "), (*(unsigned short *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2f ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2f ", *std_dev); + else + fprintf(outf, " null "); + break; + case adios_short: + if (min) + fprintf(outf, (f ? format : "%10hd "), (*(short *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10hd "), (*(short *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2f ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2f ", *std_dev); + else + fprintf(outf, " null "); + break; + + case adios_unsigned_integer: + if (min) + fprintf(outf, (f ? format : "%10u "), (*(unsigned int *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10u "), (*(unsigned int *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2f ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2f ", *std_dev); + else + fprintf(outf, " null "); + break; + case adios_integer: + if (min) + fprintf(outf, (f ? format : "%10d "), (*(int *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10d "), (*(int *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2f ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2f ", *std_dev); + else + fprintf(outf, " null "); + break; + + case adios_unsigned_long: + if (min) + fprintf(outf, (f ? format : "%10llu "), + (*(unsigned long long *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10llu "), + (*(unsigned long long *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2f ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2f ", *std_dev); + else + fprintf(outf, " null "); + break; + case adios_long: + if (min) + fprintf(outf, (f ? format : "%10lld "), (*(long long *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10lld "), (*(long long *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2f ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2f ", *std_dev); + else + fprintf(outf, " null "); + break; + + case adios_real: + if (min) + fprintf(outf, (f ? format : "%10.2g "), (*(float *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10.2g "), (*(float *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2g ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2g ", *std_dev); + else + fprintf(outf, " null "); + break; + case adios_double: + if (min) + fprintf(outf, (f ? format : "%10.2g "), (*(double *)min)); + else + fprintf(outf, " null "); + if (max) + fprintf(outf, (f ? format : "%10.2g "), (*(double *)max)); + else + fprintf(outf, " null "); + if (avg) + fprintf(outf, "%10.2g ", *avg); + else + fprintf(outf, " null "); + if (std_dev) + fprintf(outf, "%10.2g ", *std_dev); + else + fprintf(outf, " null "); + break; + + case adios_long_double: + fprintf(outf, "????????"); + break; + + // TO DO + /* + case adios_complex: + fprintf(outf,(f ? format : "(%g,i%g) "), ((float *) data)[2*item], + ((float *) data)[2*item+1]); + break; + + case adios_double_complex: + fprintf(outf,(f ? format : "(%g,i%g)" ), ((double *) data)[2*item], + ((double *) data)[2*item+1]); + break; + */ + default: + break; + } // end switch + return 0; +} + +int print_data(const void *data, int item, enum ADIOS_DATATYPES adiosvartype, + bool allowformat) +{ + bool f = formatgiven && allowformat; + if (data == NULL) + { + fprintf(outf, "null "); + return 0; + } + // print next data item + switch (adiosvartype) + { + case adios_unsigned_byte: + fprintf(outf, (f ? format : "%hhu"), ((unsigned char *)data)[item]); + break; + case adios_byte: + fprintf(outf, (f ? format : "%hhd"), ((signed char *)data)[item]); + break; + + case adios_string: + fprintf(outf, (f ? format : "\"%s\""), ((char *)data) + item); + break; + case adios_string_array: + // we expect one elemet of the array here + fprintf(outf, (f ? format : "\"%s\""), *((char **)data + item)); + break; + + case adios_unsigned_short: + fprintf(outf, (f ? format : "%hu"), ((unsigned short *)data)[item]); + break; + case adios_short: + fprintf(outf, (f ? format : "%hd"), ((signed short *)data)[item]); + break; + + case adios_unsigned_integer: + fprintf(outf, (f ? format : "%u"), ((unsigned int *)data)[item]); + break; + case adios_integer: + fprintf(outf, (f ? format : "%d"), ((signed int *)data)[item]); + break; + + case adios_unsigned_long: + fprintf(outf, (f ? format : "%llu"), + ((unsigned long long *)data)[item]); + break; + case adios_long: + fprintf(outf, (f ? format : "%lld"), ((signed long long *)data)[item]); + break; + + case adios_real: + fprintf(outf, (f ? format : "%g"), ((float *)data)[item]); + break; + + case adios_double: + fprintf(outf, (f ? format : "%g"), ((double *)data)[item]); + break; + + case adios_long_double: + fprintf(outf, (f ? format : "%Lg"), ((long double *)data)[item]); + // fprintf(outf,(f ? format : "????????")); + break; + + case adios_complex: + fprintf(outf, (f ? format : "(%g,i%g)"), ((float *)data)[2 * item], + ((float *)data)[2 * item + 1]); + break; + + case adios_double_complex: + fprintf(outf, (f ? format : "(%g,i%g)"), ((double *)data)[2 * item], + ((double *)data)[2 * item + 1]); + break; + + default: + break; + } // end switch + return 0; +} + +int print_dataset(const void *data, const std::string vartype, uint64_t *s, + uint64_t *c, int tdims, int *ndigits) +{ + int i, item, steps; + char idxstr[128], buf[16]; + uint64_t ids[MAX_DIMS]; // current indices + bool roll; + enum ADIOS_DATATYPES adiosvartype = type_to_enum(vartype); + + // init current indices + steps = 1; + for (i = 0; i < tdims; i++) + { + ids[i] = s[i]; + steps *= c[i]; + } + + item = 0; // index to *data + // loop through each data item and print value + while (item < steps) + { + + // print indices if needed into idxstr; + idxstr[0] = '\0'; // empty idx string + if (nextcol == 0) + { + if (!noindex && tdims > 0) + { + sprintf(idxstr, " (%*" PRIu64, ndigits[0], ids[0]); + for (i = 1; i < tdims; i++) + { + sprintf(buf, ",%*" PRIu64, ndigits[i], ids[i]); + strcat(idxstr, buf); + } + strcat(idxstr, ") "); + } + } + + // print item + fprintf(outf, "%s", idxstr); + if (printByteAsChar && + (adiosvartype == adios_byte || adiosvartype == adios_unsigned_byte)) + { + /* special case: k-D byte array printed as (k-1)D array of + * strings + */ + if (tdims == 0) + { + print_data_as_string(data, steps, adiosvartype); + } + else + { + print_data_as_string( + (char *)data + item, c[tdims - 1], + adiosvartype); // print data of last dim as string + item += c[tdims - 1] - 1; // will be ++-ed once below + ids[tdims - 1] = + s[tdims - 1] + c[tdims - 1] - 1; // will be rolled below + } + nextcol = ncols - 1; // force new line, will be ++-ed once below + } + else + { + print_data(data, item, adiosvartype, true); + } + + // increment/reset column index + nextcol++; + if (nextcol == ncols) + { + fprintf(outf, "\n"); + nextcol = 0; + } + else + { + fprintf(outf, " "); + } + + // increment indices + item++; + roll = true; + for (i = tdims - 1; i >= 0; i--) + { + if (roll) + { + if (ids[i] == s[i] + c[i] - 1) + { + // last index in this dimension, roll upward + ids[i] = s[i]; + } + else + { + ids[i]++; + roll = false; + } + } + } + } + return 0; +} + +void print_endline(void) +{ + if (nextcol != 0) + fprintf(outf, "\n"); + nextcol = 0; +} + +template +void print_decomp(Engine *fp, IO *io, Variable *variable) +{ + /* Print block info */ + int blockid = 0; + int nsteps = variable->GetAvailableStepsCount(); + size_t ndim = variable->m_Shape.size(); + enum ADIOS_DATATYPES vartype = type_to_enum(variable->m_Type); + size_t nblocks = 1; /* FIXME: we need the number of blocks here */ + int ndigits_nsteps = ndigits(nsteps - 1); + if (ndim == 0) + { + // scalars + for (int i = 0; i < nsteps; i++) + { + fprintf(outf, " step %*d: ", ndigits_nsteps, i); +#if 0 + fprintf(outf, "%d instances available\n", vi->nblocks[i]); + if (dump && vi->statistics && vi->statistics->blocks) + { + fprintf(outf, " "); + if (vi->statistics->blocks->mins) + { + int col = 0; + for (int j = 0; j < vi->nblocks[i]; j++) + { + if (vartype == adios_complex || + vartype == adios_double_complex) + { + print_data(vi->statistics->blocks->mins[blockid], 0, + adios_double_complex, true); + } + else + { + print_data(vi->statistics->blocks->mins[blockid], 0, + vartype, true); + } + ++col; + if (j < vi->nblocks[i] - 1) + { + if (col < ncols) + { + fprintf(outf, " "); + } + else + { + fprintf(outf, "\n "); + col = 0; + } + } + ++blockid; + } + fprintf(outf, "\n"); + } + } +#else + fprintf(outf, "\n"); +#endif + } + return; + } + else + { + // arrays + int ndigits_nblocks; + int ndigits_procid; + int ndigits_time; + int ndigits_dims[32]; + for (int k = 0; k < ndim; k++) + { + // get digit lengths for each dimension + ndigits_dims[k] = ndigits(variable->m_Shape[k] - 1); + } + + for (int i = 0; i < nsteps; i++) + { + fprintf(outf, " step %*d: ", ndigits_nsteps, i); + fprintf(outf, "\n"); +#if 0 + ndigits_nblocks = ndigits(vi->nblocks[i] - 1); + ndigits_procid = + ndigits(vi->blockinfo[blockid + vi->nblocks[i] - 1].process_id); + ndigits_time = + ndigits(vi->blockinfo[blockid + vi->nblocks[i] - 1].time_index); + for (int j = 0; j < vi->nblocks[i]; j++) + { + if (verbose < 1) + { + fprintf(outf, " block %*d: [", ndigits_nblocks, j); + } + else + { + fprintf(outf, " block %*d proc %*u time %*u: [", + ndigits_nblocks, j, ndigits_procid, + vi->blockinfo[blockid].process_id, ndigits_time, + vi->blockinfo[blockid].time_index); + } + for (int k = 0; k < ndim; k++) + { + if (vi->blockinfo[blockid].count[k]) + { + fprintf(outf, "%*" PRIu64 ":%*" PRIu64, ndigits_dims[k], + vi->blockinfo[blockid].start[k], + ndigits_dims[k], + vi->blockinfo[blockid].start[k] + + vi->blockinfo[blockid].count[k] - 1); + } + else + { + fprintf(outf, "%-*s", 2 * ndigits_dims[k] + 1, "null"); + } + if (k < ndim - 1) + fprintf(outf, ", "); + } + fprintf(outf, "]"); + + /* Print per-block statistics if available */ + if (longopt && vi->statistics->blocks) + { + fprintf(outf, " = "); + if (vi->statistics->blocks->mins) + { + if (vartype == adios_complex || + vartype == adios_double_complex) + { + print_data(vi->statistics->blocks->mins[blockid], 0, + adios_double_complex, false); + } + else + { + print_data(vi->statistics->blocks->mins[blockid], 0, + vartype, false); + } + } + else + { + fprintf(outf, "N/A "); + } + + fprintf(outf, " / "); + if (vi->statistics->blocks->maxs) + { + if (vartype == adios_complex || + vartype == adios_double_complex) + { + print_data(vi->statistics->blocks->maxs[blockid], 0, + adios_double_complex, false); + } + else + { + print_data(vi->statistics->blocks->maxs[blockid], 0, + vartype, false); + } + } + else + { + fprintf(outf, "N/A "); + } + + fprintf(outf, "/ "); + if (vi->statistics->blocks->avgs) + { + if (vartype == adios_complex || + vartype == adios_double_complex) + { + print_data(vi->statistics->blocks->avgs[blockid], 0, + adios_double_complex, false); + } + else + { + print_data(vi->statistics->blocks->avgs[blockid], 0, + adios_double, false); + } + } + else + { + fprintf(outf, "N/A "); + } + + fprintf(outf, "/ "); + if (vi->statistics->blocks->avgs) + { + if (vartype == adios_complex || + vartype == adios_double_complex) + { + print_data( + vi->statistics->blocks->std_devs[blockid], 0, + adios_double_complex, false); + } + else + { + print_data( + vi->statistics->blocks->std_devs[blockid], 0, + adios_double, false); + } + } + else + { + fprintf(outf, "N/A "); + } + } + fprintf(outf, "\n"); + if (dump) + { + readVarBlock(fp, io, variable, blockid); + } + blockid++; + } +#endif + } + } +} + +// parse a string "0, 3; 027" into an integer array +// of [0,3,27] +// exits if parsing failes +void parseDimSpec(char *str, int64_t *dims) +{ + char *token, *saveptr; + char *s; // copy of s; strtok modifies the string + int i = 0; + + s = strndup(str, 256); + token = strtok_r(s, " ,;x\t\n", &saveptr); + while (token != NULL && i < MAX_DIMS) + { + // printf("\t|%s|", token); + errno = 0; + dims[i] = (int64_t)strtoll(token, (char **)NULL, 0); + if (errno) + { + fprintf(stderr, "Error: could not convert field into a value: " + "%s from \"%s\"\n", + token, str); + exit(200); + } + + // get next item + token = strtok_r(NULL, " ,;x\t\n", &saveptr); + i++; + } + // if (i>0) printf("\n"); + + if (i > ndimsspecified) + ndimsspecified = i; + + // check if number of dims specified is larger than we can handle + if (token != NULL) + { + fprintf(stderr, "Error: More dimensions specified in \"%s\" than we " + "can handle (%d)\n", + str, MAX_DIMS); + exit(200); + } +} + +int compile_regexp_masks(void) +{ + int i, errcode; + char buf[256]; + for (i = 0; i < nmasks; i++) + { + errcode = regcomp(&(varregex[i]), varmask[i], REG_EXTENDED); + if (errcode) + { + regerror(errcode, &(varregex[i]), buf, sizeof(buf)); + fprintf(stderr, "Error: var %s is an invalid extended regular " + "expression: %s\n", + varmask[i], buf); + return 2; + } + } + return 0; +} + +// end of namespace +} +} + +int main(int argc, char *argv[]) +{ + MPI_Init(&argc, &argv); + return adios2::utils::bplsMain(argc, argv); + MPI_Finalize(); +} diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h new file mode 100644 index 0000000000..ae27d55d03 --- /dev/null +++ b/source/utils/bpls/bpls.h @@ -0,0 +1,116 @@ +/* + * ADIOS is freely available under the terms of the BSD license described + * in the COPYING file in the top level directory of this source distribution. + * + * Copyright (c) 2008 - 2009. UT-BATTELLE, LLC. All rights reserved. + */ + +#include "adios2/ADIOSConfig.h" +#include "adios2/ADIOSMPI.h" +#include "adios2/ADIOSMPICommOnly.h" +#include "adios2/ADIOSMacros.h" +#include "adios2/core/ADIOS.h" +#include "adios2/core/IO.h" +#include "adios2/core/Variable.h" +#include "adios2/engine/bp/BPFileReader.h" +#include "adios2/helper/adiosFunctions.h" + +#include + +namespace adios2 +{ +namespace utils +{ + +/* definitions for bpls.c */ +#define myfree(p) \ + if (p) \ + { \ + free(p); \ + p = NULL; \ + } + +#define CUT_TO_BYTE(x) (x < 0 ? 0 : (x > 255 ? 255 : x)) + +#define MAX_DIMS 16 +#define MAX_MASKS 10 +#define MAX_BUFFERSIZE (10 * 1024 * 1024) + +/* global defines needed for the type creation/setup functions */ +enum ADIOS_DATATYPES +{ + adios_unknown = -1, + adios_byte = 0, + adios_short = 1, + adios_integer = 2, + adios_long = 4, + adios_unsigned_byte = 50, + adios_unsigned_short = 51, + adios_unsigned_integer = 52, + adios_unsigned_long = 54, + adios_real = 5, + adios_double = 6, + adios_long_double = 7, + adios_string = 9, + adios_complex = 10, + adios_double_complex = 11, + adios_string_array = 12 +}; + +struct Entry +{ + bool isVar; + std::string typeName; + unsigned int typeIndex; + Entry(bool b, std::string name, unsigned idx) + : isVar(b), typeName(name), typeIndex(idx) + { + } +}; + +// how to print one data item of an array +// enum PrintDataType {STRING, INT, FLOAT, DOUBLE, COMPLEX}; + +void init_globals(); +void processDimSpecs(); +void parseDimSpec(char *str, int64_t *dims); +int compile_regexp_masks(void); +void printSettings(void); +int doList(const char *path); +void mergeLists(int nV, char **listV, int nA, char **listA, char **mlist, + bool *isVar); + +enum ADIOS_DATATYPES type_to_enum(std::string type); + +template +int printVariableInfo(Engine *fp, IO *io, Variable *variable); + +template +int readVar(Engine *fp, IO *io, Variable *variable); + +template +int readVarBlock(Engine *fp, IO *io, Variable *variable, int blockid); + +int cmpstringp(const void *p1, const void *p2); +bool grpMatchesMask(char *name); +bool matchesAMask(const char *name); +int print_start(const char *fname); +void print_slice_info(VariableBase *variable, uint64_t *s, uint64_t *c); +int print_data(const void *data, int item, enum ADIOS_DATATYPES adiosvartypes, + bool allowformat); +int print_dataset(const void *data, const std::string vartype, uint64_t *s, + uint64_t *c, int tdims, int *ndigits); +void print_endline(void); +void print_stop(void); +int print_data_hist(VariableBase *vi, char *varname); +int print_data_characteristics(void *min, void *max, double *avg, + double *std_dev, + enum ADIOS_DATATYPES adiosvartypes, + bool allowformat); + +template +void print_decomp(Engine *fp, IO *io, Variable *variable); + +// close namespace +} +} diff --git a/source/utils/bpls2/BPLS2.cpp b/source/utils/bpls2/BPLS2.cpp deleted file mode 100644 index 1e13b72d86..0000000000 --- a/source/utils/bpls2/BPLS2.cpp +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPLS2.cpp - * - * Created on: Oct 5, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "BPLS2.h" - -#include -#include - -#include "adios2/ADIOSMPICommOnly.h" -#include "adios2/ADIOSMacros.h" -#include "adios2/core/ADIOS.h" -#include "adios2/core/IO.h" -#include "adios2/engine/bp/BPFileReader.h" -#include "adios2/helper/adiosFunctions.h" - -namespace adios2 -{ -namespace utils -{ - -const std::string BPLS2::m_HelpMessage = "For usage run either:\n" - "\t bpls2 --help\n" - "\t bpls2 -h \n"; - -const std::map BPLS2::m_Options = { - {"long", "l"}, {"attrs", "a"}, {"attrsonly", "A"}, - {"dump", "d"}, {"verbose", "v"}, {"help", "h"}}; - -BPLS2::BPLS2(int argc, char *argv[]) : Utils("bpls2", argc, argv) {} - -void BPLS2::Run() -{ - ParseArguments(); - ProcessParameters(); - ProcessTransport(); -} - -// PRIVATE -void BPLS2::ParseArguments() -{ - if (m_Arguments.size() == 1) - { - throw std::invalid_argument( - "ERROR: Missing adios2 bp file or arguments\n" + m_HelpMessage); - } - - bool isFileSet = false; - - for (auto itArg = m_Arguments.begin() + 1; itArg != m_Arguments.end(); - ++itArg) - { - if (itArg->find("--") == 0) // long argument - { - const std::string argument(itArg->substr(2)); - if (argument.size() == 1) - { - throw std::invalid_argument( - "ERROR: unknown single character option, did you mean -" + - argument + " ?\n"); - } - SetParameters(argument, true); - } - else if (itArg->find("-") == 0) // short argument - { - const std::string argument(itArg->substr(1)); - SetParameters(argument, false); - if (argument == "s" || argument == "c") - { - ++itArg; - // SetParameters(*itArg); - } - } - else - { - if (isFileSet) - { - throw std::invalid_argument( - "ERROR: only 1 bpfile is allowed\n" + m_HelpMessage); - } - - m_FileName = *itArg; - isFileSet = true; - } - } -} - -void BPLS2::ProcessParameters() const -{ - if (m_Parameters.count("help") == 1) - { - PrintUsage(); - if (m_Parameters.size() > 1) - { - std::cout << "\n"; - std::cout << "Found --help , -h option, discarding others\n"; - std::cout << "Rerun without --help , -h option\n"; - } - throw std::invalid_argument(""); - } - - if (m_FileName.empty()) - { - throw std::invalid_argument("ERROR: file not passed to bpls2\n" + - m_HelpMessage); - } -} - -void BPLS2::PrintUsage() const noexcept -{ - std::cout << "This is ADIOS2 binary pack listing (bpls2) utility. Usage:\n"; - std::cout << "\t bpls2 [OPTIONS] bpfile [MASK1 MASK2 ...]\n"; - std::cout << "\n"; - std::cout << "[OPTIONS]:\n"; - std::cout << "\n"; - std::cout << "-l , --long Print variables and attributes metadata\n"; - std::cout << " information, no overhead\n"; - std::cout << "-a , --attributes List attributes metadata\n"; - std::cout << "-v , --verbose Added file information\n"; - std::cout << "\n"; - std::cout << "Example: bpls2 -lav bpfile" << std::endl; -} - -void BPLS2::PrintExamples() const noexcept {} - -void BPLS2::SetParameters(const std::string argument, const bool isLong) -{ - if (argument == "-" || argument.empty()) - { - throw std::invalid_argument("ERROR: invalid argument: -" + argument + - "\n"); - } - bool isOption = false; - - if (m_Options.count(argument) == 1 && isLong) - { - isOption = true; - m_Parameters[argument] = ""; - } - else if (!isLong) - { - for (const auto &optionPair : m_Options) - { - if (argument == optionPair.second) - { - isOption = true; - m_Parameters[optionPair.first] = ""; - break; - } - } - } - - if (isOption) - { - return; - } - - // look for multiple options by character - for (const char argChar : argument) - { - const std::string argCharString(1, argChar); - isOption = false; - - for (const auto &optionPair : m_Options) - { - if (argCharString == optionPair.second) - { - m_Parameters[optionPair.first] = ""; - isOption = true; - } - } - - if (!isOption) - { - throw std::invalid_argument("ERROR: unknown option " + - argCharString + " in argument " + - argument + "\n"); - } - } -} - -void BPLS2::ProcessTransport() const -{ - auto lf_PrintVerboseHeader = [](const BPFileReader &bpFileReader) { - - const std::map variablesInfo = - bpFileReader.m_IO.GetAvailableVariables(); - - const std::map attributesInfo = - bpFileReader.m_IO.GetAvailableAttributes(); - - const auto &metadataSet = bpFileReader.m_BP3Deserializer.m_MetadataSet; - std::cout << "File info:\n"; - std::cout << " groups: " << metadataSet.DataPGCount << "\n"; - std::cout << " variables: " << variablesInfo.size() << "\n"; - std::cout << " attributes: " << attributesInfo.size() << "\n"; - std::cout << " meshes: TODO\n"; - std::cout << " steps: " << metadataSet.StepsCount << "\n"; - std::cout << " file size: " - << bpFileReader.m_FileManager.GetFileSize(0) << " bytes\n"; - - const auto &minifooter = bpFileReader.m_BP3Deserializer.m_Minifooter; - std::cout << " bp version: " << std::to_string(minifooter.Version) - << "\n"; - std::string endianness("Little Endian"); - if (!minifooter.IsLittleEndian) - { - endianness = "Big Endian"; - } - std::cout << " endianness: " << endianness << "\n"; - std::cout << " statistics: Min / Max\n"; - std::cout << "\n"; - }; - - auto lf_PrintVariables = [&](BPFileReader &bpFileReader) { - - const std::map &variablesInfo = - bpFileReader.m_IO.GetAvailableVariables(); - - size_t maxTypeSize = 0; - size_t maxNameSize = 0; - for (const auto &variablePair : variablesInfo) - { - const size_t nameSize = variablePair.first.size(); - if (nameSize > maxNameSize) - { - maxNameSize = nameSize; - } - - const Params ¶meters = variablePair.second; - const size_t typeSize = parameters.at("Type").size(); - if (typeSize > maxTypeSize) - { - maxTypeSize = typeSize; - } - } - - for (const auto &variablePair : variablesInfo) - { - const std::string name(variablePair.first); - const Params &variableParameters = variablePair.second; - const std::string type(variableParameters.at("Type")); - - std::cout << " "; - std::cout << std::left << std::setw(maxTypeSize) << type << " "; - std::cout << std::left << std::setw(maxNameSize) << name << " "; - - // print min max - if (m_Parameters.count("long") == 1) - { - if (variableParameters.at("SingleValue") == "false") - { - std::cout << variableParameters.at("AvailableStepsCount") - << "*{" << variableParameters.at("Shape") - << "} "; - std::cout << variableParameters.at("Min") << " / " - << variableParameters.at("Max"); - } - else - { - std::cout << variableParameters.at("AvailableStepsCount") - << "*scalar " << variableParameters.at("Value"); - } - } - std::cout << "\n"; - } - std::cout << std::endl; - }; - - auto lf_PrintAttributes = [&](IO &io) { - const std::map &attributesInfo = - io.GetAvailableAttributes(); - - size_t maxTypeSize = 0; - size_t maxNameSize = 0; - for (const auto &attributePair : attributesInfo) - { - const size_t nameSize = attributePair.first.size(); - if (nameSize > maxNameSize) - { - maxNameSize = nameSize; - } - - const Params ¶meters = attributePair.second; - const size_t typeSize = parameters.at("Type").size(); - if (typeSize > maxTypeSize) - { - maxTypeSize = typeSize; - } - } - - for (const auto &attributePair : attributesInfo) - { - const std::string name(attributePair.first); - const Params ¶meters = attributePair.second; - const std::string type(parameters.at("Type")); - - std::cout << " "; - std::cout << std::left << std::setw(maxTypeSize) << type << " "; - std::cout << std::left << std::setw(maxNameSize) << name << " "; - std::cout << "attribute = " << parameters.at("Value"); - std::cout << "\n"; - } - std::cout << std::endl; - }; - - ADIOS adios(true); - IO &io = adios.DeclareIO("bpls2"); - BPFileReader bpFileReader(io, m_FileName, Mode::Read, io.m_MPIComm); - - if (m_Parameters.count("verbose") == 1) - { - lf_PrintVerboseHeader(bpFileReader); - } - - if (m_Parameters.count("attrsonly") == 0) - { - lf_PrintVariables(bpFileReader); - } - - if (m_Parameters.count("attrs") == 1 || - m_Parameters.count("attrsonly") == 1) - { - lf_PrintAttributes(io); - } -} - -} // end namespace utils -} // end namespace adios2 diff --git a/source/utils/bpls2/BPLS2.h b/source/utils/bpls2/BPLS2.h deleted file mode 100644 index a42113e9f0..0000000000 --- a/source/utils/bpls2/BPLS2.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * bpls2.h - * - * Created on: Oct 5, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef UTILS_BPLS2_BPLS2_H_ -#define UTILS_BPLS2_BPLS2_H_ - -#include "utils/Utils.h" - -namespace adios2 -{ -namespace utils -{ - -class BPLS2 : public Utils -{ -public: - BPLS2(int argc, char *argv[]); - - ~BPLS2() = default; - - void Run() final; - -private: - static const std::string m_HelpMessage; - static const Params m_Options; - - std::string m_FileName; - - void ParseArguments() final; - void ProcessParameters() const final; - void PrintUsage() const noexcept final; - void PrintExamples() const noexcept final; - void SetParameters(const std::string argument, const bool isLong) final; - - void ProcessTransport() const; -}; - -} // end namespace utils -} // end namespace adios2 - -#endif /* UTILS_BPLS2_BPLS2_H_ */ diff --git a/source/utils/bpls2/main.cpp b/source/utils/bpls2/main.cpp deleted file mode 100644 index 383080ad0e..0000000000 --- a/source/utils/bpls2/main.cpp +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * bpls2Main.cpp - * - * Created on: Oct 5, 2017 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include -#include - -#include "adios2/ADIOSMPI.h" -#include "utils/bpls2/BPLS2.h" - -int main(int argc, char *argv[]) -{ - MPI_Init(&argc, &argv); - - try - { - adios2::utils::BPLS2 bpls2(argc, argv); - bpls2.Run(); - } - catch (std::exception &e) - { - std::cout << e.what() << "\n"; - } - - MPI_Finalize(); -} From 38c4ac5fd501cfeddf05a810414ac963ffcbbf25 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 7 May 2018 08:54:11 -0400 Subject: [PATCH 2/2] Porting to Windows: - Use PathMatchSpec in WIN32 instead of fnmatch - Use KWSys command line parameter processing instead of getopt() in bpls for portability - Use C++ regex for portability and C regex for pre-GCC 4.9 - Retry parsing invalid -xyz arguments as -x -y -z - Added mystrndup for a portable strndup - Change from strtok_r to strtok. --- source/utils/CMakeLists.txt | 2 +- source/utils/bpls/bpls.cpp | 654 ++++++++++++++++++-------------- source/utils/bpls/bpls.h | 5 +- thirdparty/KWSys/CMakeLists.txt | 1 + 4 files changed, 374 insertions(+), 288 deletions(-) diff --git a/source/utils/CMakeLists.txt b/source/utils/CMakeLists.txt index c8abc1e4e7..83da686fac 100644 --- a/source/utils/CMakeLists.txt +++ b/source/utils/CMakeLists.txt @@ -5,7 +5,7 @@ # BPLS add_executable(bpls ./bpls/bpls.cpp) -target_link_libraries(bpls adios2) +target_link_libraries(bpls adios2 adios2sys_interface) install(TARGETS bpls EXPORT adios2 RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 1c2e6f8af7..dc6399e0ed 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -21,14 +21,37 @@ //#include #include #include +#include #include #include "./bpls.h" #include -#include // shell pattern matching -#include + +#if defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) +#if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 +/* pre GCC 4.9 cannot handle the C++ regex implementation. Will use C-lib + * regex"*/ +#define USE_C_REGEX +#endif +#endif + +#ifdef USE_C_REGEX #include // regular expression matching +#else +#include +#endif + +#ifdef _WIN32 +#include "shlwapi.h" +#include "windows.h" +#pragma comment(lib, "shlwapi.lib") +#pragma warning(disable : 4101) // unreferenced local variable +#else +#include +#endif + +#include "adios2sys/CommandLineArguments.hxx" namespace adios2 { @@ -39,16 +62,17 @@ using EntryMap = std::map; // global variables // Values from the arguments or defaults -char *outpath; // output files' starting path (can be extended with subdirs, - // names, indexes) + +// output files' starting path (can be extended with subdirs, +// names, indexes) +std::string outpath; char *varmask[MAX_MASKS]; // can have many -var masks (either shell patterns or // extended regular expressions) int nmasks; // number of masks specified char *vfile; // file name to bpls -char *start; // dimension spec starting points -char *count; // dimension spec counts -char format[32]; // format string for one data element (e.g. %6.2f) -bool formatgiven; // true if format string is provided as argument +std::string start; // dimension spec starting points +std::string count; // dimension spec counts +std::string format; // format string for one data element (e.g. %6.2f) // Flags from arguments or defaults bool dump; // dump data not just list info @@ -73,47 +97,23 @@ char *prgname; /* argv[0] */ // long timefrom, timeto; int64_t istart[MAX_DIMS], icount[MAX_DIMS]; // negative values are allowed int ndimsspecified = 0; +#ifdef USE_C_REGEX regex_t varregex[MAX_MASKS]; // compiled regular expressions of varmask +#else +std::vector varregex; +#endif int ncols = 6; // how many values to print in one row (only for -p) int verbose = 0; FILE *outf; // file to print to or stdout char commentchar; -struct option options[] = { - {"help", no_argument, NULL, 'h'}, - {"verbose", no_argument, NULL, 'v'}, - {"dump", no_argument, NULL, 'd'}, - {"group", no_argument, NULL, 'g'}, - {"regexp", no_argument, NULL, 'e'}, - {"plot", no_argument, NULL, 'p'}, - {"output", required_argument, NULL, 'o'}, - {"xml", no_argument, NULL, 'x'}, - {"start", required_argument, NULL, 's'}, - {"count", required_argument, NULL, 'c'}, - {"noindex", no_argument, NULL, 'y'}, - {"timestep", no_argument, NULL, 't'}, - {"attrs", no_argument, NULL, 'a'}, - {"attrsonly", no_argument, NULL, 'A'}, - {"meshes", no_argument, NULL, 'm'}, - {"long", no_argument, NULL, 'l'}, - {"string", no_argument, NULL, 'S'}, - {"columns", required_argument, NULL, 'n'}, - {"format", required_argument, NULL, 'f'}, - {"hidden_attrs", no_argument, &hidden_attrs_flag, true}, - {"decomp", no_argument, NULL, 'D'}, - // {"sort", no_argument, NULL, 'r'}, - // {"time", required_argument, NULL, 't'}, - {NULL, 0, NULL, 0}}; - -static const char *optstring = "hvepyrtaAmldSDg:o:x:s:c:n:f:"; - // help function void display_help() { // printf( "Usage: %s \n", prgname); printf( "usage: bpls [OPTIONS] file [mask1 mask2 ...]\n" - "\nList/dump content of a BP file. \n" + "\nList/dump content of a BP/HDF5 file. \n" "A mask can be a shell pattern like with 'ls' e.g. \"*/x?\".\n" "Variables with multiple timesteps are reported with an extra " "dimensions.\n" @@ -130,7 +130,6 @@ void display_help() " --sort | -r Sort names before listing\n" */ " --timestep | -t Print values of timestep elements\n" - " --group | -g List/dump groups matching the mask only\n" " --dump | -d Dump matched variables/attributes\n" " To match attributes too, add option " "-a\n" @@ -185,143 +184,192 @@ void display_help() "Typical use: bpls -lav \n"); } -/** Main */ -int bplsMain(int argc, char *argv[]) +bool option_help_was_called = false; +int optioncb_help(const char *argument, const char *value, void *call_data) { - int retval = 0; - int i, timearg = false; - long int tmp; + // adios2sys::CommandLineArguments *arg = + // static_cast(call_data); + // printf("%s\n", arg->GetHelp()); + display_help(); + option_help_was_called = true; + return 1; +} - init_globals(); +int optioncb_verbose(const char *argument, const char *value, void *call_data) +{ + verbose++; + return 1; +} + +int process_unused_args(adios2sys::CommandLineArguments &arg) +{ + int nuargs; + char **uargs; + arg.GetUnusedArguments(&nuargs, &uargs); - ////prgname = strdup(argv[0]); + std::vector retry_args; + retry_args.push_back(new char[4]); - /* other variables */ - int c; - // int last_c = '_'; - /* Process the arguments */ - while ((c = getopt_long(argc, argv, optstring, options, NULL)) != -1) + // first arg is argv[0], so skip that + for (int i = 1; i < nuargs; i++) { - switch (c) + if (uargs[i] != NULL && uargs[i][0] == '-') { - case 'a': - listattrs = true; - break; - case 'A': - listattrs = true; - attrsonly = true; - break; - case 'c': - count = strndup(optarg, 256); - break; - case 'd': - dump = true; - break; - case 'e': - use_regexp = true; - break; - case 'f': - snprintf(format, sizeof(format), "%s", optarg); - formatgiven = true; - break; - case 'h': - display_help(); - return 0; - case 'r': - // sortnames = true; - break; - case 'l': - longopt = true; - readattrs = true; - break; - case 'm': - listmeshes = true; - break; - case 'n': - errno = 0; - tmp = strtol(optarg, (char **)NULL, 0); - if (errno) + if (uargs[i][1] == '-') { - fprintf(stderr, - "Error: could not convert --columns value: %s\n", - optarg); + fprintf(stderr, "Unknown long option: %s\n", uargs[i]); + arg.DeleteRemainingArguments(nuargs, &uargs); return 1; } - ncols = tmp; - break; - case 'o': - outpath = strndup(optarg, 256); - break; - case 'p': - plot = true; - break; - case 's': - start = strndup(optarg, 256); - break; - case 'S': - printByteAsChar = true; - break; - case 't': - timestep = true; - break; - case 'x': - output_xml = true; - break; - case 'y': - noindex = true; - break; - case 'v': - verbose++; - break; - case 'D': - show_decomp = true; - break; - case 1: - /* This means a field is unknown, or could be multiple arg or bad - * arg*/ - /* - if (last_c=='t') { // --time 2nd arg (or not if not a number) - errno = 0; - tmp = strtol(optarg, (char **)NULL, 0); - if (!errno) { - timeto = tmp; - printf("Time set to %d - %d\n", timefrom, timeto); - timearg=true; - } - } - */ - if (!timearg) + else { - fprintf(stderr, "Unrecognized argument: %s\n", optarg); - return 1; + // Maybe -abc is -a -b -c? + size_t len = strlen(uargs[i]); + for (size_t j = 1; j < len; ++j) + { + char *opt = new char[3]; + opt[0] = '-'; + opt[1] = uargs[i][j]; + opt[2] = '\0'; + retry_args.push_back(opt); + } } - break; + } + else if (vfile == NULL) + { + vfile = mystrndup(uargs[i], 4096); + // fprintf(stderr, "Set file argument: %s\n", vfile); + } + else + { + varmask[nmasks] = mystrndup(uargs[i], 256); + // fprintf(stderr, "Set mask %d argument: %s\n", nmasks, + // varmask[nmasks]); + nmasks++; + } + } + arg.DeleteRemainingArguments(nuargs, &uargs); - default: - printf("Processing default: %c\n", c); - break; - } /* end switch */ - // last_c = c; - } /* end while */ + if (retry_args.size() > 1) + { + // Run a new parse on the -a single letter arguments + // fprintf(stderr, "Rerun parse on %zu options\n", retry_args.size()); + arg.Initialize(static_cast(retry_args.size()), retry_args.data()); + arg.StoreUnusedArguments(false); + if (!arg.Parse()) + { + fprintf(stderr, "Parsing arguments failed\n"); + return 1; + } + for (size_t j = 0; j < retry_args.size(); ++j) + { + delete[] retry_args[j]; + } + } + else + { + delete[] retry_args[0]; + } - /* Check if we have a file defined */ - if (optind >= argc) + return 0; +} + +/** Main */ +int bplsMain(int argc, char *argv[]) +{ + int retval = 0; + + init_globals(); + + adios2sys::CommandLineArguments arg; + arg.Initialize(argc, argv); + typedef adios2sys::CommandLineArguments argT; + arg.StoreUnusedArguments(true); + arg.AddCallback("--help", argT::NO_ARGUMENT, optioncb_help, &arg, "Help"); + arg.AddCallback("-h", argT::NO_ARGUMENT, optioncb_help, &arg, ""); + arg.AddCallback("--verbose", argT::NO_ARGUMENT, optioncb_verbose, nullptr, + "Print information about what bpls is doing"); + arg.AddCallback("-v", argT::NO_ARGUMENT, optioncb_verbose, nullptr, ""); + arg.AddBooleanArgument("--dump", &dump, + "Dump matched variables/attributes"); + arg.AddBooleanArgument("-d", &dump, ""); + arg.AddBooleanArgument( + "--long", &longopt, + "Print values of all scalars and attributes and min/max " + "values of arrays"); + arg.AddBooleanArgument("-l", &longopt, ""); + arg.AddBooleanArgument("--regexp", &use_regexp, + "| -e Treat masks as extended regular expressions"); + arg.AddBooleanArgument("-e", &use_regexp, ""); + arg.AddArgument("--output", argT::SPACE_ARGUMENT, &outpath, + "| -o opt Print to a file instead of stdout"); + arg.AddArgument("-o", argT::SPACE_ARGUMENT, &outpath, ""); + arg.AddArgument("--start", argT::SPACE_ARGUMENT, &start, + "| -s opt Offset indices in each dimension (default is " + "0 for all dimensions). opt<0 is handled as in python (-1 " + "is last)"); + arg.AddArgument("-s", argT::SPACE_ARGUMENT, &start, ""); + arg.AddArgument("--count", argT::SPACE_ARGUMENT, &count, + "| -c opt Number of elements in each dimension. -1 " + "denotes 'until end' of dimension. default is -1 for all " + "dimensions"); + arg.AddArgument("-c", argT::SPACE_ARGUMENT, &count, ""); + arg.AddBooleanArgument("--noindex", &noindex, + " | -y Print data without array indices"); + arg.AddBooleanArgument("-y", &noindex, ""); + arg.AddBooleanArgument("--timestep", ×tep, + " | -t Print values of timestep elements"); + arg.AddBooleanArgument("-t", ×tep, ""); + arg.AddBooleanArgument("--attrs", &listattrs, + " | -a List/match attributes too"); + arg.AddBooleanArgument("-a", &listattrs, ""); + arg.AddBooleanArgument("--attrsonly", &attrsonly, + " | -A List/match attributes only (no variables)"); + arg.AddBooleanArgument("-A", &attrsonly, ""); + arg.AddBooleanArgument("--meshes", &listmeshes, " | -m List meshes"); + arg.AddBooleanArgument("-m", &listmeshes, ""); + arg.AddBooleanArgument("--string", &printByteAsChar, + " | -S Print 8bit integer arrays as strings"); + arg.AddBooleanArgument("-S", &printByteAsChar, ""); + arg.AddArgument("--columns", argT::SPACE_ARGUMENT, &ncols, + "| -n opt Number of data elements per row to print"); + arg.AddArgument("-n", argT::SPACE_ARGUMENT, &ncols, ""); + arg.AddArgument("--format", argT::SPACE_ARGUMENT, &format, + "| -f opt Format string to use for one data item "); + arg.AddArgument("-f", argT::SPACE_ARGUMENT, &format, ""); + arg.AddBooleanArgument("--hidden_attrs", &hidden_attrs, + " Show hidden ADIOS attributes in the file"); + arg.AddBooleanArgument( + "--decompose", &show_decomp, + "| -D Show decomposition of variables as layed out in file"); + arg.AddBooleanArgument("-D", &show_decomp, ""); + + if (!arg.Parse()) { - fprintf(stderr, "Missing file name\n"); + fprintf(stderr, "Parsing arguments failed\n"); return 1; } - vfile = strdup(argv[optind++]); - while (optind < argc) + if (option_help_was_called) + return 0; + + retval = process_unused_args(arg); + if (retval) + { + return retval; + } + if (option_help_was_called) + return 0; + + /* Check if we have a file defined */ + if (vfile == NULL) { - varmask[nmasks] = strndup(argv[optind++], 256); - nmasks++; + fprintf(stderr, "Missing file name\n"); + return 1; } /* Process dimension specifications */ - if (start != NULL) - parseDimSpec(start, istart); - if (count != NULL) - parseDimSpec(count, icount); + parseDimSpec(start, istart); + parseDimSpec(count, icount); // process the regular expressions if (use_regexp) @@ -352,12 +400,14 @@ int bplsMain(int argc, char *argv[]) print_stop(); /* Free allocated memories */ - // myfree(prgname); - myfree(outpath); - for (i = 0; i < nmasks; i++) + for (int i = 0; i < nmasks; i++) { myfree(varmask[i]); +#ifdef USE_C_REGEX regfree(&(varregex[i])); +#else + varregex.clear(); +#endif } myfree(vfile); @@ -368,13 +418,10 @@ void init_globals() { int i; // variables for arguments - outpath = NULL; for (i = 0; i < MAX_MASKS; i++) varmask[i] = NULL; nmasks = 0; vfile = NULL; - start = NULL; - count = NULL; verbose = 0; ncols = 6; // by default when printing ascii, print "X Y", not X: Y1 Y2... dump = false; @@ -393,7 +440,6 @@ void init_globals() plot = false; hidden_attrs = false; hidden_attrs_flag = 0; - formatgiven = false; printByteAsChar = false; show_decomp = false; for (i = 0; i < MAX_DIMS; i++) @@ -437,14 +483,14 @@ void printSettings(void) printf("%s ", varmask[i]); printf("\n"); printf(" file : %s\n", vfile); - printf(" output : %s\n", (outpath ? outpath : "stdout")); + printf(" output : %s\n", (outpath.empty() ? "stdout" : outpath.c_str())); - if (start != NULL) + if (start.size()) { PRINT_DIMS_INT64(" start", istart, ndimsspecified, i); printf("\n"); } - if (count != NULL) + if (count.size()) { PRINT_DIMS_INT64(" count", icount, ndimsspecified, i); printf("\n"); @@ -464,8 +510,8 @@ void printSettings(void) printf(" -d : dump matching variables and attributes\n"); if (use_regexp) printf(" -e : handle masks as regular expressions\n"); - if (formatgiven) - printf(" -f : dump using printf format \"%s\"\n", format); + if (format.size()) + printf(" -f : dump using printf format \"%s\"\n", format.c_str()); if (output_xml) printf(" -x : output data in XML format\n"); if (show_decomp) @@ -499,22 +545,16 @@ void print_file_size(uint64_t size) printf(" file size: %" PRIu64 " %s\n", s, sm[idx]); } -static inline int ndigits(int64_t n) +static inline int ndigits(size_t n) { static char digitstr[32]; - return snprintf(digitstr, 32, "%" PRId64, n); + return snprintf(digitstr, 32, "%zu", n); } int nEntriesMatched = 0; int doList_vars(Engine *fp, IO *io) { - enum ADIOS_DATATYPES vartype; - int i, j, n; // loop vars - int attrsize; // info about one attribute - int len, maxlen, maxtypelen; - bool timed; // variable has multiple timesteps - const DataMap &variables = io->GetVariablesDataMap(); const DataMap &attributes = io->GetAttributesDataMap(); @@ -537,17 +577,17 @@ int doList_vars(Engine *fp, IO *io) } } - int nNames = entries.size(); + // size_t nNames = entries.size(); // calculate max length of variable names and type names in the first round - maxlen = 4; - maxtypelen = 7; + int maxlen = 4; // need int for printf formatting + int maxtypelen = 7; for (const auto &entrypair : entries) { - len = entrypair.first.size(); + int len = static_cast(entrypair.first.size()); if (len > maxlen) maxlen = len; - len = entrypair.second.typeName.size(); + len = static_cast(entrypair.second.typeName.size()); if (len > maxtypelen) maxtypelen = len; } @@ -598,7 +638,6 @@ int doList_vars(Engine *fp, IO *io) } else { - VariableBase *v = nullptr; if (entry.typeName == "compound") { // not supported @@ -637,7 +676,7 @@ int printVariableInfo(Engine *fp, IO *io, Variable *variable) if (variable->m_Shape.size() > 0) { fprintf(outf, "{%zu", variable->m_Shape[0]); - for (int j = 1; j < variable->m_Shape.size(); j++) + for (size_t j = 1; j < variable->m_Shape.size(); j++) { fprintf(outf, ", %zu", variable->m_Shape[j]); } @@ -1005,8 +1044,8 @@ void printMeshes(Engine *fp) std::vector getEnginesList(const std::string path) { std::vector list; - size_t slen = path.length(); #ifdef ADIOS2_HAVE_HDF5 + size_t slen = path.length(); if (slen >= 3 && path.compare(slen - 3, 3, ".h5") == 0) { list.push_back("HDF5"); @@ -1025,12 +1064,6 @@ std::vector getEnginesList(const std::string path) int doList(const char *path) { - int grpid; - int status; - int mpi_comm_dummy = 0; - int nGroupsMatched = 0; - int nGroups; // number of groups - char **group_namelist; char init_params[128]; int adios_verbose = 2; @@ -1062,7 +1095,8 @@ int doList(const char *path) catch (std::exception &e) { if (verbose > 2) - printf("Failed to open with %s engine.\n", engineName.c_str()); + printf("Failed to open with %s engine: %s\n", + engineName.c_str(), e.what()); } if (fp != nullptr) break; @@ -1326,20 +1360,19 @@ int readVar(Engine *fp, IO *io, Variable *variable) // size_t elemsize; // size in bytes of one element uint64_t st, ct; T *data; - uint64_t sum; // working var to sum up things - int maxreadn; // max number of elements to read once up to a limit (10MB - // of - // data) - int actualreadn; // our decision how much to read at once - int readn[MAX_DIMS]; // how big chunk to read in in each dimension? - int status; - bool incdim; // used in incremental reading in - int ndigits_dims[32]; // # of digits (to print) of each dimension + uint64_t sum; // working var to sum up things + uint64_t + maxreadn; // max number of elements to read once up to a limit (10MB + // of + // data) + uint64_t actualreadn; // our decision how much to read at once + uint64_t readn[MAX_DIMS]; // how big chunk to read in in each dimension? + bool incdim; // used in incremental reading in + int ndigits_dims[32]; // # of digits (to print) of each dimension const size_t elemsize = variable->m_ElementSize; - const int ndim = variable->m_Shape.size(); - const int nsteps = variable->GetAvailableStepsCount(); - + const int nsteps = static_cast(variable->GetAvailableStepsCount()); + const int ndim = static_cast(variable->m_Shape.size()); // create the counter arrays with the appropriate lengths // transfer start and count arrays to format dependent arrays @@ -1369,9 +1402,9 @@ int readVar(Engine *fp, IO *io, Variable *variable) tidx = 1; } - tdims = variable->m_Shape.size() + tidx; + tdims = ndim + tidx; - for (j = 0; j < variable->m_Shape.size(); j++) + for (j = 0; j < ndim; j++) { if (istart[j + tidx] < 0) // negative index means last-|index| st = variable->m_Shape[j] + istart[j + tidx]; @@ -1403,7 +1436,7 @@ int readVar(Engine *fp, IO *io, Variable *variable) print_slice_info(variable, start_t, count_t); - maxreadn = MAX_BUFFERSIZE / elemsize; + maxreadn = (uint64_t)MAX_BUFFERSIZE / elemsize; if (nelems < maxreadn) maxreadn = nelems; @@ -1441,12 +1474,12 @@ int readVar(Engine *fp, IO *io, Variable *variable) readn[i] = count_t[i]; } if (verbose > 1) - printf(" dim %d: read %d elements\n", i, readn[i]); + printf(" dim %d: read %" PRIu64 " elements\n", i, readn[i]); sum = sum * (uint64_t)count_t[i]; actualreadn = actualreadn * readn[i]; } if (verbose > 1) - printf(" read %d elements at once, %" PRIu64 + printf(" read %" PRIu64 " elements at once, %" PRIu64 " in total (nelems=%" PRIu64 ")\n", actualreadn, sum, nelems); @@ -1476,7 +1509,7 @@ int readVar(Engine *fp, IO *io, Variable *variable) printf("adios_read_var name=%s ", variable->m_Name.c_str()); PRINT_DIMS_UINT64(" start", s, tdims, j); PRINT_DIMS_UINT64(" count", c, tdims, j); - printf(" read %d elems\n", actualreadn); + printf(" read %" PRIu64 " elems\n", actualreadn); } // read a slice finally @@ -1808,19 +1841,21 @@ int readVarBlock(Engine *fp, IO *io, Variable *variable, int blockid) bool matchesAMask(const char *name) { - int excode; - int i; int startpos = 0; // to match with starting / or without +#ifdef USE_C_REGEX regmatch_t pmatch[1] = {{(regoff_t)-1, (regoff_t)-1}}; +#else +#endif if (nmasks == 0) return true; - for (i = 0; i < nmasks; i++) + for (int i = 0; i < nmasks; i++) { if (use_regexp) { - excode = regexec(&(varregex[i]), name, 1, pmatch, 0); +#ifdef USE_C_REGEX + int excode = regexec(&(varregex[i]), name, 1, pmatch, 0); if (name[0] == '/') // have to check if it matches from the second // character too startpos = 1; @@ -1829,13 +1864,16 @@ bool matchesAMask(const char *name) pmatch[0].rm_so == startpos) && // from the beginning pmatch[0].rm_eo == strlen(name) // to the very end of the name ) +#else + bool matches = std::regex_match(name, varregex[i]); + if (!matches && name[0] == '/') + matches = std::regex_match(name + 1, varregex[i]); + if (matches) +#endif { if (verbose > 1) printf("Name %s matches regexp %i %s\n", name, i, varmask[i]); - // printf("Match from %d to %d\n", (int) pmatch[0].rm_so, - // (int) - // pmatch[0].rm_eo); return true; } } @@ -1844,7 +1882,11 @@ bool matchesAMask(const char *name) // use shell pattern matching if (varmask[i][0] != '/' && name[0] == '/') startpos = 1; +#ifdef _WIN32 + if (PathMatchSpec(name + startpos, varmask[i])) +#else if (fnmatch(varmask[i], name + startpos, FNM_FILE_NAME) == 0) +#endif { if (verbose > 1) printf("Name %s matches varmask %i %s\n", name, i, @@ -1856,18 +1898,18 @@ bool matchesAMask(const char *name) return false; } -int print_start(const char *fname) +int print_start(const std::string &fname) { - if (fname == NULL) + if (fname.empty()) { outf = stdout; } else { - if ((outf = fopen(fname, "w")) == NULL) + if ((outf = fopen(fname.c_str(), "w")) == NULL) { - fprintf(stderr, "Error at opening for writing file %s: %s\n", fname, - strerror(errno)); + fprintf(stderr, "Error at opening for writing file %s: %s\n", + fname.c_str(), strerror(errno)); return 30; } } @@ -1885,7 +1927,6 @@ void print_slice_info(VariableBase *variable, uint64_t *s, uint64_t *c) // not the complete variable is read size_t ndim = variable->m_Shape.size(); size_t nsteps = variable->m_AvailableStepsCount; - int i; bool isaslice = false; int tidx = (nsteps > 1 ? 1 : 0); size_t tdim = ndim + tidx; @@ -1894,7 +1935,7 @@ void print_slice_info(VariableBase *variable, uint64_t *s, uint64_t *c) if (c[0] < nsteps) isaslice = true; } - for (i = 0; i < ndim; i++) + for (size_t i = 0; i < ndim; i++) { if (c[i + tidx] < variable->m_Shape[i]) isaslice = true; @@ -1903,7 +1944,7 @@ void print_slice_info(VariableBase *variable, uint64_t *s, uint64_t *c) { fprintf(outf, "%c slice (%" PRIu64 ":%" PRIu64, commentchar, s[0], s[0] + c[0] - 1); - for (i = 1; i < tdim; i++) + for (size_t i = 1; i < tdim; i++) { fprintf(outf, ", %" PRIu64 ":%" PRIu64, s[i], s[i] + c[i] - 1); } @@ -1932,7 +1973,6 @@ const std::map adios_types_map = { enum ADIOS_DATATYPES type_to_enum(std::string type) { - enum ADIOS_DATATYPES t; auto itType = adios_types_map.find(type); if (itType == adios_types_map.end()) { @@ -1987,17 +2027,18 @@ int print_data_characteristics(void *min, void *max, double *avg, enum ADIOS_DATATYPES adiosvartype, bool allowformat) { - bool f = formatgiven && allowformat; + bool f = format.size() && allowformat; + const char *fmt = format.c_str(); switch (adiosvartype) { case adios_unsigned_byte: if (min) - fprintf(outf, (f ? format : "%10hhu "), *((unsigned char *)min)); + fprintf(outf, (f ? fmt : "%10hhu "), *((unsigned char *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10hhu "), *((unsigned char *)max)); + fprintf(outf, (f ? fmt : "%10hhu "), *((unsigned char *)max)); else fprintf(outf, " null "); if (avg) @@ -2011,11 +2052,11 @@ int print_data_characteristics(void *min, void *max, double *avg, break; case adios_byte: if (min) - fprintf(outf, (f ? format : "%10hhd "), *((char *)min)); + fprintf(outf, (f ? fmt : "%10hhd "), *((char *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10hhd "), *((char *)max)); + fprintf(outf, (f ? fmt : "%10hhd "), *((char *)max)); else fprintf(outf, " null "); if (avg) @@ -2032,11 +2073,11 @@ int print_data_characteristics(void *min, void *max, double *avg, case adios_unsigned_short: if (min) - fprintf(outf, (f ? format : "%10hu "), (*(unsigned short *)min)); + fprintf(outf, (f ? fmt : "%10hu "), (*(unsigned short *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10hu "), (*(unsigned short *)max)); + fprintf(outf, (f ? fmt : "%10hu "), (*(unsigned short *)max)); else fprintf(outf, " null "); if (avg) @@ -2050,11 +2091,11 @@ int print_data_characteristics(void *min, void *max, double *avg, break; case adios_short: if (min) - fprintf(outf, (f ? format : "%10hd "), (*(short *)min)); + fprintf(outf, (f ? fmt : "%10hd "), (*(short *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10hd "), (*(short *)max)); + fprintf(outf, (f ? fmt : "%10hd "), (*(short *)max)); else fprintf(outf, " null "); if (avg) @@ -2069,11 +2110,11 @@ int print_data_characteristics(void *min, void *max, double *avg, case adios_unsigned_integer: if (min) - fprintf(outf, (f ? format : "%10u "), (*(unsigned int *)min)); + fprintf(outf, (f ? fmt : "%10u "), (*(unsigned int *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10u "), (*(unsigned int *)max)); + fprintf(outf, (f ? fmt : "%10u "), (*(unsigned int *)max)); else fprintf(outf, " null "); if (avg) @@ -2087,11 +2128,11 @@ int print_data_characteristics(void *min, void *max, double *avg, break; case adios_integer: if (min) - fprintf(outf, (f ? format : "%10d "), (*(int *)min)); + fprintf(outf, (f ? fmt : "%10d "), (*(int *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10d "), (*(int *)max)); + fprintf(outf, (f ? fmt : "%10d "), (*(int *)max)); else fprintf(outf, " null "); if (avg) @@ -2106,13 +2147,11 @@ int print_data_characteristics(void *min, void *max, double *avg, case adios_unsigned_long: if (min) - fprintf(outf, (f ? format : "%10llu "), - (*(unsigned long long *)min)); + fprintf(outf, (f ? fmt : "%10llu "), (*(unsigned long long *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10llu "), - (*(unsigned long long *)max)); + fprintf(outf, (f ? fmt : "%10llu "), (*(unsigned long long *)max)); else fprintf(outf, " null "); if (avg) @@ -2126,11 +2165,11 @@ int print_data_characteristics(void *min, void *max, double *avg, break; case adios_long: if (min) - fprintf(outf, (f ? format : "%10lld "), (*(long long *)min)); + fprintf(outf, (f ? fmt : "%10lld "), (*(long long *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10lld "), (*(long long *)max)); + fprintf(outf, (f ? fmt : "%10lld "), (*(long long *)max)); else fprintf(outf, " null "); if (avg) @@ -2145,11 +2184,11 @@ int print_data_characteristics(void *min, void *max, double *avg, case adios_real: if (min) - fprintf(outf, (f ? format : "%10.2g "), (*(float *)min)); + fprintf(outf, (f ? fmt : "%10.2g "), (*(float *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10.2g "), (*(float *)max)); + fprintf(outf, (f ? fmt : "%10.2g "), (*(float *)max)); else fprintf(outf, " null "); if (avg) @@ -2163,11 +2202,11 @@ int print_data_characteristics(void *min, void *max, double *avg, break; case adios_double: if (min) - fprintf(outf, (f ? format : "%10.2g "), (*(double *)min)); + fprintf(outf, (f ? fmt : "%10.2g "), (*(double *)min)); else fprintf(outf, " null "); if (max) - fprintf(outf, (f ? format : "%10.2g "), (*(double *)max)); + fprintf(outf, (f ? fmt : "%10.2g "), (*(double *)max)); else fprintf(outf, " null "); if (avg) @@ -2205,7 +2244,8 @@ int print_data_characteristics(void *min, void *max, double *avg, int print_data(const void *data, int item, enum ADIOS_DATATYPES adiosvartype, bool allowformat) { - bool f = formatgiven && allowformat; + bool f = format.size() && allowformat; + const char *fmt = format.c_str(); if (data == NULL) { fprintf(outf, "null "); @@ -2215,62 +2255,61 @@ int print_data(const void *data, int item, enum ADIOS_DATATYPES adiosvartype, switch (adiosvartype) { case adios_unsigned_byte: - fprintf(outf, (f ? format : "%hhu"), ((unsigned char *)data)[item]); + fprintf(outf, (f ? fmt : "%hhu"), ((unsigned char *)data)[item]); break; case adios_byte: - fprintf(outf, (f ? format : "%hhd"), ((signed char *)data)[item]); + fprintf(outf, (f ? fmt : "%hhd"), ((signed char *)data)[item]); break; case adios_string: - fprintf(outf, (f ? format : "\"%s\""), ((char *)data) + item); + fprintf(outf, (f ? fmt : "\"%s\""), ((char *)data) + item); break; case adios_string_array: // we expect one elemet of the array here - fprintf(outf, (f ? format : "\"%s\""), *((char **)data + item)); + fprintf(outf, (f ? fmt : "\"%s\""), *((char **)data + item)); break; case adios_unsigned_short: - fprintf(outf, (f ? format : "%hu"), ((unsigned short *)data)[item]); + fprintf(outf, (f ? fmt : "%hu"), ((unsigned short *)data)[item]); break; case adios_short: - fprintf(outf, (f ? format : "%hd"), ((signed short *)data)[item]); + fprintf(outf, (f ? fmt : "%hd"), ((signed short *)data)[item]); break; case adios_unsigned_integer: - fprintf(outf, (f ? format : "%u"), ((unsigned int *)data)[item]); + fprintf(outf, (f ? fmt : "%u"), ((unsigned int *)data)[item]); break; case adios_integer: - fprintf(outf, (f ? format : "%d"), ((signed int *)data)[item]); + fprintf(outf, (f ? fmt : "%d"), ((signed int *)data)[item]); break; case adios_unsigned_long: - fprintf(outf, (f ? format : "%llu"), - ((unsigned long long *)data)[item]); + fprintf(outf, (f ? fmt : "%llu"), ((unsigned long long *)data)[item]); break; case adios_long: - fprintf(outf, (f ? format : "%lld"), ((signed long long *)data)[item]); + fprintf(outf, (f ? fmt : "%lld"), ((signed long long *)data)[item]); break; case adios_real: - fprintf(outf, (f ? format : "%g"), ((float *)data)[item]); + fprintf(outf, (f ? fmt : "%g"), ((float *)data)[item]); break; case adios_double: - fprintf(outf, (f ? format : "%g"), ((double *)data)[item]); + fprintf(outf, (f ? fmt : "%g"), ((double *)data)[item]); break; case adios_long_double: - fprintf(outf, (f ? format : "%Lg"), ((long double *)data)[item]); - // fprintf(outf,(f ? format : "????????")); + fprintf(outf, (f ? fmt : "%Lg"), ((long double *)data)[item]); + // fprintf(outf,(f ? fmt : "????????")); break; case adios_complex: - fprintf(outf, (f ? format : "(%g,i%g)"), ((float *)data)[2 * item], + fprintf(outf, (f ? fmt : "(%g,i%g)"), ((float *)data)[2 * item], ((float *)data)[2 * item + 1]); break; case adios_double_complex: - fprintf(outf, (f ? format : "(%g,i%g)"), ((double *)data)[2 * item], + fprintf(outf, (f ? fmt : "(%g,i%g)"), ((double *)data)[2 * item], ((double *)data)[2 * item + 1]); break; @@ -2392,18 +2431,18 @@ template void print_decomp(Engine *fp, IO *io, Variable *variable) { /* Print block info */ - int blockid = 0; - int nsteps = variable->GetAvailableStepsCount(); + // int blockid = 0; + size_t nsteps = variable->GetAvailableStepsCount(); size_t ndim = variable->m_Shape.size(); - enum ADIOS_DATATYPES vartype = type_to_enum(variable->m_Type); - size_t nblocks = 1; /* FIXME: we need the number of blocks here */ + // enum ADIOS_DATATYPES vartype = type_to_enum(variable->m_Type); + // size_t nblocks = 1; /* FIXME: we need the number of blocks here */ int ndigits_nsteps = ndigits(nsteps - 1); if (ndim == 0) { // scalars - for (int i = 0; i < nsteps; i++) + for (size_t i = 0; i < nsteps; i++) { - fprintf(outf, " step %*d: ", ndigits_nsteps, i); + fprintf(outf, " step %*zu: ", ndigits_nsteps, i); #if 0 fprintf(outf, "%d instances available\n", vi->nblocks[i]); if (dump && vi->statistics && vi->statistics->blocks) @@ -2452,19 +2491,21 @@ void print_decomp(Engine *fp, IO *io, Variable *variable) else { // arrays + /* int ndigits_nblocks; int ndigits_procid; int ndigits_time; int ndigits_dims[32]; - for (int k = 0; k < ndim; k++) + for (size_t k = 0; k < ndim; k++) { // get digit lengths for each dimension ndigits_dims[k] = ndigits(variable->m_Shape[k] - 1); } + */ - for (int i = 0; i < nsteps; i++) + for (size_t i = 0; i < nsteps; i++) { - fprintf(outf, " step %*d: ", ndigits_nsteps, i); + fprintf(outf, " step %*zu: ", ndigits_nsteps, i); fprintf(outf, "\n"); #if 0 ndigits_nblocks = ndigits(vi->nblocks[i] - 1); @@ -2604,14 +2645,18 @@ void print_decomp(Engine *fp, IO *io, Variable *variable) // parse a string "0, 3; 027" into an integer array // of [0,3,27] // exits if parsing failes -void parseDimSpec(char *str, int64_t *dims) +void parseDimSpec(const std::string &str, int64_t *dims) { - char *token, *saveptr; + if (str.empty()) + return; + + char *token; char *s; // copy of s; strtok modifies the string int i = 0; - s = strndup(str, 256); - token = strtok_r(s, " ,;x\t\n", &saveptr); + s = mystrndup(str.c_str(), 1024); + // token = strtok_r(s, " ,;x\t\n", &saveptr); + token = strtok(s, " ,;x\t\n"); while (token != NULL && i < MAX_DIMS) { // printf("\t|%s|", token); @@ -2621,12 +2666,13 @@ void parseDimSpec(char *str, int64_t *dims) { fprintf(stderr, "Error: could not convert field into a value: " "%s from \"%s\"\n", - token, str); + token, str.c_str()); exit(200); } // get next item - token = strtok_r(NULL, " ,;x\t\n", &saveptr); + // token = strtok_r(NULL, " ,;x\t\n", &saveptr); + token = strtok(NULL, " ,;x\t\n"); i++; } // if (i>0) printf("\n"); @@ -2639,30 +2685,67 @@ void parseDimSpec(char *str, int64_t *dims) { fprintf(stderr, "Error: More dimensions specified in \"%s\" than we " "can handle (%d)\n", - str, MAX_DIMS); + str.c_str(), MAX_DIMS); exit(200); } + free(s); } int compile_regexp_masks(void) { - int i, errcode; +#ifdef USE_C_REGEX + int errcode; char buf[256]; - for (i = 0; i < nmasks; i++) + for (int i = 0; i < nmasks; i++) { errcode = regcomp(&(varregex[i]), varmask[i], REG_EXTENDED); if (errcode) { regerror(errcode, &(varregex[i]), buf, sizeof(buf)); - fprintf(stderr, "Error: var %s is an invalid extended regular " + fprintf(stderr, "Error: \"%s\" is an invalid extended regular " "expression: %s\n", varmask[i], buf); return 2; } } +#else + varregex.reserve(nmasks); + for (int i = 0; i < nmasks; i++) + { + try + { + varregex.push_back(std::regex(varmask[i])); + } + catch (std::regex_error &e) + { + fprintf(stderr, "Error: \"%s\" is an invalid extended regular " + "expression. C++ regex error code: %d\n", + varmask[i], + e.code() == std::regex_constants::error_badrepeat); + return 2; + } + } +#endif return 0; } +char *mystrndup(const char *s, size_t n) +{ + char *t = nullptr; + if (n > 0) + { + size_t slen = strlen(s); + size_t len = (slen <= n ? slen : n); + t = (char *)malloc(len + 1); + if (t) + { + memcpy(t, s, len); + t[len] = '\0'; + } + } + return t; +} + // end of namespace } } @@ -2670,6 +2753,7 @@ int compile_regexp_masks(void) int main(int argc, char *argv[]) { MPI_Init(&argc, &argv); - return adios2::utils::bplsMain(argc, argv); + int retval = adios2::utils::bplsMain(argc, argv); MPI_Finalize(); + return retval; } diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h index ae27d55d03..ea0d039bdd 100644 --- a/source/utils/bpls/bpls.h +++ b/source/utils/bpls/bpls.h @@ -71,9 +71,10 @@ struct Entry // how to print one data item of an array // enum PrintDataType {STRING, INT, FLOAT, DOUBLE, COMPLEX}; +char *mystrndup(const char *s, size_t n); void init_globals(); void processDimSpecs(); -void parseDimSpec(char *str, int64_t *dims); +void parseDimSpec(const std::string &str, int64_t *dims); int compile_regexp_masks(void); void printSettings(void); int doList(const char *path); @@ -94,7 +95,7 @@ int readVarBlock(Engine *fp, IO *io, Variable *variable, int blockid); int cmpstringp(const void *p1, const void *p2); bool grpMatchesMask(char *name); bool matchesAMask(const char *name); -int print_start(const char *fname); +int print_start(const std::string &fnamestr); void print_slice_info(VariableBase *variable, uint64_t *s, uint64_t *c); int print_data(const void *data, int item, enum ADIOS_DATATYPES adiosvartypes, bool allowformat); diff --git a/thirdparty/KWSys/CMakeLists.txt b/thirdparty/KWSys/CMakeLists.txt index 952fd5e0ca..2ca60cede4 100644 --- a/thirdparty/KWSys/CMakeLists.txt +++ b/thirdparty/KWSys/CMakeLists.txt @@ -1,4 +1,5 @@ set(KWSYS_NAMESPACE adios2sys) +set(KWSYS_USE_CommandLineArguments ON) set(KWSYS_USE_DynamicLoader ON) set(KWSYS_USE_RegularExpression ON) set(KWSYS_USE_SystemTools ON)