Skip to content

Commit

Permalink
Log cuIO warnings using the libcudf logger (#13043)
Browse files Browse the repository at this point in the history
Log warning messages in the following scenarios:

- cuIO reader benchmark is ran without dropping the L3 cache.
- An nvCOMP call fails and we fall back to older API.
- Mangling of duplicated column names is disabled and some columns are dropped.
- Default compression fails in ORC writer and we fall back to uncompressed output.
- Parquet writer is writing a decimal column with precision < 10 as int64.

Authors:
  - Vukasin Milovanovic (https://github.com/vuule)

Approvers:
  - Nghia Truong (https://github.com/ttnghia)
  - Mark Harris (https://github.com/harrism)

URL: #13043
  • Loading branch information
vuule authored Apr 4, 2023
1 parent 09b114e commit bd1ef29
Show file tree
Hide file tree
Showing 5 changed files with 26 additions and 2 deletions.
17 changes: 16 additions & 1 deletion cpp/benchmarks/io/cuio_common.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/

#include <benchmarks/io/cuio_common.hpp>
#include <cudf/detail/utilities/logger.hpp>

#include <cstdio>
#include <fstream>
Expand Down Expand Up @@ -173,10 +174,24 @@ std::string exec_cmd(std::string_view cmd)
return error_out;
}

void log_l3_warning_once()
{
static bool is_logged = false;
if (not is_logged) {
CUDF_LOG_WARN(
"Running benchmarks without dropping the L3 cache; results may not reflect file IO "
"throughput");
is_logged = true;
}
}

void try_drop_l3_cache()
{
static bool is_drop_cache_enabled = std::getenv("CUDF_BENCHMARK_DROP_CACHE") != nullptr;
if (not is_drop_cache_enabled) { return; }
if (not is_drop_cache_enabled) {
log_l3_warning_once();
return;
}

std::array drop_cache_cmds{"/sbin/sysctl vm.drop_caches=3", "sudo /sbin/sysctl vm.drop_caches=3"};
CUDF_EXPECTS(std::any_of(drop_cache_cmds.cbegin(),
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/io/comp/nvcomp_adapter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,9 @@ size_t batched_compress_temp_size(compression_type compression,
compression, num_chunks, max_uncomp_chunk_size, max_total_uncomp_size);
} catch (...) {
// Ignore errors in the expanded version; fall back to the old API in case of failure
CUDF_LOG_WARN(
"CompressGetTempSizeEx call failed, falling back to CompressGetTempSize; this may increase "
"the memory usage");
}
#endif

Expand Down
4 changes: 3 additions & 1 deletion cpp/src/io/csv/reader_impl.cu
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,9 @@ table_with_metadata read_csv(cudf::io::datasource* source,
if (!reader_opts.is_enabled_mangle_dupe_cols()) {
for (auto& col_name : column_names) {
if (++col_names_counts[col_name] > 1) {
// All duplicate columns will be ignored; First appearance is parsed
CUDF_LOG_WARN("Multiple columns with name {}; only the first appearance is parsed",
col_name);

const auto idx = &col_name - column_names.data();
column_flags[idx] = column_parse::disabled;
}
Expand Down
1 change: 1 addition & 0 deletions cpp/src/io/orc/stripe_enc.cu
Original file line number Diff line number Diff line change
Expand Up @@ -1334,6 +1334,7 @@ void CompressOrcDataStreams(uint8_t* compressed_data,
[] __device__(compression_result & stat) { stat.status = compression_status::FAILURE; });
// Since SNAPPY is the default compression (may not be explicitly requested), fall back to
// writing without compression
CUDF_LOG_WARN("ORC writer: compression failed, writing uncompressed data");
}
} else if (compression == ZLIB) {
if (auto const reason = nvcomp::is_compression_disabled(nvcomp::compression_type::DEFLATE);
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/io/parquet/writer_impl.cu
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,9 @@ struct leaf_schema_fn {
if (col_meta.is_decimal_precision_set()) {
CUDF_EXPECTS(col_meta.get_decimal_precision() >= col_schema.decimal_scale,
"Precision must be equal to or greater than scale!");
if (col_schema.type == Type::INT64 and col_meta.get_decimal_precision() < 10) {
CUDF_LOG_WARN("Parquet writer: writing a decimal column with precision < 10 as int64");
}
col_schema.decimal_precision = col_meta.get_decimal_precision();
}
}
Expand Down

0 comments on commit bd1ef29

Please sign in to comment.