Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Log cuIO warnings using the libcudf logger #13043

Merged
merged 6 commits into from
Apr 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion cpp/benchmarks/io/cuio_common.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/

#include <benchmarks/io/cuio_common.hpp>
#include <cudf/detail/utilities/logger.hpp>

#include <cstdio>
#include <fstream>
Expand Down Expand Up @@ -173,10 +174,24 @@ std::string exec_cmd(std::string_view cmd)
return error_out;
}

void log_l3_warning_once()
{
static bool is_logged = false;
if (not is_logged) {
CUDF_LOG_WARN(
"Running benchmarks without dropping the L3 cache; results may not reflect file IO "
"throughput");
is_logged = true;
}
}

void try_drop_l3_cache()
{
static bool is_drop_cache_enabled = std::getenv("CUDF_BENCHMARK_DROP_CACHE") != nullptr;
if (not is_drop_cache_enabled) { return; }
if (not is_drop_cache_enabled) {
log_l3_warning_once();
return;
}

std::array drop_cache_cmds{"/sbin/sysctl vm.drop_caches=3", "sudo /sbin/sysctl vm.drop_caches=3"};
CUDF_EXPECTS(std::any_of(drop_cache_cmds.cbegin(),
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/io/comp/nvcomp_adapter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,9 @@ size_t batched_compress_temp_size(compression_type compression,
compression, num_chunks, max_uncomp_chunk_size, max_total_uncomp_size);
} catch (...) {
// Ignore errors in the expanded version; fall back to the old API in case of failure
CUDF_LOG_WARN(
"CompressGetTempSizeEx call failed, falling back to CompressGetTempSize; this may increase "
"the memory usage");
}
#endif

Expand Down
4 changes: 3 additions & 1 deletion cpp/src/io/csv/reader_impl.cu
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,9 @@ table_with_metadata read_csv(cudf::io::datasource* source,
if (!reader_opts.is_enabled_mangle_dupe_cols()) {
for (auto& col_name : column_names) {
if (++col_names_counts[col_name] > 1) {
// All duplicate columns will be ignored; First appearance is parsed
CUDF_LOG_WARN("Multiple columns with name {}; only the first appearance is parsed",
col_name);

const auto idx = &col_name - column_names.data();
column_flags[idx] = column_parse::disabled;
}
Expand Down
1 change: 1 addition & 0 deletions cpp/src/io/orc/stripe_enc.cu
Original file line number Diff line number Diff line change
Expand Up @@ -1334,6 +1334,7 @@ void CompressOrcDataStreams(uint8_t* compressed_data,
[] __device__(compression_result & stat) { stat.status = compression_status::FAILURE; });
// Since SNAPPY is the default compression (may not be explicitly requested), fall back to
// writing without compression
CUDF_LOG_WARN("ORC writer: compression failed, writing uncompressed data");
}
} else if (compression == ZLIB) {
if (auto const reason = nvcomp::is_compression_disabled(nvcomp::compression_type::DEFLATE);
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/io/parquet/writer_impl.cu
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,9 @@ struct leaf_schema_fn {
if (col_meta.is_decimal_precision_set()) {
CUDF_EXPECTS(col_meta.get_decimal_precision() >= col_schema.decimal_scale,
"Precision must be equal to or greater than scale!");
if (col_schema.type == Type::INT64 and col_meta.get_decimal_precision() < 10) {
CUDF_LOG_WARN("Parquet writer: writing a decimal column with precision < 10 as int64");
}
col_schema.decimal_precision = col_meta.get_decimal_precision();
}
}
Expand Down