From 6da1d2ec7f4e102a9d036b51e870592a315c1469 Mon Sep 17 00:00:00 2001 From: Philip de Nier Date: Fri, 26 May 2023 15:24:16 +0100 Subject: [PATCH 1/5] Add --http-disable-seek option to apps Can be used to speed up access in some cases that don't require random access or access to footer for complete header metadata. Seeking is enabled by default. --- apps/bmxtranswrap/bmxtranswrap.cpp | 7 +++++++ apps/mxf2raw/mxf2raw.cpp | 7 +++++++ include/bmx/MXFHTTPFile.h | 2 +- include/bmx/apps/AppMXFFileFactory.h | 2 ++ src/apps/AppMXFFileFactory.cpp | 8 +++++++- src/common/MXFHTTPFile.cpp | 14 +++++++++++--- src/mxf_helper/MXFFileFactory.cpp | 2 +- 7 files changed, 36 insertions(+), 6 deletions(-) diff --git a/apps/bmxtranswrap/bmxtranswrap.cpp b/apps/bmxtranswrap/bmxtranswrap.cpp index 74ea09bd..d70bffdb 100644 --- a/apps/bmxtranswrap/bmxtranswrap.cpp +++ b/apps/bmxtranswrap/bmxtranswrap.cpp @@ -419,6 +419,7 @@ static void usage(const char *cmd) if (mxf_http_is_supported()) { fprintf(stderr, " --http-min-read \n"); fprintf(stderr, " Set the minimum number of bytes to read when accessing a file over HTTP. The default is %u.\n", DEFAULT_HTTP_MIN_READ); + fprintf(stderr, " --http-disable-seek Disable seeking when reading file over HTTP\n"); } fprintf(stderr, " --no-precharge Don't output clip/track with precharge. Adjust the start position and duration instead\n"); fprintf(stderr, " --no-rollout Don't output clip/track with rollout. Adjust the duration instead\n"); @@ -906,6 +907,7 @@ int main(int argc, const char** argv) uint16_t rdd6_lines[2] = {DEFAULT_RDD6_LINES[0], DEFAULT_RDD6_LINES[1]}; uint8_t rdd6_sdid = DEFAULT_RDD6_SDID; uint32_t http_min_read = DEFAULT_HTTP_MIN_READ; + bool http_enable_seek = true; bool mp_track_num = false; #if defined(_WIN32) && !defined(__MINGW32__) bool use_mmap_file = false; @@ -1280,6 +1282,10 @@ int main(int argc, const char** argv) http_min_read = (uint32_t)(uvalue); cmdln_index++; } + else if (strcmp(argv[cmdln_index], "--http-disable-seek") == 0) + { + http_enable_seek = false; + } else if (strcmp(argv[cmdln_index], "--no-precharge") == 0) { no_precharge = true; @@ -2856,6 +2862,7 @@ int main(int argc, const char** argv) if (rw_interleave) file_factory.SetRWInterleave(rw_interleave_size); file_factory.SetHTTPMinReadSize(http_min_read); + file_factory.SetHTTPEnableSeek(http_enable_seek); #if defined(_WIN32) && !defined(__MINGW32__) file_factory.SetUseMMapFile(use_mmap_file); #endif diff --git a/apps/mxf2raw/mxf2raw.cpp b/apps/mxf2raw/mxf2raw.cpp index b011400f..260b1665 100644 --- a/apps/mxf2raw/mxf2raw.cpp +++ b/apps/mxf2raw/mxf2raw.cpp @@ -1690,6 +1690,7 @@ static void usage(const char *cmd) if (mxf_http_is_supported()) { fprintf(stderr, " --http-min-read \n"); fprintf(stderr, " Set the minimum number of bytes to read when accessing a file over HTTP. The default is %u.\n", DEFAULT_HTTP_MIN_READ); + fprintf(stderr, " --http-disable-seek Disable seeking when reading file over HTTP\n"); } fprintf(stderr, "\n"); fprintf(stderr, " --text-out Extract text based objects to files starting with \n"); @@ -1762,6 +1763,7 @@ int main(int argc, const char** argv) float gf_retry_delay = DEFAULT_GF_RETRY_DELAY; float gf_rate_after_fail = DEFAULT_GF_RATE_AFTER_FAIL; uint32_t http_min_read = DEFAULT_HTTP_MIN_READ; + bool http_enable_seek = true; ChecksumType checkum_type; #if defined(_WIN32) && !defined(__MINGW32__) bool use_mmap_file = false; @@ -2249,6 +2251,10 @@ int main(int argc, const char** argv) http_min_read = (uint32_t)(uvalue); cmdln_index++; } + else if (strcmp(argv[cmdln_index], "--http-disable-seek") == 0) + { + http_enable_seek = false; + } else if (strcmp(argv[cmdln_index], "--regtest") == 0) { BMX_REGRESSION_TEST = true; @@ -2409,6 +2415,7 @@ int main(int argc, const char** argv) file_factory.SetInputChecksumTypes(file_checksum_types); file_factory.SetInputFlags(file_flags); file_factory.SetHTTPMinReadSize(http_min_read); + file_factory.SetHTTPEnableSeek(http_enable_seek); #if defined(_WIN32) && !defined(__MINGW32__) file_factory.SetUseMMapFile(use_mmap_file); #endif diff --git a/include/bmx/MXFHTTPFile.h b/include/bmx/MXFHTTPFile.h index 079f09d4..f9489eca 100644 --- a/include/bmx/MXFHTTPFile.h +++ b/include/bmx/MXFHTTPFile.h @@ -45,7 +45,7 @@ bool mxf_http_is_supported(); bool mxf_http_is_url(const std::string &url_str); -MXFFile* mxf_http_file_open_read(const std::string &url_str, uint32_t min_read_size); +MXFFile* mxf_http_file_open_read(const std::string &url_str, uint32_t min_read_size, bool enable_seek); }; diff --git a/include/bmx/apps/AppMXFFileFactory.h b/include/bmx/apps/AppMXFFileFactory.h index 56ab1116..c0ac4f6c 100644 --- a/include/bmx/apps/AppMXFFileFactory.h +++ b/include/bmx/apps/AppMXFFileFactory.h @@ -66,6 +66,7 @@ class AppMXFFileFactory : public MXFFileFactory void SetInputFlags(int flags); void SetRWInterleave(uint32_t rw_interleave_size); void SetHTTPMinReadSize(uint32_t size); + void SetHTTPEnableSeek(bool enable); // Default true #if defined(_WIN32) && !defined(__MINGW32__) void SetUseMMapFile(bool enable); #endif @@ -105,6 +106,7 @@ class AppMXFFileFactory : public MXFFileFactory std::vector mInputChecksumFiles; MXFRWInterleaver *mRWInterleaver; uint32_t mHTTPMinReadSize; + bool mHTTPEnableSeek; #if defined(_WIN32) && !defined(__MINGW32__) bool mUseMMapFile; #endif diff --git a/src/apps/AppMXFFileFactory.cpp b/src/apps/AppMXFFileFactory.cpp index a8d09e88..0dfa8ea1 100644 --- a/src/apps/AppMXFFileFactory.cpp +++ b/src/apps/AppMXFFileFactory.cpp @@ -54,6 +54,7 @@ AppMXFFileFactory::AppMXFFileFactory() mInputFlags = 0; mRWInterleaver = 0; mHTTPMinReadSize = 64 * 1024; + mHTTPEnableSeek = true; #if defined(_WIN32) && !defined(__MINGW32__) mUseMMapFile = false; #endif @@ -98,6 +99,11 @@ void AppMXFFileFactory::SetHTTPMinReadSize(uint32_t size) mHTTPMinReadSize = size; } +void AppMXFFileFactory::SetHTTPEnableSeek(bool enable) +{ + mHTTPEnableSeek = enable; +} + #if defined(_WIN32) && !defined(__MINGW32__) void AppMXFFileFactory::SetUseMMapFile(bool enable) { @@ -152,7 +158,7 @@ File* AppMXFFileFactory::OpenRead(string filename) uri_str = "stdin:"; } else { if (mxf_http_is_url(filename)) { - mxf_file = mxf_http_file_open_read(filename, mHTTPMinReadSize); + mxf_file = mxf_http_file_open_read(filename, mHTTPMinReadSize, mHTTPEnableSeek); uri_str = filename; } else { #if defined(_WIN32) diff --git a/src/common/MXFHTTPFile.cpp b/src/common/MXFHTTPFile.cpp index 749ab1a1..63466a72 100644 --- a/src/common/MXFHTTPFile.cpp +++ b/src/common/MXFHTTPFile.cpp @@ -44,6 +44,7 @@ #include #include +#include #include #include @@ -452,7 +453,7 @@ bool bmx::mxf_http_is_url(const string &url_str) url_str.compare(0, 8, "https://") == 0; } -MXFFile* bmx::mxf_http_file_open_read(const string &url_str, uint32_t min_read_size) +MXFFile* bmx::mxf_http_file_open_read(const string &url_str, uint32_t min_read_size, bool enable_seek) { MXFFile *http_file = 0; try @@ -490,7 +491,13 @@ MXFFile* bmx::mxf_http_file_open_read(const string &url_str, uint32_t min_read_s http_file->size = http_file_size; http_file->free_sys_data = free_http_file; - return http_file; + if (enable_seek) { + return http_file; + } else { + MXFFile *stream_file = 0; + BMX_CHECK(mxf_stream_file_wrap(http_file, true, &stream_file)); + return stream_file; + } } catch (...) { @@ -525,10 +532,11 @@ bool bmx::mxf_http_is_url(const string &url_str) url_str.compare(0, 8, "https://") == 0; } -MXFFile* bmx::mxf_http_file_open_read(const string &url_str, uint32_t min_read_size) +MXFFile* bmx::mxf_http_file_open_read(const string &url_str, uint32_t min_read_size, bool enable_seek) { (void)url_str; (void)min_read_size; + (void)enable_seek; BMX_EXCEPTION(("HTTP file access is not supported in this build")); } diff --git a/src/mxf_helper/MXFFileFactory.cpp b/src/mxf_helper/MXFFileFactory.cpp index d0b9c3b7..5ac78d98 100644 --- a/src/mxf_helper/MXFFileFactory.cpp +++ b/src/mxf_helper/MXFFileFactory.cpp @@ -59,7 +59,7 @@ File* DefaultMXFFileFactory::OpenRead(string filename) BMX_CHECK(mxf_stdin_wrap_read(&mxf_file)); return new File(mxf_file); } else if (mxf_http_is_url(filename)) { - return new File(mxf_http_file_open_read(filename, 64 * 1024)); + return new File(mxf_http_file_open_read(filename, 64 * 1024), true); } else { return File::openRead(filename); } From 81235c6e81c547eba98827368aa179b9e1e15951 Mon Sep 17 00:00:00 2001 From: Philip de Nier Date: Fri, 26 May 2023 15:30:13 +0100 Subject: [PATCH 2/5] Change internal library default HTTP read sizes --- src/apps/AppMXFFileFactory.cpp | 2 +- src/mxf_helper/MXFFileFactory.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/AppMXFFileFactory.cpp b/src/apps/AppMXFFileFactory.cpp index 0dfa8ea1..4e65b2f1 100644 --- a/src/apps/AppMXFFileFactory.cpp +++ b/src/apps/AppMXFFileFactory.cpp @@ -53,7 +53,7 @@ AppMXFFileFactory::AppMXFFileFactory() { mInputFlags = 0; mRWInterleaver = 0; - mHTTPMinReadSize = 64 * 1024; + mHTTPMinReadSize = 1024 * 1024; mHTTPEnableSeek = true; #if defined(_WIN32) && !defined(__MINGW32__) mUseMMapFile = false; diff --git a/src/mxf_helper/MXFFileFactory.cpp b/src/mxf_helper/MXFFileFactory.cpp index 5ac78d98..e893e756 100644 --- a/src/mxf_helper/MXFFileFactory.cpp +++ b/src/mxf_helper/MXFFileFactory.cpp @@ -59,7 +59,7 @@ File* DefaultMXFFileFactory::OpenRead(string filename) BMX_CHECK(mxf_stdin_wrap_read(&mxf_file)); return new File(mxf_file); } else if (mxf_http_is_url(filename)) { - return new File(mxf_http_file_open_read(filename, 64 * 1024), true); + return new File(mxf_http_file_open_read(filename, 1024 * 1024, true)); } else { return File::openRead(filename); } From 0a14e44aff1c79bf93df7e87da444f427721e6d7 Mon Sep 17 00:00:00 2001 From: Philip de Nier Date: Fri, 26 May 2023 16:03:24 +0100 Subject: [PATCH 3/5] apps: clarify incomplete file message and make debug when not seekable --- apps/bmxtranswrap/bmxtranswrap.cpp | 7 +++++-- apps/mxf2raw/mxf2raw.cpp | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/bmxtranswrap/bmxtranswrap.cpp b/apps/bmxtranswrap/bmxtranswrap.cpp index d70bffdb..1a9ab750 100644 --- a/apps/bmxtranswrap/bmxtranswrap.cpp +++ b/apps/bmxtranswrap/bmxtranswrap.cpp @@ -403,7 +403,7 @@ static void usage(const char *cmd) fprintf(stderr, " --dur Set the duration in input edit rate units. Default is minimum input duration\n"); fprintf(stderr, " --check-end Check at the start that the last (start + duration - 1) edit unit can be read\n"); fprintf(stderr, " --check-end-tolerance Allow output duration shorter than input declared duration\n"); - fprintf(stderr, " --check-complete Check that the input file is complete\n"); + fprintf(stderr, " --check-complete Check that the input file structure can be read and is complete\n"); fprintf(stderr, " --group Use the group reader instead of the sequence reader\n"); fprintf(stderr, " Use this option if the files have different material packages\n"); fprintf(stderr, " but actually belong to the same virtual package / group\n"); @@ -2940,7 +2940,10 @@ int main(int argc, const char** argv) log_error("Input file is incomplete\n"); throw false; } - log_warn("Input file is incomplete\n"); + if (reader->IsSeekable()) + log_warn("Input file is incomplete\n"); + else + log_debug("Input file is incomplete, probably because the file is not seekable\n"); } reader->SetEmptyFrames(true); diff --git a/apps/mxf2raw/mxf2raw.cpp b/apps/mxf2raw/mxf2raw.cpp index 260b1665..10c79b8d 100644 --- a/apps/mxf2raw/mxf2raw.cpp +++ b/apps/mxf2raw/mxf2raw.cpp @@ -1624,7 +1624,7 @@ static void usage(const char *cmd) fprintf(stderr, " Use this option for files with broken timecode\n"); fprintf(stderr, "\n"); fprintf(stderr, " --check-end Check that the last edit unit (start + duration - 1) can be read when opening the files\n"); - fprintf(stderr, " --check-complete Check that the input files are complete\n"); + fprintf(stderr, " --check-complete Check that the input file structure info can be read and is complete\n"); fprintf(stderr, " --check-app-issues Check that there are no known issues with the APP (Archive Preservation Project) file\n"); fprintf(stderr, " --check-app-crc32 Check APP essence CRC-32 data\n"); fprintf(stderr, "\n"); @@ -2501,7 +2501,7 @@ int main(int argc, const char** argv) if (reader->IsSeekable()) log_warn("Input file is incomplete\n"); else - log_debug("Input file is not seekable\n"); + log_debug("Input file is incomplete, probably because the file is not seekable\n"); } mxfRational edit_rate = reader->GetEditRate(); From 3a7109e3219a15e127e45582fbf0f2f5ddb72eef Mon Sep 17 00:00:00 2001 From: Philip de Nier Date: Fri, 26 May 2023 16:33:07 +0100 Subject: [PATCH 4/5] mxf2raw: extend cases where info is output by default E.g. if `--http-disable-seek` is used then still output the file info if not other action has been selected. --- apps/mxf2raw/mxf2raw.cpp | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/apps/mxf2raw/mxf2raw.cpp b/apps/mxf2raw/mxf2raw.cpp index 10c79b8d..8508d527 100644 --- a/apps/mxf2raw/mxf2raw.cpp +++ b/apps/mxf2raw/mxf2raw.cpp @@ -1708,6 +1708,7 @@ static void usage(const char *cmd) int main(int argc, const char** argv) { + bool have_action = false; // true when an option is selected to take a specific action std::vector input_filenames; const char *log_filename = 0; LogLevel log_level = INFO_LOG; @@ -1795,6 +1796,7 @@ int main(int argc, const char** argv) return 0; } fprintf(stderr, "%s\n", get_app_version_info(APP_NAME).c_str()); + have_action = true; } else if (strcmp(argv[cmdln_index], "-l") == 0) { @@ -1838,6 +1840,7 @@ int main(int argc, const char** argv) return 1; } file_checksum_only_types.insert(checkum_type); + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--group") == 0) @@ -1851,25 +1854,30 @@ int main(int argc, const char** argv) else if (strcmp(argv[cmdln_index], "--check-end") == 0) { check_end = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--check-complete") == 0) { check_complete = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--check-app-issues") == 0) { check_app_issues = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--check-app-crc32") == 0) { check_app_crc32 = true; do_ess_read = true; do_write_info = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "-i") == 0 || strcmp(argv[cmdln_index], "--info") == 0) { do_write_info = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--info-format") == 0) { @@ -1917,6 +1925,7 @@ int main(int argc, const char** argv) track_checksum_types.insert(checkum_type); do_write_info = true; do_ess_read = true; + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--file-chksum") == 0) @@ -1935,22 +1944,26 @@ int main(int argc, const char** argv) } file_checksum_types.insert(checkum_type); do_write_info = true; + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--as11") == 0) { do_as11_info = true; do_write_info = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--as10") == 0) { do_as10_info = true; do_write_info = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--app") == 0) { do_app_info = true; do_write_info = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--app-events") == 0) { @@ -1968,6 +1981,7 @@ int main(int argc, const char** argv) } do_app_info = true; do_write_info = true; + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--no-app-events-tc") == 0) @@ -1984,6 +1998,7 @@ int main(int argc, const char** argv) } app_crc32_filename = argv[cmdln_index + 1]; do_ess_read = true; + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--app-tc") == 0) @@ -1996,6 +2011,7 @@ int main(int argc, const char** argv) } app_tc_filename = argv[cmdln_index + 1]; do_ess_read = true; + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--all-tc") == 0) @@ -2008,6 +2024,7 @@ int main(int argc, const char** argv) } all_tc_filename = argv[cmdln_index + 1]; do_ess_read = true; + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--index") == 0) @@ -2015,11 +2032,13 @@ int main(int argc, const char** argv) do_index_info = true; do_write_info = true; do_parse_read = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--avid") == 0) { do_avid_info = true; do_write_info = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--st436-mf") == 0) { @@ -2037,6 +2056,7 @@ int main(int argc, const char** argv) } st436_manifest_count = (uint32_t)(uvalue); do_write_info = true; + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--rdd6") == 0) @@ -2054,12 +2074,14 @@ int main(int argc, const char** argv) return 1; } rdd6_filename = argv[cmdln_index + 2]; + have_action = true; cmdln_index += 2; } else if (strcmp(argv[cmdln_index], "--mca-detail") == 0) { mca_detail = true; do_write_info = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "-p") == 0 || strcmp(argv[cmdln_index], "--ess-out") == 0) @@ -2072,6 +2094,7 @@ int main(int argc, const char** argv) } ess_output_prefix = argv[cmdln_index + 1]; do_ess_read = true; + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--wrap-klv") == 0) @@ -2093,6 +2116,7 @@ int main(int argc, const char** argv) else if (strcmp(argv[cmdln_index], "--read-ess") == 0) { do_ess_read = true; + have_action = true; } else if (strcmp(argv[cmdln_index], "--deint") == 0) { @@ -2232,6 +2256,7 @@ int main(int argc, const char** argv) return 1; } text_output_prefix = argv[cmdln_index + 1]; + have_action = true; cmdln_index++; } else if (strcmp(argv[cmdln_index], "--http-min-read") == 0) @@ -2275,8 +2300,8 @@ int main(int argc, const char** argv) return 1; } - if (cmdln_index == 1) { - // default to outputting info if no options are given + if (!have_action) { + // default to outputting info if no specific action has been selected do_write_info = true; } From 81eb62bac9618c9fe72599ee8a4b49721fe5e1eb Mon Sep 17 00:00:00 2001 From: Philip de Nier Date: Fri, 26 May 2023 19:10:24 +0100 Subject: [PATCH 5/5] app: add --disable-indexing-file option for faster file access Using this option will stop the reader indexing the locations of essence. That is useful if there are many partitions for example and it's more efficient to not index and instead treat the file as incomplete. It may not be efficient if essence is read and the --start option is used with a position not near the start. The footer is still searched for and used to extract header metadata (if available). This ensures that the most up-to-date metadata is used. The footer is found either using the RIP or the footerPartition property in the header partition pack. --- apps/bmxtranswrap/bmxtranswrap.cpp | 12 ++- apps/mxf2raw/mxf2raw.cpp | 12 ++- deps/libMXFpp | 2 +- include/bmx/mxf_reader/MXFFileReader.h | 2 + src/mxf_reader/MXFFileReader.cpp | 108 ++++++++++++++++++------- 5 files changed, 105 insertions(+), 31 deletions(-) diff --git a/apps/bmxtranswrap/bmxtranswrap.cpp b/apps/bmxtranswrap/bmxtranswrap.cpp index 1a9ab750..09bf2d63 100644 --- a/apps/bmxtranswrap/bmxtranswrap.cpp +++ b/apps/bmxtranswrap/bmxtranswrap.cpp @@ -416,6 +416,8 @@ static void usage(const char *cmd) fprintf(stderr, " --gf-delay Set the delay (in seconds) between a failure to read and a retry. The default is %f.\n", DEFAULT_GF_RETRY_DELAY); fprintf(stderr, " --gf-rate Limit the read rate to realtime rate x after a read failure. The default is %f\n", DEFAULT_GF_RATE_AFTER_FAIL); fprintf(stderr, " value 1.0 results in realtime rate, value < 1.0 slower and > 1.0 faster\n"); + fprintf(stderr, " --disable-indexing-file Use this option to stop the reader creating an index of the partitions and essence positions in the file up front\n"); + fprintf(stderr, " This option can be used to avoid indexing files containing many partitions\n"); if (mxf_http_is_supported()) { fprintf(stderr, " --http-min-read \n"); fprintf(stderr, " Set the minimum number of bytes to read when accessing a file over HTTP. The default is %u.\n", DEFAULT_HTTP_MIN_READ); @@ -883,6 +885,7 @@ int main(int argc, const char** argv) unsigned int gf_retries = DEFAULT_GF_RETRIES; float gf_retry_delay = DEFAULT_GF_RETRY_DELAY; float gf_rate_after_fail = DEFAULT_GF_RATE_AFTER_FAIL; + bool enable_indexing_file = true; bool product_info_set = false; string company_name; string product_name; @@ -1265,6 +1268,10 @@ int main(int argc, const char** argv) growing_file = true; cmdln_index++; } + else if (strcmp(argv[cmdln_index], "--disable-indexing-file") == 0) + { + enable_indexing_file = false; + } else if (strcmp(argv[cmdln_index], "--http-min-read") == 0) { if (cmdln_index + 1 >= argc) @@ -2876,6 +2883,7 @@ int main(int argc, const char** argv) grp_file_reader->SetFileFactory(&file_factory, false); grp_file_reader->GetPackageResolver()->SetFileFactory(&file_factory, false); grp_file_reader->SetST436ManifestFrameCount(st436_manifest_count); + grp_file_reader->SetEnableIndexFile(enable_indexing_file); result = grp_file_reader->Open(input_filenames[i]); if (result != MXFFileReader::MXF_RESULT_SUCCESS) { log_error("Failed to open MXF file '%s': %s\n", input_filenames[i], @@ -2899,6 +2907,7 @@ int main(int argc, const char** argv) seq_file_reader->SetFileFactory(&file_factory, false); seq_file_reader->GetPackageResolver()->SetFileFactory(&file_factory, false); seq_file_reader->SetST436ManifestFrameCount(st436_manifest_count); + seq_file_reader->SetEnableIndexFile(enable_indexing_file); result = seq_file_reader->Open(input_filenames[i]); if (result != MXFFileReader::MXF_RESULT_SUCCESS) { log_error("Failed to open MXF file '%s': %s\n", input_filenames[i], @@ -2919,6 +2928,7 @@ int main(int argc, const char** argv) file_reader->SetFileFactory(&file_factory, false); file_reader->GetPackageResolver()->SetFileFactory(&file_factory, false); file_reader->SetST436ManifestFrameCount(st436_manifest_count); + file_reader->SetEnableIndexFile(enable_indexing_file); if (pass_dm && clip_sub_type == AS11_CLIP_SUB_TYPE) AS11Info::RegisterExtensions(file_reader->GetHeaderMetadata()); if (pass_dm && clip_sub_type == AS10_CLIP_SUB_TYPE) @@ -2940,7 +2950,7 @@ int main(int argc, const char** argv) log_error("Input file is incomplete\n"); throw false; } - if (reader->IsSeekable()) + if (reader->IsSeekable() && enable_indexing_file) log_warn("Input file is incomplete\n"); else log_debug("Input file is incomplete, probably because the file is not seekable\n"); diff --git a/apps/mxf2raw/mxf2raw.cpp b/apps/mxf2raw/mxf2raw.cpp index 8508d527..8b842752 100644 --- a/apps/mxf2raw/mxf2raw.cpp +++ b/apps/mxf2raw/mxf2raw.cpp @@ -1687,6 +1687,8 @@ static void usage(const char *cmd) fprintf(stderr, " --gf-delay Set the delay (in seconds) between a failure to read and a retry. The default is %f.\n", DEFAULT_GF_RETRY_DELAY); fprintf(stderr, " --gf-rate Limit the read rate to realtime rate x after a read failure. The default is %f\n", DEFAULT_GF_RATE_AFTER_FAIL); fprintf(stderr, " value 1.0 results in realtime rate, value < 1.0 slower and > 1.0 faster\n"); + fprintf(stderr, " --disable-indexing-file Use this option to stop the reader creating an index of the partitions and essence positions in the file up front\n"); + fprintf(stderr, " This option can be used to avoid indexing files containing many partitions\n"); if (mxf_http_is_supported()) { fprintf(stderr, " --http-min-read \n"); fprintf(stderr, " Set the minimum number of bytes to read when accessing a file over HTTP. The default is %u.\n", DEFAULT_HTTP_MIN_READ); @@ -1763,6 +1765,7 @@ int main(int argc, const char** argv) unsigned int gf_retries = DEFAULT_GF_RETRIES; float gf_retry_delay = DEFAULT_GF_RETRY_DELAY; float gf_rate_after_fail = DEFAULT_GF_RATE_AFTER_FAIL; + bool enable_indexing_file = true; uint32_t http_min_read = DEFAULT_HTTP_MIN_READ; bool http_enable_seek = true; ChecksumType checkum_type; @@ -2247,6 +2250,10 @@ int main(int argc, const char** argv) growing_file = true; cmdln_index++; } + else if (strcmp(argv[cmdln_index], "--disable-indexing-file") == 0) + { + enable_indexing_file = false; + } else if (strcmp(argv[cmdln_index], "--text-out") == 0) { if (cmdln_index + 1 >= argc) @@ -2455,6 +2462,7 @@ int main(int argc, const char** argv) grp_file_reader->SetFileFactory(&file_factory, false); grp_file_reader->GetPackageResolver()->SetFileFactory(&file_factory, false); grp_file_reader->SetST436ManifestFrameCount(st436_manifest_count); + grp_file_reader->SetEnableIndexFile(enable_indexing_file); result = grp_file_reader->Open(input_filenames[i], input_open_flags); if (result != MXFFileReader::MXF_RESULT_SUCCESS) { log_error("Failed to open MXF file '%s': %s\n", get_input_filename(input_filenames[i]), @@ -2478,6 +2486,7 @@ int main(int argc, const char** argv) seq_file_reader->SetFileFactory(&file_factory, false); seq_file_reader->GetPackageResolver()->SetFileFactory(&file_factory, false); seq_file_reader->SetST436ManifestFrameCount(st436_manifest_count); + seq_file_reader->SetEnableIndexFile(enable_indexing_file); result = seq_file_reader->Open(input_filenames[i], input_open_flags); if (result != MXFFileReader::MXF_RESULT_SUCCESS) { log_error("Failed to open MXF file '%s': %s\n", get_input_filename(input_filenames[i]), @@ -2498,6 +2507,7 @@ int main(int argc, const char** argv) file_reader->SetFileFactory(&file_factory, false); file_reader->GetPackageResolver()->SetFileFactory(&file_factory, false); file_reader->SetST436ManifestFrameCount(st436_manifest_count); + file_reader->SetEnableIndexFile(enable_indexing_file); if (do_as11_info) as11_register_extensions(file_reader); if (do_as10_info) @@ -2523,7 +2533,7 @@ int main(int argc, const char** argv) complete_result = false; cmd_result = 1; } - if (reader->IsSeekable()) + if (reader->IsSeekable() && enable_indexing_file) log_warn("Input file is incomplete\n"); else log_debug("Input file is incomplete, probably because the file is not seekable\n"); diff --git a/deps/libMXFpp b/deps/libMXFpp index 7fae6854..28c954c4 160000 --- a/deps/libMXFpp +++ b/deps/libMXFpp @@ -1 +1 @@ -Subproject commit 7fae6854663388f464b2d875fa36bbc890b00f9a +Subproject commit 28c954c429f515ceb7ecff59089967c8e2566101 diff --git a/include/bmx/mxf_reader/MXFFileReader.h b/include/bmx/mxf_reader/MXFFileReader.h index fe6465e5..0248d78d 100644 --- a/include/bmx/mxf_reader/MXFFileReader.h +++ b/include/bmx/mxf_reader/MXFFileReader.h @@ -99,6 +99,7 @@ class MXFFileReader : public MXFReader void SetST436ManifestFrameCount(uint32_t count); // default: 2 frames used to extract manifest virtual void SetFileIndex(MXFFileIndex *file_index, bool take_ownership); virtual void SetMCALabelIndex(MXFMCALabelIndex *label_index, bool take_ownership); + void SetEnableIndexFile(bool enable); // Default true OpenResult Open(std::string filename, int mode_flags=0); OpenResult Open(mxfpp::File *file, std::string filename, int mode_flags=0); @@ -259,6 +260,7 @@ class MXFFileReader : public MXFReader std::vector mTextObjects; // internal and external text objects std::vector mInternalTextObjects; + bool mEnableIndexFile; EssenceReader *mEssenceReader; uint32_t mRequireFrameInfoCount; diff --git a/src/mxf_reader/MXFFileReader.cpp b/src/mxf_reader/MXFFileReader.cpp index e34479dd..64052924 100644 --- a/src/mxf_reader/MXFFileReader.cpp +++ b/src/mxf_reader/MXFFileReader.cpp @@ -162,6 +162,7 @@ MXFFileReader::MXFFileReader() mReadStartPosition = 0; mReadDuration = -1; mFileOrigin = 0; + mEnableIndexFile = true; mEssenceReader = 0; mRequireFrameInfoCount = 0; mST436ManifestCount = 2; @@ -250,6 +251,11 @@ void MXFFileReader::SetMCALabelIndex(MXFMCALabelIndex *label_index, bool take_ow mExternalReaders[i]->SetMCALabelIndex(label_index, false); } +void MXFFileReader::SetEnableIndexFile(bool enable) +{ + mEnableIndexFile = enable; +} + MXFFileReader::OpenResult MXFFileReader::Open(string filename, int mode_flags) { File *file = 0; @@ -337,47 +343,93 @@ MXFFileReader::OpenResult MXFFileReader::Open(File *file, const URI &abs_uri, co } - // try read all partitions find and get last partition with header metadata + // try read all partitions find and the last partition with header metadata + + bool file_is_complete = false; + Partition *metadata_partition = 0; + bool own_metadata_partition = false; + if (mEnableIndexFile) { + if (mFile->isSeekable()) { + file_is_complete = mFile->readPartitions(); + if (!file_is_complete) { + BMX_ASSERT(mFile->getPartitions().size() == 1); + if (mFile->getPartition(0).isClosed() || mFile->getPartition(0).getFooterPartition() != 0) + log_warn("Failed to read all partitions. File may be incomplete or invalid\n"); + } + } - bool file_is_complete; - if (mFile->isSeekable()) { - file_is_complete = mFile->readPartitions(); - if (!file_is_complete) { - BMX_ASSERT(mFile->getPartitions().size() == 1); - if (mFile->getPartition(0).isClosed() || mFile->getPartition(0).getFooterPartition() != 0) - log_warn("Failed to read all partitions. File may be incomplete or invalid\n"); + if (file_is_complete) { + const vector &partitions = mFile->getPartitions(); + for (i = partitions.size(); i > 0 ; i--) { + if (partitions[i - 1]->getHeaderByteCount() > 0) { + metadata_partition = partitions[i - 1]; + break; + } + } + } else { + metadata_partition = &header_partition; } } else { - file_is_complete = false; - } - const vector &partitions = mFile->getPartitions(); - Partition *metadata_partition = 0; - for (i = partitions.size(); i > 0 ; i--) { - if (partitions[i - 1]->getHeaderByteCount() > 0) { - metadata_partition = partitions[i - 1]; - break; + // Only try reading the footer partition to see if it has metadata (if seeking is possible) + if (mFile->isSeekable()) { + Partition *footer_partition = mFile->readFooterPartition(); + if (footer_partition) { + if (footer_partition->getHeaderByteCount() > 0) { + metadata_partition = footer_partition; + own_metadata_partition = true; + } else { + delete footer_partition; + } + } + } + if (!metadata_partition && header_partition.getHeaderByteCount() > 0) { + metadata_partition = &header_partition; } } + if (!metadata_partition) THROW_RESULT(MXF_RESULT_NO_HEADER_METADATA); // read and process the header metadata - mxfKey key; - uint8_t llen; - uint64_t len; - if (mFile->isSeekable()) { - mFile->seek(metadata_partition->getThisPartition(), SEEK_SET); - mFile->readKL(&key, &llen, &len); - mFile->skip(len); - } - mFile->readNextNonFillerKL(&key, &llen, &len); - BMX_CHECK(mxf_is_header_metadata(&key)); + try + { + mxfKey key; + uint8_t llen; + uint64_t len; + if (mFile->isSeekable()) { + mFile->seek(metadata_partition->getThisPartition(), SEEK_SET); + mFile->readKL(&key, &llen, &len); + mFile->skip(len); + } + mFile->readNextNonFillerKL(&key, &llen, &len); + BMX_CHECK(mxf_is_header_metadata(&key)); + + mHeaderMetadata->read(mFile, metadata_partition, &key, llen, len); - mHeaderMetadata->read(mFile, metadata_partition, &key, llen, len); + ProcessMetadata(metadata_partition); + + if (!file_is_complete && metadata_partition != &header_partition && mFile->isSeekable()) { + // The mFile->getPartitions().size() == 1 when the file is incomplete and so the + // EssenceReader will assume that the file was positioned after the header partition pack. + // In this case the header metadata was read from the footer and so a seek is needed back + // to after the header partition pack. + mFile->seek(header_partition.getThisPartition(), SEEK_SET); + mFile->readKL(&key, &llen, &len); + mFile->skip(len); + } - ProcessMetadata(metadata_partition); + if (own_metadata_partition) + delete metadata_partition; + metadata_partition = 0; + } + catch (...) + { + if (own_metadata_partition) + delete metadata_partition; + throw; + } // create internal essence reader