From 2ed2c9aa7d700327b648bb7930bce99a1a9ff66d Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Sat, 28 Nov 2020 11:16:33 +0100 Subject: [PATCH] Changes for EWF write --- .github/workflows/build.yml | 3 ++ appveyor.yml | 30 ++++++++------- configure.ac | 26 ++++++++++++- ewftools/process_status.c | 6 ++- libewf.nuspec | 2 +- libewf/libewf_handle.c | 60 +++++++++++++++++++++++++++++- libewf/libewf_write_io_handle.c | 7 +++- libmfdata/libmfdata_list_element.c | 2 +- m4/libbfio.m4 | 40 +++++++++++++++++++- pyewf/pyewf.c | 18 ++++----- runtests.sh | 24 ++++++------ tests/Makefile.am | 23 +++++++----- tests/ewf_test_chunk_data.c | 4 ++ tests/ewf_test_compression.c | 4 ++ 14 files changed, 196 insertions(+), 53 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 400f387..08b2358 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,9 @@ jobs: - architecture: 'x64' compiler: 'gcc' configure_options: '--enable-wide-character-type' + - architecture: 'x64' + compiler: 'gcc' + configure_options: '--with-zlib=no' - architecture: 'x64' compiler: 'gcc' configure_options: '--with-openssl=no' diff --git a/appveyor.yml b/appveyor.yml index 5e87815..771b849 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,6 @@ environment: + nuget_token: + secure: s3Sl1ZiRw+J7YD+2wr1elu0unq6koZek+JmOcYKfDPfwrz2NigSu5SD8WvlwdIcH matrix: - TARGET: vs2008 BUILD_ENVIRONMENT: msbuild @@ -153,7 +155,8 @@ install: - cmd: git clone https://github.com/codecov/codecov-bash.git ..\codecov-bash - cmd: if [%BUILD_ENVIRONMENT%]==[msbuild] ( git clone https://github.com/libyal/vstools.git ..\vstools ) -- ps: .\syncwinflexbison.ps1 +- ps: If ($env:BUILD_ENVIRONMENT -eq "msbuild") { + .\syncwinflexbison.ps1 } - ps: If ($env:BUILD_ENVIRONMENT -eq "msbuild") { .\synczlib.ps1 } - ps: If ($env:BUILD_ENVIRONMENT -eq "msbuild") { @@ -200,9 +203,9 @@ install: Else { .\builddokan.ps1 -UseLegacyVersion } } -- sh: if test ${BUILD_ENVIRONMENT} = "xcode" || test ${BUILD_ENVIRONMENT} = "python"; then brew update; fi -- sh: if test ${BUILD_ENVIRONMENT} = "xcode" || test ${BUILD_ENVIRONMENT} = "python"; then brew install gettext gnu-sed || true; fi -- sh: if test ${BUILD_ENVIRONMENT} = "python"; then brew python3 || true; fi +- sh: if test ${BUILD_ENVIRONMENT} = "xcode" || test ${BUILD_ENVIRONMENT} = "python"; then brew update -q; fi +- sh: if test ${BUILD_ENVIRONMENT} = "xcode" || test ${BUILD_ENVIRONMENT} = "python"; then brew install -q gettext gnu-sed || true; fi +- sh: if test ${BUILD_ENVIRONMENT} = "python"; then brew install -q python3 || true; fi - ps: If ($env:BUILD_ENVIRONMENT -eq "python") { Invoke-Expression "${env:PYTHON} -m pip install -U pip setuptools twine wheel" } - cmd: if [%BUILD_ENVIRONMENT%]==[cygwin] ( @@ -231,6 +234,10 @@ build_script: - ps: If ($env:TARGET -eq "vs2017") { .\build.ps1 -VisualStudioVersion 2017 -Platform Win32 -VSToolsOptions "--extend-with-x64 --no-python-dll --with-dokany"; .\build.ps1 -VisualStudioVersion 2017 -Platform x64 -VSToolsOptions "--extend-with-x64 --no-python-dll --with-dokany" } +- cmd: if [%TARGET%]==[vs2017] ( + move msvscpp vs2008 && + move vs2017 msvscpp && + nuget pack libewf.nuspec ) - ps: If ($env:TARGET -eq "vs2019") { .\build.ps1 -VisualStudioVersion 2019 -VSToolsOptions "--extend-with-x64 --no-python-dll --with-dokany" } - sh: if test ${BUILD_ENVIRONMENT} = "xcode" || test ${BUILD_ENVIRONMENT} = "python"; then export PATH="/usr/local/opt/gettext/bin:$PATH"; export SED="/usr/local/bin/gsed"; tests/build.sh ${CONFIGURE_OPTIONS}; fi @@ -272,14 +279,9 @@ after_test: artifacts: - path: \*.nupkg +- path: dist\*.whl + +deploy_script: +- ps: If ($env:APPVEYOR_REPO_TAG -eq "true" -and $isWindows -and $env:TARGET -eq "vs2017") { + Invoke-Expression "nuget push *.nupkg -NonInteractive -NoSymbols -Source https://api.nuget.org/v3/index.json -ApiKey ${env:NUGET_TOKEN}" } -deploy: -- provider: NuGet - skip_symbols: true - api_key: - secure: gCkz3PNGORdxJGl+aegOa7OJ4IQFHC/FqgCzbirMk7nyNFCuZzc1/ne2174eYxsw - artifact: /.*\.nupkg/ - on: - branch: master - appveyor_repo_tag: true - target: vs2017 diff --git a/configure.ac b/configure.ac index 6c10e02..3cd3271 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ( 2.59 ) AC_INIT( [libewf], - [20140810], + [20140811], [joachim.metz@gmail.com]) AC_CONFIG_SRCDIR( @@ -129,6 +129,29 @@ AX_LIBHMAC_CHECK_ENABLE dnl Check if libewf required headers and functions are available AX_LIBEWF_CHECK_LOCAL +AS_IF( + [test "x$ac_cv_uncompress" = xzlib], + [ac_cv_enable_write_support=yes], + [ac_cv_enable_write_support=no]) + +AS_IF( + [test "x$ac_cv_uncompress" = xzlib], + [AC_DEFINE( + [HAVE_WRITE_SUPPORT], + [1], + [Define to 1 if write support is available.]) + AC_SUBST( + [HAVE_WRITE_SUPPORT], + [1]) ], + [AC_SUBST( + [HAVE_WRITE_SUPPORT], + [0]) + ]) + +AM_CONDITIONAL( + HAVE_WRITE_TESTS, + [test "x$ac_cv_uncompress" = xzlib]) + dnl Check if libewf should be build with version 1 API AX_LIBEWF_CHECK_ENABLE_V1_API @@ -287,6 +310,7 @@ Building: Features: Multi-threading support: $ac_cv_libcthreads_multi_threading Wide character type support: $ac_cv_enable_wide_character_type + Write support: $ac_cv_enable_write_support ewftools are build as static executables: $ac_cv_enable_static_executables Python (pyewf) support: $ac_cv_enable_python Verbose output: $ac_cv_enable_verbose_output diff --git a/ewftools/process_status.c b/ewftools/process_status.c index 934bd35..f25ec98 100644 --- a/ewftools/process_status.c +++ b/ewftools/process_status.c @@ -480,10 +480,14 @@ int process_status_update( process_status->output_stream, bytes_total, total_number_of_seconds ); + + fprintf( + process_status->output_stream, + ".\n" ); } fprintf( process_status->output_stream, - ".\n\n" ); + "\n" ); } } return( 1 ); diff --git a/libewf.nuspec b/libewf.nuspec index ad9f235..9451424 100644 --- a/libewf.nuspec +++ b/libewf.nuspec @@ -2,7 +2,7 @@ libewf - 20140810 + 20140811 Joachim Metz joachimmetz LGPL-3.0-or-later diff --git a/libewf/libewf_handle.c b/libewf/libewf_handle.c index 76edf13..eb1b4cf 100644 --- a/libewf/libewf_handle.c +++ b/libewf/libewf_handle.c @@ -1016,6 +1016,19 @@ int libewf_handle_open( return( -1 ); } +#if !defined( HAVE_WRITE_SUPPORT ) + if( ( access_flags & LIBEWF_ACCESS_FLAG_WRITE ) != 0 ) + { + libcerror_error_set( + error, + LIBCERROR_ERROR_DOMAIN_ARGUMENTS, + LIBCERROR_ARGUMENT_ERROR_UNSUPPORTED_VALUE, + "%s: write access currently not supported - compiled without zlib.", + function ); + + return( -1 ); + } +#endif if( libbfio_pool_initialize( &file_io_pool, 0, @@ -1418,6 +1431,19 @@ int libewf_handle_open_wide( return( -1 ); } +#if !defined( HAVE_WRITE_SUPPORT ) + if( ( access_flags & LIBEWF_ACCESS_FLAG_WRITE ) != 0 ) + { + libcerror_error_set( + error, + LIBCERROR_ERROR_DOMAIN_ARGUMENTS, + LIBCERROR_ARGUMENT_ERROR_UNSUPPORTED_VALUE, + "%s: write access currently not supported - compiled without zlib.", + function ); + + return( -1 ); + } +#endif if( libbfio_pool_initialize( &file_io_pool, 0, @@ -1738,7 +1764,8 @@ int libewf_handle_open_wide( } return( -1 ); } -#endif + +#endif /* defined( HAVE_WIDE_CHARACTER_TYPE ) */ /* Opens a set of EWF file(s) using a Basic File IO (bfio) pool * Returns 1 if successful or -1 on error @@ -1894,6 +1921,19 @@ int libewf_handle_open_file_io_pool( return( -1 ); } +#if !defined( HAVE_WRITE_SUPPORT ) + if( ( access_flags & LIBEWF_ACCESS_FLAG_WRITE ) != 0 ) + { + libcerror_error_set( + error, + LIBCERROR_ERROR_DOMAIN_ARGUMENTS, + LIBCERROR_ARGUMENT_ERROR_UNSUPPORTED_VALUE, + "%s: write access currently not supported - compiled without zlib.", + function ); + + return( -1 ); + } +#endif if( internal_handle->segment_table == NULL ) { if( libewf_segment_table_initialize( @@ -3937,7 +3977,7 @@ int libewf_handle_close( "%s: unable to finalize write.", function ); - return( -1 ); + result = -1; } } if( internal_handle->file_io_pool_created_in_library != 0 ) @@ -3985,6 +4025,22 @@ int libewf_handle_close( result = -1; } + if( internal_handle->chunk_data != NULL ) + { + if( libewf_chunk_data_free( + &( internal_handle->chunk_data ), + error ) != 1 ) + { + libcerror_error_set( + error, + LIBCERROR_ERROR_DOMAIN_RUNTIME, + LIBCERROR_RUNTIME_ERROR_FINALIZE_FAILED, + "%s: unable to free chunk data.", + function ); + + result = -1; + } + } if( internal_handle->read_io_handle != NULL ) { if( libewf_read_io_handle_free( diff --git a/libewf/libewf_write_io_handle.c b/libewf/libewf_write_io_handle.c index 378d5b3..4f11afa 100644 --- a/libewf/libewf_write_io_handle.c +++ b/libewf/libewf_write_io_handle.c @@ -253,7 +253,12 @@ int libewf_write_io_handle_clone( "%s: unable to copy source to destination write IO handle.", function ); - goto on_error; + memory_free( + *destination_write_io_handle ); + + *destination_write_io_handle = NULL; + + return( -1 ); } ( *destination_write_io_handle )->data_section = NULL; ( *destination_write_io_handle )->table_offsets = NULL; diff --git a/libmfdata/libmfdata_list_element.c b/libmfdata/libmfdata_list_element.c index 5288864..e881203 100644 --- a/libmfdata/libmfdata_list_element.c +++ b/libmfdata/libmfdata_list_element.c @@ -179,7 +179,7 @@ int libmfdata_list_element_free( { if( internal_element->group_values != NULL ) { - if( internal_element->group_values->number_of_elements == 1 ) + if( internal_element->group_values->number_of_elements <= 1 ) { if( libmfdata_group_free( &( internal_element->group_values ), diff --git a/m4/libbfio.m4 b/m4/libbfio.m4 index 60b61a8..db18a7f 100644 --- a/m4/libbfio.m4 +++ b/m4/libbfio.m4 @@ -1,6 +1,6 @@ dnl Checks for libbfio required headers and functions dnl -dnl Version: 20191230 +dnl Version: 20201125 dnl Function to detect if libbfio is available dnl ac_libbfio_dummy is used to prevent AC_CHECK_LIB adding unnecessary -l arguments @@ -26,7 +26,7 @@ AC_DEFUN([AX_LIBBFIO_CHECK_LIB], [test "x$cross_compiling" != "xyes" && test "x$PKGCONFIG" != "x"], [PKG_CHECK_MODULES( [libbfio], - [libbfio >= 20191230], + [libbfio >= 20201125], [ac_cv_libbfio=yes], [ac_cv_libbfio=check]) ]) @@ -100,11 +100,21 @@ AC_DEFUN([AX_LIBBFIO_CHECK_LIB], libbfio_handle_read_buffer, [ac_cv_libbfio_dummy=yes], [ac_cv_libbfio=no]) + AC_CHECK_LIB( + bfio, + libbfio_handle_read_buffer_at_offset, + [ac_cv_libbfio_dummy=yes], + [ac_cv_libbfio=no]) AC_CHECK_LIB( bfio, libbfio_handle_write_buffer, [ac_cv_libbfio_dummy=yes], [ac_cv_libbfio=no]) + AC_CHECK_LIB( + bfio, + libbfio_handle_write_buffer_at_offset, + [ac_cv_libbfio_dummy=yes], + [ac_cv_libbfio=no]) AC_CHECK_LIB( bfio, libbfio_handle_seek_offset, @@ -261,11 +271,21 @@ AC_DEFUN([AX_LIBBFIO_CHECK_LIB], libbfio_pool_read_buffer, [ac_cv_libbfio_dummy=yes], [ac_cv_libbfio=no]) + AC_CHECK_LIB( + bfio, + libbfio_pool_read_buffer_at_offset, + [ac_cv_libbfio_dummy=yes], + [ac_cv_libbfio=no]) AC_CHECK_LIB( bfio, libbfio_pool_write_buffer, [ac_cv_libbfio_dummy=yes], [ac_cv_libbfio=no]) + AC_CHECK_LIB( + bfio, + libbfio_pool_write_buffer_at_offset, + [ac_cv_libbfio_dummy=yes], + [ac_cv_libbfio=no]) AC_CHECK_LIB( bfio, libbfio_pool_seek_offset, @@ -282,6 +302,22 @@ AC_DEFUN([AX_LIBBFIO_CHECK_LIB], [ac_cv_libbfio_dummy=yes], [ac_cv_libbfio=no]) + dnl File pool functions + AC_CHECK_LIB( + bfio, + libbfio_file_pool_open, + [ac_cv_libbfio_dummy=yes], + [ac_cv_libbfio=no]) + + AS_IF( + [test "x$ac_cv_enable_wide_character_type" != xno], + [AC_CHECK_LIB( + bfio, + libbfio_file_pool_open_wide, + [ac_cv_libbfio_dummy=yes], + [ac_cv_libbfio=no]) + ]) + ac_cv_libbfio_LIBADD="-lbfio"]) ]) AS_IF( diff --git a/pyewf/pyewf.c b/pyewf/pyewf.c index 5fffa48..7b8e341 100644 --- a/pyewf/pyewf.c +++ b/pyewf/pyewf.c @@ -24,7 +24,7 @@ #include #include -#if defined( HAVE_STDLIB_H ) +#if defined( HAVE_STDLIB_H ) || defined( HAVE_WINAPI ) #include #endif @@ -40,11 +40,13 @@ #include "pyewf_unused.h" #if !defined( LIBEWF_HAVE_BFIO ) + LIBEWF_EXTERN \ int libewf_check_file_signature_file_io_handle( libbfio_handle_t *file_io_handle, libewf_error_t **error ); -#endif + +#endif /* !defined( LIBEWF_HAVE_BFIO ) */ /* The pyewf module methods */ @@ -66,7 +68,7 @@ PyMethodDef pyewf_module_methods[] = { { "check_file_signature_file_object", (PyCFunction) pyewf_check_file_signature_file_object, METH_VARARGS | METH_KEYWORDS, - "check_file_signature_file_object(filename) -> Boolean\n" + "check_file_signature_file_object(file_object) -> Boolean\n" "\n" "Checks if a file has an Expert Witness Compression Format (EWF) signature using a file-like object." }, @@ -89,10 +91,7 @@ PyMethodDef pyewf_module_methods[] = { /* TODO: open file-like object using pool - list of file objects */ /* Sentinel */ - { NULL, - NULL, - 0, - NULL} + { NULL, NULL, 0, NULL } }; /* Retrieves the pyewf/libewf version @@ -207,7 +206,7 @@ PyObject *pyewf_check_file_signature_file_object( if( PyArg_ParseTupleAndKeywords( arguments, keywords, - "|O", + "O|", keyword_list, &file_object ) == 0 ) { @@ -487,8 +486,9 @@ PyMODINIT_FUNC initpyewf( return; #endif } +#if PY_VERSION_HEX < 0x03070000 PyEval_InitThreads(); - +#endif gil_state = PyGILState_Ensure(); /* Setup the handle type object diff --git a/runtests.sh b/runtests.sh index 741c0c8..d3f6072 100755 --- a/runtests.sh +++ b/runtests.sh @@ -215,9 +215,9 @@ HAVE_WITH_PTHREAD=$?; echo "${CONFIGURE_HELP}" | grep -- '--with-zlib' > /dev/null; -# HAVE_WITH_ZLIB=$?; -# -# echo "${CONFIGURE_HELP}" | grep -- '--with-openssl' > /dev/null; +HAVE_WITH_ZLIB=$?; + +echo "${CONFIGURE_HELP}" | grep -- '--with-openssl' > /dev/null; HAVE_WITH_OPENSSL=$?; @@ -272,18 +272,18 @@ then fi fi -# if test ${HAVE_WITH_ZLIB} -eq 0; -# then +if test ${HAVE_WITH_ZLIB} -eq 0; +then # Test "./configure && make && make check" with fallback zlib implementation. -# run_configure_make_check "--with-zlib=no"; -# RESULT=$?; + run_configure_make_check "--with-zlib=no"; + RESULT=$?; -# if test ${RESULT} -ne ${EXIT_SUCCESS}; -# then -# exit ${EXIT_FAILURE}; -# fi -# fi + if test ${RESULT} -ne ${EXIT_SUCCESS}; + then + exit ${EXIT_FAILURE}; + fi +fi if test ${HAVE_WITH_OPENSSL} -eq 0; then diff --git a/tests/Makefile.am b/tests/Makefile.am index 139b00a..b2cefcc 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -26,23 +26,28 @@ TESTS_PYEWF = \ test_pyewf.sh endif -TESTS = \ - test_library.sh \ - test_glob.sh \ - test_seek.sh \ - test_read.sh \ +if HAVE_WRITE_TESTS +TESTS_WRITE = \ test_write.sh \ test_read_write.sh \ test_read_write_delta.sh \ - test_ewfverify.sh \ - test_ewfverify_logical.sh \ test_ewfacquire.sh \ test_ewfacquire_optical.sh \ test_ewfacquirestream.sh \ test_ewfacquire_resume.sh \ - test_ewfinfo.sh \ test_ewfexport.sh \ - test_ewfexport_logical.sh \ + test_ewfexport_logical.sh +endif + +TESTS = \ + test_library.sh \ + test_glob.sh \ + test_seek.sh \ + test_read.sh \ + test_ewfverify.sh \ + test_ewfverify_logical.sh \ + test_ewfinfo.sh \ + $(TESTS_WRITE) \ $(TESTS_PYEWF) check_SCRIPTS = \ diff --git a/tests/ewf_test_chunk_data.c b/tests/ewf_test_chunk_data.c index ad1f90e..bb2ea3f 100644 --- a/tests/ewf_test_chunk_data.c +++ b/tests/ewf_test_chunk_data.c @@ -611,10 +611,14 @@ int main( "libewf_chunk_data_free", ewf_test_chunk_data_free ); +#if defined( HAVE_WRITE_SUPPORT ) + EWF_TEST_RUN( "libewf_chunk_data_pack", ewf_test_chunk_data_pack ); +#endif /* defined( HAVE_WRITE_SUPPORT ) */ + EWF_TEST_RUN( "libewf_chunk_data_unpack", ewf_test_chunk_data_unpack ); diff --git a/tests/ewf_test_compression.c b/tests/ewf_test_compression.c index 8ca589b..f3ae8ae 100644 --- a/tests/ewf_test_compression.c +++ b/tests/ewf_test_compression.c @@ -1083,10 +1083,14 @@ int main( #if defined( __GNUC__ ) && !defined( LIBEWF_DLL_IMPORT ) +#if defined( HAVE_WRITE_SUPPORT ) + EWF_TEST_RUN( "libewf_compress_data", ewf_test_compress_data ); +#endif /* defined( HAVE_WRITE_SUPPORT ) */ + EWF_TEST_RUN( "libewf_decompress_data", ewf_test_decompress_data );