merged RC_2_0 into master

.github/workflows/python.yml

@@ -100,7 +100,7 @@ jobs:
       if: runner.os == 'Windows'
       shell: cmd
       run: |
-        git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.72.0 https://github.com/boostorg/boost.git
+        git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.78.0 https://github.com/boostorg/boost.git
         cd boost
         bootstrap.bat
 
@@ -115,7 +115,8 @@ jobs:
       if: runner.os == 'Windows'
       uses: nick-invision/retry@v2
       with:
-        timeout_minutes: 30
+        shell: cmd
+        timeout_minutes: 5
         retry_wait_seconds: 4
         max_attempts: 3
         command: choco install openssl --limitoutput --no-progress

.github/workflows/windows.yml

@@ -47,7 +47,7 @@ jobs:
 
       - name: install boost
         run: |
-          git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.72.0 https://github.com/boostorg/boost.git
+          git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.78.0 https://github.com/boostorg/boost.git
           cd boost
           bootstrap.bat
 
@@ -92,7 +92,7 @@ jobs:
 
       - name: install boost
         run: |
-          git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.72.0 https://github.com/boostorg/boost.git
+          git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.78.0 https://github.com/boostorg/boost.git
           cd boost
           bootstrap.bat
 
@@ -141,7 +141,7 @@ jobs:
 
       - name: install boost
         run: |
-          git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.72.0 https://github.com/boostorg/boost.git
+          git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.78.0 https://github.com/boostorg/boost.git
           cd boost
           bootstrap.bat
 

CMakeLists.txt

@@ -89,6 +89,7 @@ set(libtorrent_include_files
 	torrent_info.hpp
 	torrent_status.hpp
 	tracker_event.hpp
+	truncate.hpp
 	units.hpp
 	upnp.hpp
 	version.hpp
@@ -303,6 +304,7 @@ set(sources
 	chained_buffer.cpp
 	choker.cpp
 	close_reason.cpp
+	copy_file.cpp
 	cpuid.cpp
 	crc32c.cpp
 	create_torrent.cpp
@@ -404,6 +406,7 @@ set(sources
 	torrent_peer_allocator.cpp
 	torrent_status.cpp
 	tracker_manager.cpp
+	truncate.cpp
 	udp_socket.cpp
 	udp_tracker_connection.cpp
 	upnp.cpp

ChangeLog

@@ -5,6 +5,12 @@
 	* added support for WebTorrent
 
 
+	* fix issue creating a v2 torrent from torrent_info containing an empty file
+	* make recheck files also update which files use partfile
+	* add write_through disk_io_write_mode, which flushes pieces to disk immediately
+	* improve copy file function to preserve sparse regions (when supported)
+	* add function to truncate over-sized files part of a torrent
+	* fix directory creation on windows shared folders
 	* add flag to make add_files() not record file attributes
 	* deprecate (unused) allow_partial_disk_writes settings
 	* fix disk-full error reporting in mmap_disk_io
@@ -103,6 +109,7 @@
 	* added support for GnuTLS for HTTPS and torrents over SSL
 
 
+	* send User-Agent field in anonymous mode
 	* fix python binding for settings_pack conversion
 	* fix DHT announce timer issue
 	* use DSCP_TRAFFIC_TYPE socket option on windows

Jamfile

@@ -165,9 +165,15 @@ rule linking ( properties * )
 	{
 		# for backtraces in assertion failures
 		# which only works on ELF targets with gcc
-		result += <linkflags>-Wl,-export-dynamic <linkflags>-rdynamic ;
+		result += <linkflags>-Wl,--export-dynamic <linkflags>-rdynamic ;
 		result += <local-visibility>global ;
 	}
+	else
+	{
+		# backtraces don't work with visibility=hidden, so we only add that in
+		# the else-block
+		result += <visibility>hidden ;
+	}
 
 	local BOOST_VERSION_TAG = [ modules.peek boostcpp : BOOST_VERSION_TAG ] ;
 	local json ;
@@ -339,9 +345,9 @@ rule building ( properties * )
 	if <toolset>msvc in $(properties) || <toolset>intel-win in $(properties)
 	{
 		# allow larger .obj files (with more sections)
-		result += <cflags>/bigobj <cxxflags>/bigobj ;
+		result += <cxxflags>/bigobj ;
 		# https://docs.microsoft.com/en-us/cpp/build/reference/utf-8-set-source-and-executable-character-sets-to-utf-8?view=msvc-170
-		result += <cflags>/utf-8 ;
+		result += <cxxflags>/utf-8 ;
 	}
 
 	if <toolset>gcc in $(properties) && <target-os>windows in $(properties)
@@ -731,6 +737,7 @@ SOURCES =
 	chained_buffer
 	choker
 	close_reason
+	copy_file
 	cpuid
 	crc32c
 	create_torrent
@@ -843,6 +850,7 @@ SOURCES =
 	posix_part_file
 	posix_storage
 	ssl
+	truncate
 	rtc_signaling
 	rtc_stream
 	websocket_stream
@@ -945,7 +953,6 @@ lib torrent
 	<threading>multi
 	<cxxstd>17
 	<c++-template-depth>512
-	<local-visibility>hidden
 
 	: # usage requirements
 	$(usage-requirements)

Makefile

@@ -307,6 +307,7 @@ SOURCES = \
   chained_buffer.cpp              \
   choker.cpp                      \
   close_reason.cpp                \
+  copy_file.cpp                   \
   cpuid.cpp                       \
   crc32c.cpp                      \
   create_torrent.cpp              \
@@ -408,6 +409,7 @@ SOURCES = \
   torrent_peer_allocator.cpp      \
   torrent_status.cpp              \
   tracker_manager.cpp             \
+  truncate.cpp                    \
   udp_socket.cpp                  \
   udp_tracker_connection.cpp      \
   upnp.cpp                        \
@@ -503,6 +505,7 @@ HEADERS = \
   torrent_info.hpp             \
   torrent_status.hpp           \
   tracker_event.hpp            \
+  truncate.hpp                 \
   units.hpp                    \
   upnp.hpp                     \
   version.hpp                  \
@@ -853,6 +856,7 @@ TEST_SOURCES = \
   test_bloom_filter.cpp \
   test_buffer.cpp \
   test_checking.cpp \
+  test_copy_file.cpp \
   test_crc32.cpp \
   test_create_torrent.cpp \
   test_dht.cpp \
@@ -930,6 +934,7 @@ TEST_SOURCES = \
   test_torrent_list.cpp \
   test_tracker.cpp \
   test_tracker_manager.cpp \
+  test_truncate.cpp \
   test_transfer.cpp \
   test_upnp.cpp \
   test_url_seed.cpp \

bindings/c/include/libtorrent_alerts.h

@@ -97,6 +97,7 @@ enum alert_types_t {
 	ALERT_ALERTS_DROPPED =  95,
 	ALERT_SOCKS5 =  96,
 	ALERT_FILE_PRIO =  97,
+	ALERT_OVERSIZED_FILE =  98,
 };
 
 #endif // LIBTORRENT_ALERTS_H

bindings/python/src/peer_info.cpp

@@ -94,7 +94,7 @@ void bind_peer_info()
         .def_readonly("downloading_progress", &peer_info::downloading_progress)
         .def_readonly("downloading_total", &peer_info::downloading_total)
         .add_property("client", get_peer_info_client)
-        .def_readonly("connection_type", &peer_info::connection_type)
+        .add_property("connection_type", make_getter(&peer_info::connection_type, by_value()))
         .def_readonly("pending_disk_bytes", &peer_info::pending_disk_bytes)
         .def_readonly("send_quota", &peer_info::send_quota)
         .def_readonly("receive_quota", &peer_info::receive_quota)

bindings/python/src/session_settings.cpp

@@ -63,6 +63,7 @@ void bind_session_settings()
         .value("disable_os_cache_for_aligned_files", settings_pack::disable_os_cache_for_aligned_files)
 #endif
         .value("disable_os_cache", settings_pack::disable_os_cache)
+        .value("write_through", settings_pack::write_through)
     ;
 
     enum_<settings_pack::bandwidth_mixed_algo_t>("bandwidth_mixed_algo_t")

examples/Jamfile

@@ -14,11 +14,12 @@ project client_test
 	<threading>multi <library>/torrent//torrent
 	<toolset>darwin:<cflags>-Wno-unused-command-line-argument
 # disable warning C4275: non DLL-interface classkey 'identifier' used as base for DLL-interface classkey 'identifier'
-	<toolset>msvc:<cflags>/wd4275
+	<toolset>msvc:<cxxflags>/wd4275
 	# C4268: 'identifier' : 'const' static/global data initialized
 	#        with compiler generated default constructor fills the object with zeros
-	<toolset>msvc:<cflags>/wd4268
-	<toolset>msvc:<cflags>/wd4373
+	<toolset>msvc:<cxxflags>/wd4268
+	<toolset>msvc:<cxxflags>/wd4373
+	<toolset>clang:<cflags>-Wno-implicit-int-float-conversion
 	<conditional>@warnings
 	: default-build
 	<link>static

examples/client_test.cpp

@@ -610,6 +610,7 @@ void assign_setting(lt::settings_pack& settings, std::string const& key, char co
 				{"anti_leech"_sv, settings_pack::anti_leech},
 				{"enable_os_cache"_sv, settings_pack::enable_os_cache},
 				{"disable_os_cache"_sv, settings_pack::disable_os_cache},
+				{"write_through"_sv, settings_pack::write_through},
 				{"prefer_tcp"_sv, settings_pack::prefer_tcp},
 				{"peer_proportional"_sv, settings_pack::peer_proportional},
 				{"pe_forced"_sv, settings_pack::pe_forced},

include/libtorrent/alert_types.hpp

@@ -73,7 +73,7 @@ namespace libtorrent {
 	constexpr int user_alert_id = 10000;
 
 	// this constant represents "max_alert_index" + 1
-	constexpr int num_alert_types = 98;
+	constexpr int num_alert_types = 99;
 
 	// internal
 	constexpr int abi_alert_count = 128;
@@ -2933,6 +2933,26 @@ TORRENT_VERSION_NAMESPACE_3
 
 TORRENT_VERSION_NAMESPACE_3_END
 
+	// this alert may be posted when the initial checking of resume data and files
+	// on disk (just existence, not piece hashes) completes. If a file belonging
+	// to the torrent is found on disk, but is larger than the file in the
+	// torrent, that's when this alert is posted.
+	// the client may want to call truncate_files() in that case, or perhaps
+	// interpret it as a sign that some other file is in the way, that shouldn't
+	// be overwritten.
+	struct TORRENT_EXPORT oversized_file_alert final : torrent_alert
+	{
+		// internal
+		explicit oversized_file_alert(aux::stack_allocator& alloc, torrent_handle h);
+		TORRENT_DEFINE_ALERT(oversized_file_alert, 98)
+
+		static constexpr alert_category_t static_category = alert_category::storage;
+		std::string message() const override;
+
+		// hidden
+		file_index_t reserved;
+	};
+
 	// internal
 	TORRENT_EXTRA_EXPORT char const* performance_warning_str(performance_alert::performance_warning_t i);
 

include/libtorrent/assert.hpp

@@ -70,12 +70,12 @@ extern TORRENT_EXPORT char const* libtorrent_assert_log;
 
 #if TORRENT_USE_IOSTREAM
 #define TORRENT_ASSERT_VAL(x, y) \
-	do { if (x) {} else { std::stringstream __s__; __s__ << #y ": " << y; \
-	libtorrent::assert_fail(#x, __LINE__, __FILE__, __func__, __s__.str().c_str(), 0); } } TORRENT_WHILE_0
+	do { if (x) {} else { std::stringstream _s; _s << #y ": " << y; \
+	libtorrent::assert_fail(#x, __LINE__, __FILE__, __func__, _s.str().c_str(), 0); } } TORRENT_WHILE_0
 
 #define TORRENT_ASSERT_FAIL_VAL(y) \
-	do { std::stringstream __s__; __s__ << #y ": " << y; \
-	libtorrent::assert_fail("<unconditional>", __LINE__, __FILE__, __func__, __s__.str().c_str(), 0); } TORRENT_WHILE_0
+	do { std::stringstream _s; _s << #y ": " << y; \
+	libtorrent::assert_fail("<unconditional>", __LINE__, __FILE__, __func__, _s.str().c_str(), 0); } TORRENT_WHILE_0
 
 #else
 #define TORRENT_ASSERT_VAL(x, y) TORRENT_ASSERT(x)

include/libtorrent/aux_/disk_buffer_pool.hpp

@@ -24,7 +24,7 @@ see LICENSE file.
 
 #include "libtorrent/io_context.hpp"
 #include "libtorrent/span.hpp"
-#include "libtorrent/aux_/storage_utils.hpp" // for iovec_t
+#include "libtorrent/disk_buffer_holder.hpp" // for buffer_allocator_interface
 
 namespace libtorrent {
 
@@ -33,7 +33,8 @@ namespace libtorrent {
 
 namespace aux {
 
-	struct TORRENT_EXTRA_EXPORT disk_buffer_pool
+	struct TORRENT_EXTRA_EXPORT disk_buffer_pool final
+		: buffer_allocator_interface
 	{
 		explicit disk_buffer_pool(io_context& ios);
 		~disk_buffer_pool();
@@ -43,6 +44,7 @@ namespace aux {
 		char* allocate_buffer(char const* category);
 		char* allocate_buffer(bool& exceeded, std::shared_ptr<disk_observer> o
 			, char const* category);
+		void free_disk_buffer(char* b) override { free_buffer(b); }
 		void free_buffer(char* buf);
 		void free_multiple_buffers(span<char*> bufvec);
 

include/libtorrent/aux_/mmap.hpp

@@ -128,6 +128,10 @@ namespace aux {
 		// anytime soon
 		void dont_need(span<byte const> range);
 
+		// hint the kernel that the given (dirty) range of pages should be
+		// flushed to disk
+		void page_out(span<byte const> range);
+
 		std::int64_t m_size;
 #if TORRENT_HAVE_MAP_VIEW_OF_FILE
 		file_mapping_handle m_file;
@@ -162,6 +166,13 @@ namespace aux {
 			m_mapping->dont_need(range);
 		}
 
+		void page_out(span<byte const> range)
+		{
+			TORRENT_ASSERT(m_mapping);
+			m_mapping->page_out(range);
+		}
+
+
 	private:
 		explicit file_view(std::shared_ptr<file_mapping> m) : m_mapping(std::move(m)) {}
 		std::shared_ptr<file_mapping> m_mapping;

include/libtorrent/aux_/mmap_storage.hpp

@@ -69,7 +69,7 @@ namespace libtorrent::aux {
 			, storage_error&);
 		void release_files(storage_error&);
 		void delete_files(remove_flags_t options, storage_error&);
-		void initialize(settings_interface const&, storage_error&);
+		status_t initialize(settings_interface const&, storage_error&);
 		std::pair<status_t, std::string> move_storage(std::string save_path
 			, move_flags_t, storage_error&);
 		bool verify_resume_data(add_torrent_params const& rd
@@ -78,9 +78,13 @@ namespace libtorrent::aux {
 		bool tick();
 
 		int readv(settings_interface const&, span<iovec_t const> bufs
-			, piece_index_t piece, int offset, aux::open_mode_t mode, storage_error&);
+			, piece_index_t piece, int offset, aux::open_mode_t mode
+			, disk_job_flags_t flags
+			, storage_error&);
 		int writev(settings_interface const&, span<iovec_t const> bufs
-			, piece_index_t piece, int offset, aux::open_mode_t mode, storage_error&);
+			, piece_index_t piece, int offset, aux::open_mode_t mode
+			, disk_job_flags_t flags
+			, storage_error&);
 		int hashv(settings_interface const&, hasher& ph, std::ptrdiff_t len
 			, piece_index_t piece, int offset, aux::open_mode_t mode
 			, disk_job_flags_t flags, storage_error&);

include/libtorrent/aux_/path.hpp

@@ -94,8 +94,6 @@ namespace libtorrent {
 	TORRENT_EXTRA_EXPORT bool exists(std::string const& f, error_code& ec);
 	TORRENT_EXTRA_EXPORT bool is_directory(std::string const& f
 		, error_code& ec);
-	TORRENT_EXTRA_EXPORT void recursive_copy(std::string const& old_path
-		, std::string const& new_path, error_code& ec);
 	TORRENT_EXTRA_EXPORT void copy_file(std::string const& f
 		, std::string const& newf, error_code& ec);
 	TORRENT_EXTRA_EXPORT void move_file(std::string const& f

include/libtorrent/aux_/posix_storage.hpp

@@ -62,7 +62,7 @@ namespace aux {
 
 		void rename_file(file_index_t const index, std::string const& new_filename, storage_error& ec);
 
-		void initialize(settings_interface const&, storage_error& ec);
+		status_t initialize(settings_interface const&, storage_error& ec);
 
 	private:
 

include/libtorrent/aux_/storage_free_list.hpp

@@ -29,6 +29,8 @@ namespace aux {
 
 		void add(storage_index_t const i) { m_free_slots.push_back(i); }
 
+		std::size_t size() const { return m_free_slots.size(); }
+
 	private:
 
 		storage_index_t pop()

include/libtorrent/aux_/storage_utils.hpp

@@ -92,6 +92,7 @@ namespace aux {
 		, aux::vector<download_priority_t, file_index_t> const& file_priority
 		, std::function<void(file_index_t, storage_error&)> create_file
 		, std::function<void(std::string const&, std::string const&, storage_error&)> create_link
+		, std::function<void(file_index_t, std::int64_t)> oversized_file
 		, storage_error& ec);
 
 	TORRENT_EXTRA_EXPORT void create_symlink(