merge RC_1_1 into master

Author: arvidn

ChangeLog

@@ -76,6 +76,9 @@
 	* resume data no longer has timestamps of files
 	* require C++11 to build libtorrent
 
+	* fix issue with very long tracker- and web seed URLs
+	* don't attempt to create empty files on startup, if they already exist
+	* fix force-recheck issue (new files would not be picked up)
 	* fix inconsistency in file_priorities and override_resume_data behavior
 
 1.1.4 release

include/libtorrent/torrent.hpp

@@ -1024,7 +1024,7 @@ namespace libtorrent {
 
 		// renames the file with the given index to the new name
 		// the name may include a directory path
-		// returns false on failure
+		// posts alert to indicate success or failure
 		void rename_file(file_index_t index, std::string name);
 
 		// unless this returns true, new connections must wait

src/http_connection.cpp

@@ -163,13 +163,7 @@ void http_connection::get(std::string const& url, time_duration timeout, int pri
 	bool ssl = false;
 	if (protocol == "https") ssl = true;
 
-	char request[4096];
-	char* end = request + sizeof(request);
-	char* ptr = request;
-
-#define APPEND_FMT(fmt) ptr += std::snprintf(ptr, std::size_t(end - ptr), fmt)
-#define APPEND_FMT1(fmt, arg) ptr += std::snprintf(ptr, std::size_t(end - ptr), fmt, arg)
-#define APPEND_FMT2(fmt, arg1, arg2) ptr += std::snprintf(ptr, std::size_t(end - ptr), fmt, arg1, arg2)
+	std::stringstream request;
 
 	// exclude ssl here, because SSL assumes CONNECT support in the
 	// proxy and is handled at the lower layer
@@ -179,40 +173,39 @@ void http_connection::get(std::string const& url, time_duration timeout, int pri
 	{
 		// if we're using an http proxy and not an ssl
 		// connection, just do a regular http proxy request
-		APPEND_FMT1("GET %s HTTP/1.1\r\n", url.c_str());
+		request << "GET " << url << " HTTP/1.1\r\n";
 		if (ps->type == settings_pack::http_pw)
-			APPEND_FMT1("Proxy-Authorization: Basic %s\r\n", base64encode(
-				ps->username + ":" + ps->password).c_str());
+			request << "Proxy-Authorization: Basic " << base64encode(
+				ps->username + ":" + ps->password) << "\r\n";
 
 		hostname = ps->hostname;
 		port = ps->port;
 
-		APPEND_FMT1("Host: %s", hostname.c_str());
-		if (port != default_port) APPEND_FMT1(":%d\r\n", port);
-		else APPEND_FMT("\r\n");
+		request << "Host: " << hostname;
+		if (port != default_port) request << ":" << port << "\r\n";
+		else request << "\r\n";
 	}
 	else
 	{
-		APPEND_FMT2("GET %s HTTP/1.1\r\n"
-			"Host: %s", path.c_str(), hostname.c_str());
-		if (port != default_port) APPEND_FMT1(":%d\r\n", port);
-		else APPEND_FMT("\r\n");
+		request << "GET " << path << " HTTP/1.1\r\nHost: " << hostname;
+		if (port != default_port) request << ":" << port << "\r\n";
+		else request << "\r\n";
 	}
 
-//	APPEND_FMT("Accept: */*\r\n");
+//	request << "Accept: */*\r\n";
 
 	if (!m_user_agent.empty())
-		APPEND_FMT1("User-Agent: %s\r\n", m_user_agent.c_str());
+		request << "User-Agent: " << m_user_agent << "\r\n";
 
 	if (m_bottled)
-		APPEND_FMT("Accept-Encoding: gzip\r\n");
+		request << "Accept-Encoding: gzip\r\n";
 
 	if (!auth.empty())
-		APPEND_FMT1("Authorization: Basic %s\r\n", base64encode(auth).c_str());
+		request << "Authorization: Basic " << base64encode(auth) << "\r\n";
 
-	APPEND_FMT("Connection: close\r\n\r\n");
+	request << "Connection: close\r\n\r\n";
 
-	m_sendbuffer.assign(request);
+	m_sendbuffer.assign(request.str());
 	m_url = url;
 	start(hostname, port, timeout, prio
 		, ps, ssl, handle_redirects, bind_addr, m_resolve_flags

src/storage.cpp

@@ -242,7 +242,7 @@ namespace libtorrent {
 			// it's supposed to be, truncate it
 			// if the file is empty, just create it either way.
 			if ((!err && size > files().file_size(file_index))
-				|| files().file_size(file_index) == 0)
+				|| (files().file_size(file_index) == 0 && err == boost::system::errc::no_such_file_or_directory))
 			{
 				std::string file_path = files().file_path(file_index, m_save_path);
 				std::string dir = parent_path(file_path);

src/torrent.cpp

@@ -2227,6 +2227,10 @@ namespace libtorrent {
 		TORRENT_ASSERT(m_outstanding_check_files == false);
 		m_add_torrent_params.reset();
 
+		// this will clear the stat cache, to make us actually query the
+		// filesystem for files again
+		m_ses.disk_thread().async_release_files(m_storage, []{});
+
 		aux::vector<std::string, file_index_t> links;
 		m_ses.disk_thread().async_check_files(m_storage, nullptr
 			, links, std::bind(&torrent::on_force_recheck
@@ -2657,6 +2661,7 @@ namespace libtorrent {
 	void torrent::announce_with_tracker(std::uint8_t e)
 	{
 		TORRENT_ASSERT(is_single_thread());
+		TORRENT_ASSERT(e == tracker_request::stopped || state() != torrent_status::checking_files);
 		INVARIANT_CHECK;
 
 		if (m_trackers.empty())
@@ -8831,6 +8836,7 @@ namespace libtorrent {
 	void torrent::start_announcing()
 	{
 		TORRENT_ASSERT(is_single_thread());
+		TORRENT_ASSERT(state() != torrent_status::checking_files);
 		if (is_paused())
 		{
 #ifndef TORRENT_DISABLE_LOGGING

test/setup_transfer.cpp

@@ -640,7 +640,8 @@ std::shared_ptr<lt::torrent_info> make_torrent(const int file_sizes[]
 	return std::make_shared<torrent_info>(buf, from_span);
 }
 
-void create_random_files(std::string const& path, const int file_sizes[], int num_files)
+void create_random_files(std::string const& path, const int file_sizes[], int num_files
+	, file_storage* fs)
 {
 	error_code ec;
 	aux::vector<char> random_data(300000);
@@ -657,6 +658,7 @@ void create_random_files(std::string const& path, const int file_sizes[], int nu
 		full_path = combine_path(full_path, filename);
 
 		int to_write = file_sizes[i];
+		if (fs) fs->add_file(full_path, to_write);
 		file f(full_path, open_mode::write_only, ec);
 		if (ec) std::printf("failed to create file \"%s\": (%d) %s\n"
 			, full_path.c_str(), ec.value(), ec.message().c_str());

test/setup_transfer.hpp

@@ -37,9 +37,14 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "test.hpp"
 #include "libtorrent/session.hpp"
 #include "libtorrent/units.hpp"
-#include "libtorrent/span.hpp"
-#include "libtorrent/alert.hpp"
-#include "libtorrent/add_torrent_params.hpp"
+
+namespace libtorrent
+{
+	class alert;
+	struct add_torrent_params;
+	class file_storage;
+	class session;
+}
 
 EXPORT int print_failures();
 EXPORT unsigned char random_byte();
@@ -89,7 +94,7 @@ EXPORT lt::file_storage make_file_storage(const int file_sizes[], int num_files
 EXPORT std::shared_ptr<lt::torrent_info> make_torrent(const int file_sizes[]
 	, int num_files, int piece_size);
 EXPORT void create_random_files(std::string const& path, const int file_sizes[]
-	, int num_files);
+	, int num_files, libtorrent::file_storage* fs = nullptr);
 
 EXPORT std::shared_ptr<lt::torrent_info> create_torrent(std::ostream* file = 0
 	, char const* name = "temporary", int piece_size = 16 * 1024, int num_pieces = 13

test/test_checking.cpp

@@ -30,18 +30,20 @@ POSSIBILITY OF SUCH DAMAGE.
 
 */
 
+#include <sys/stat.h> // for chmod
+
 #include "libtorrent/session.hpp"
 #include "test.hpp"
 #include "setup_transfer.hpp"
 #include "libtorrent/create_torrent.hpp"
-#include <sys/stat.h> // for chmod
+#include "libtorrent/alert_types.hpp"
 #include "libtorrent/torrent_info.hpp"
 #include "libtorrent/torrent_status.hpp"
 #include "libtorrent/hex.hpp" // to_hex
 #include "libtorrent/aux_/path.hpp"
 
 static const int file_sizes[] =
-{ 5, 16 - 5, 16000, 17, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1
+{ 0, 5, 16 - 5, 16000, 17, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1
 	,1,1,1,1,1,1,13,65000,34,75,2,30,400,500,23000,900,43000,400,4300,6, 4};
 const int num_files = sizeof(file_sizes)/sizeof(file_sizes[0]);
 
@@ -56,67 +58,47 @@ enum
 	corrupt_files = 2,
 
 	incomplete_files = 4,
+
+	// make the files not be there when starting up, move the files in place and
+	// force-recheck. Make sure the stat cache is cleared and let us pick up the
+	// new files
+	force_recheck = 8,
 };
 
 void test_checking(int flags = read_only_files)
 {
 	using namespace lt;
 
-	std::printf("\n==== TEST CHECKING %s%s%s=====\n\n"
+	std::printf("\n==== TEST CHECKING %s%s%s%s=====\n\n"
 		, (flags & read_only_files) ? "read-only-files ":""
 		, (flags & corrupt_files) ? "corrupt ":""
-		, (flags & incomplete_files) ? "incomplete ":"");
-
-	// make the files writable again
-	for (int i = 0; i < num_files; ++i)
-	{
-		char name[1024];
-		std::snprintf(name, sizeof(name), "test%d", i);
-		char dirname[200];
-		std::snprintf(dirname, sizeof(dirname), "test_dir%d", i / 5);
-		std::string path = combine_path(combine_path("tmp1_checking", "test_torrent_dir"), dirname);
-		path = combine_path(path, name);
-#ifdef TORRENT_WINDOWS
-		SetFileAttributesA(path.c_str(), FILE_ATTRIBUTE_NORMAL);
-#else
-		chmod(path.c_str(), S_IRUSR | S_IWUSR);
-#endif
-	}
+		, (flags & incomplete_files) ? "incomplete ":""
+		, (flags & force_recheck) ? "force_recheck ":"");
 
-	// in case the previous run was terminated
 	error_code ec;
-	remove_all("tmp1_checking", ec);
-	if (ec) std::printf("ERROR: removing tmp1_checking: (%d) %s\n"
-		, ec.value(), ec.message().c_str());
-
-	create_directory("tmp1_checking", ec);
-	if (ec) std::printf("ERROR: creating directory tmp1_checking: (%d) %s\n"
-		, ec.value(), ec.message().c_str());
-	create_directory(combine_path("tmp1_checking", "test_torrent_dir"), ec);
-	if (ec) std::printf("ERROR: creating directory test_torrent_dir: (%d) %s\n"
+	create_directory("test_torrent_dir", ec);
+	if (ec) fprintf(stdout, "ERROR: creating directory test_torrent_dir: (%d) %s\n"
 		, ec.value(), ec.message().c_str());
 
 	file_storage fs;
 	std::srand(10);
 	int piece_size = 0x4000;
 
-	create_random_files(combine_path("tmp1_checking", "test_torrent_dir")
-		, file_sizes, num_files);
+	create_random_files("test_torrent_dir", file_sizes, num_files, &fs);
 
-	add_files(fs, combine_path("tmp1_checking", "test_torrent_dir"));
 	lt::create_torrent t(fs, piece_size, 0x4000
 		, lt::create_torrent::optimize_alignment);
 
 	// calculate the hash for all pieces
-	set_piece_hashes(t, "tmp1_checking", ec);
+	set_piece_hashes(t, ".", ec);
 	if (ec) std::printf("ERROR: set_piece_hashes: (%d) %s\n"
 		, ec.value(), ec.message().c_str());
 
 	std::vector<char> buf;
 	bencode(std::back_inserter(buf), t.generate());
 	auto ti = std::make_shared<torrent_info>(buf, ec, from_span);
 
-	std::printf("generated torrent: %s tmp1_checking/test_torrent_dir\n"
+	std::printf("generated torrent: %s test_torrent_dir\n"
 		, aux::to_hex(ti->info_hash()).c_str());
 
 	// truncate every file in half
@@ -128,7 +110,7 @@ void test_checking(int flags = read_only_files)
 			std::snprintf(name, sizeof(name), "test%d", i);
 			char dirname[200];
 			std::snprintf(dirname, sizeof(dirname), "test_dir%d", i / 5);
-			std::string path = combine_path(combine_path("tmp1_checking", "test_torrent_dir"), dirname);
+			std::string path = combine_path("test_torrent_dir", dirname);
 			path = combine_path(path, name);
 
 			error_code ec;
@@ -148,9 +130,9 @@ void test_checking(int flags = read_only_files)
 		// increase the size of some files. When they're read only that forces
 		// the checker to open them in write-mode to truncate them
 		static const int file_sizes2[] =
-		{ 5, 16 - 5, 16001, 30, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1
+		{ 0, 5, 16 - 5, 16001, 30, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1
 			,1,1,1,1,1,1,13,65000,34,75,2,30,400,500,23000,900,43000,400,4300,6, 4};
-		create_random_files(combine_path("tmp1_checking", "test_torrent_dir"), file_sizes2, num_files);
+		create_random_files("test_torrent_dir", file_sizes2, num_files);
 	}
 
 	// make the files read only
@@ -164,7 +146,7 @@ void test_checking(int flags = read_only_files)
 			char dirname[200];
 			std::snprintf(dirname, sizeof(dirname), "test_dir%d", i / 5);
 
-			std::string path = combine_path(combine_path("tmp1_checking", "test_torrent_dir"), dirname);
+			std::string path = combine_path("test_torrent_dir", dirname);
 			path = combine_path(path, name);
 			std::printf("   %s\n", path.c_str());
 
@@ -176,6 +158,16 @@ void test_checking(int flags = read_only_files)
 		}
 	}
 
+	if (flags & force_recheck)
+	{
+		remove_all("test_torrent_dir_tmp", ec);
+		if (ec) std::printf("ERROR: removing \"test_torrent_dir_tmp\": (%d) %s\n"
+			, ec.value(), ec.message().c_str());
+		rename("test_torrent_dir", "test_torrent_dir_tmp", ec);
+		if (ec) std::printf("ERROR: renaming dir \"test_torrent_dir\": (%d) %s\n"
+			, ec.value(), ec.message().c_str());
+	}
+
 	auto const mask = alert::all_categories
 		& ~(alert::progress_notification
 			| alert::performance_warning
@@ -192,11 +184,30 @@ void test_checking(int flags = read_only_files)
 	lt::session ses1(pack);
 
 	add_torrent_params p;
-	p.save_path = "tmp1_checking";
+	p.save_path = ".";
 	p.ti = ti;
 	torrent_handle tor1 = ses1.add_torrent(p, ec);
 	TEST_CHECK(!ec);
 
+	if (flags & force_recheck)
+	{
+		// first make sure the session tries to check for the file and can't find
+		// them
+		libtorrent::alert const* a = wait_for_alert(
+			ses1, torrent_checked_alert::alert_type, "checking");
+		TEST_CHECK(a);
+
+		// now, move back the files and force-recheck. make sure we pick up the
+		// files this time
+		remove_all("test_torrent_dir", ec);
+		if (ec) fprintf(stdout, "ERROR: removing \"test_torrent_dir\": (%d) %s\n"
+			, ec.value(), ec.message().c_str());
+		rename("test_torrent_dir_tmp", "test_torrent_dir", ec);
+		if (ec) fprintf(stdout, "ERROR: renaming dir \"test_torrent_dir_tmp\": (%d) %s\n"
+			, ec.value(), ec.message().c_str());
+		tor1.force_recheck();
+	}
+
 	torrent_status st;
 	for (int i = 0; i < 20; ++i)
 	{
@@ -217,6 +228,7 @@ void test_checking(int flags = read_only_files)
 		if (st.errc) break;
 		std::this_thread::sleep_for(lt::milliseconds(500));
 	}
+
 	if (flags & incomplete_files)
 	{
 		TEST_CHECK(!st.is_seeding);
@@ -273,7 +285,7 @@ void test_checking(int flags = read_only_files)
 			char dirname[200];
 			std::snprintf(dirname, sizeof(dirname), "test_dir%d", i / 5);
 
-			std::string path = combine_path(combine_path("tmp1_checking", "test_torrent_dir"), dirname);
+			std::string path = combine_path("test_torrent_dir", dirname);
 			path = combine_path(path, name);
 #ifdef TORRENT_WINDOWS
 			SetFileAttributesA(path.c_str(), FILE_ATTRIBUTE_NORMAL);
@@ -283,8 +295,8 @@ void test_checking(int flags = read_only_files)
 		}
 	}
 
-	remove_all("tmp1_checking", ec);
-	if (ec) std::printf("ERROR: removing tmp1_checking: (%d) %s\n"
+	remove_all("test_torrent_dir", ec);
+	if (ec) std::printf("ERROR: removing test_torrent_dir: (%d) %s\n"
 		, ec.value(), ec.message().c_str());
 }
 
@@ -313,3 +325,8 @@ TORRENT_TEST(corrupt)
 	test_checking(corrupt_files);
 }
 
+TORRENT_TEST(force_recheck)
+{
+	test_checking(force_recheck);
+}
+