From 2a81fd466d304d71dad16db93224db78525e08ce Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 25 Apr 2008 17:44:03 +0000 Subject: [PATCH] 2008-04-26 Tatsuhiro Tsujikawa Added build-in "Accept-Feature" support. For now only "metalink" is used in this header field. This "metalink" value is removed from the list when connecting URLs fed by metalink file to avoid loop in "transparent" metlaink. * src/HttpRequest.cc * src/HttpRequest.h: Renamed _userHeaders as _headers. Accept-Feature header is also held in this variable. Also renamed setUserHeaders as addHeader and it was rewritten to add header not just to clear the old value. * src/HttpRequestCommand.cc * src/Metalink2RequestGroup.cc: Added the code to remove "metalink" from "Accept-Feature" list. * src/RequestGroup.cc: Added "metalink" to "Accept-Feature" by default. * src/RequestGroup.h * src/TaggedItem.cc: Moved Concat class to a2functional.h. * src/a2functional.h: Included because Concat class depends on it. * test/HttpRequestTest.cc --- ChangeLog | 21 +++++++++++++++++++++ src/HttpRequest.cc | 10 +++++----- src/HttpRequest.h | 7 ++++--- src/HttpRequestCommand.cc | 23 +++++++++++++++++++---- src/Metalink2RequestGroup.cc | 8 +++++++- src/RequestGroup.cc | 28 ++++++++++++++++++++++++++++ src/RequestGroup.h | 10 ++++++++++ src/TaggedItem.cc | 13 +------------ src/a2functional.h | 13 +++++++++++++ test/HttpRequestTest.cc | 8 ++++---- 10 files changed, 112 insertions(+), 29 deletions(-) diff --git a/ChangeLog b/ChangeLog index 60d556680..6b5e3f98e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,24 @@ +2008-04-26 Tatsuhiro Tsujikawa + + Added build-in "Accept-Feature" support. For now only "metalink" is + used in this header field. + This "metalink" value is removed from the list when connecting URLs + fed by metalink file to avoid loop in "transparent" metlaink. + * src/HttpRequest.cc + * src/HttpRequest.h: Renamed _userHeaders as _headers. Accept-Feature + header is also held in this variable. Also renamed setUserHeaders as + addHeader and it was rewritten to add header not just to clear the old + value. + * src/HttpRequestCommand.cc + * src/Metalink2RequestGroup.cc: Added the code to remove "metalink" + from "Accept-Feature" list. + * src/RequestGroup.cc: Added "metalink" to "Accept-Feature" by default. + * src/RequestGroup.h + * src/TaggedItem.cc: Moved Concat class to a2functional.h. + * src/a2functional.h: Included because Concat class depends on + it. + * test/HttpRequestTest.cc + 2008-04-26 Tatsuhiro Tsujikawa Added comment diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index caf467a56..d303e9fc1 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -192,8 +192,8 @@ std::string HttpRequest::createRequest() const requestLine += std::string("Cookie: ")+cookiesValue+"\r\n"; } // append additional headers given by user. - for(std::deque::const_iterator i = _userHeaders.begin(); - i != _userHeaders.end(); ++i) { + for(std::deque::const_iterator i = _headers.begin(); + i != _headers.end(); ++i) { requestLine += (*i)+"\r\n"; } @@ -225,11 +225,11 @@ std::string HttpRequest::getProxyAuthString() const { Base64::encode(AuthConfigFactorySingleton::instance()->createAuthConfigForHttpProxy(request)->getAuthText())+"\r\n"; } -void HttpRequest::setUserHeaders(const std::string& userHeadersString) +void HttpRequest::addHeader(const std::string& headersString) { std::deque headers; - Util::slice(headers, userHeadersString, '\n', true); - _userHeaders = headers; + Util::slice(headers, headersString, '\n', true); + _headers.insert(_headers.end(), headers.begin(), headers.end()); } void HttpRequest::configure(const Option* option) diff --git a/src/HttpRequest.h b/src/HttpRequest.h index ae39bb47b..6a1cc1244 100644 --- a/src/HttpRequest.h +++ b/src/HttpRequest.h @@ -66,7 +66,7 @@ private: std::string userAgent; - std::deque _userHeaders; + std::deque _headers; std::string getHostText(const std::string& host, uint16_t port) const; @@ -166,8 +166,9 @@ public: { this->userAgent = userAgent; } - - void setUserHeaders(const std::string& userHeaders); + + // accepts multiline headers, deliminated by LF + void addHeader(const std::string& headers); }; typedef SharedHandle HttpRequestHandle; diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index cc76a9be4..e552007ce 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -44,6 +44,9 @@ #include "Option.h" #include "Socket.h" #include "prefs.h" +#include "a2functional.h" +#include "Util.h" +#include namespace aria2 { @@ -67,14 +70,24 @@ static SharedHandle createHttpRequest(const SharedHandle& req, const SharedHandle& segment, uint64_t totalLength, - const Option* option) + const Option* option, + const std::deque& acceptFeatures) { HttpRequestHandle httpRequest(new HttpRequest()); httpRequest->setUserAgent(option->get(PREF_USER_AGENT)); httpRequest->setRequest(req); httpRequest->setSegment(segment); httpRequest->setEntityLength(totalLength); - httpRequest->setUserHeaders(option->get(PREF_HEADER)); + httpRequest->addHeader(option->get(PREF_HEADER)); + if(acceptFeatures.size()) { + std::string acceptFeaturesHeader = "Accept-Features: "+ + Util::trim + (std::accumulate(acceptFeatures.begin()+1, acceptFeatures.end(), + *acceptFeatures.begin(), + Concat(",")), + ","); + httpRequest->addHeader(acceptFeaturesHeader); + } httpRequest->configure(option); return httpRequest; @@ -89,7 +102,8 @@ bool HttpRequestCommand::executeInternal() { if(_segments.empty()) { HttpRequestHandle httpRequest (createHttpRequest(req, SharedHandle(), - _requestGroup->getTotalLength(), e->option)); + _requestGroup->getTotalLength(), e->option, + _requestGroup->getAcceptFeatures())); _httpConnection->sendRequest(httpRequest); } else { for(Segments::iterator itr = _segments.begin(); itr != _segments.end(); ++itr) { @@ -97,7 +111,8 @@ bool HttpRequestCommand::executeInternal() { if(!_httpConnection->isIssued(segment)) { HttpRequestHandle httpRequest (createHttpRequest(req, segment, - _requestGroup->getTotalLength(), e->option)); + _requestGroup->getTotalLength(), e->option, + _requestGroup->getAcceptFeatures())); _httpConnection->sendRequest(httpRequest); } } diff --git a/src/Metalink2RequestGroup.cc b/src/Metalink2RequestGroup.cc index f033d9929..46dcc2bcc 100644 --- a/src/Metalink2RequestGroup.cc +++ b/src/Metalink2RequestGroup.cc @@ -169,6 +169,9 @@ Metalink2RequestGroup::createRequestGroup(std::deque torrentRg->setDownloadContext(dctx); torrentRg->clearPreDowloadHandler(); torrentRg->clearPostDowloadHandler(); + // remove "metalink" from Accept-Feature list to avoid loop in tranparent + // metalink + torrentRg->removeAcceptFeatureHeader(RequestGroup::FEATURE_METALINK); // make it in-memory download SharedHandle preh(new MemoryBufferPreDownloadHandler()); { @@ -219,7 +222,10 @@ Metalink2RequestGroup::createRequestGroup(std::deque std::min(_option->getAsInt(PREF_METALINK_SERVERS), entry->maxConnections)); // In metalink, multi connection to a single host is not allowed by default. rg->setSingleHostMultiConnectionEnabled(!_option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)); - + // remove "metalink" from Accept-Feature list to avoid loop in tranparent + // metalink + rg->removeAcceptFeatureHeader(RequestGroup::FEATURE_METALINK); + #ifdef ENABLE_BITTORRENT // Inject depenency between rg and torrentRg here if torrentRg.isNull() == false if(!torrentRg.isNull()) { diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index bc695c9dc..31772af8a 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -103,6 +103,8 @@ namespace aria2 { int32_t RequestGroup::_gidCounter = 0; +const std::string RequestGroup::FEATURE_METALINK = "metalink"; + RequestGroup::RequestGroup(const Option* option, const std::deque& uris): _gid(++_gidCounter), @@ -124,6 +126,12 @@ RequestGroup::RequestGroup(const Option* option, } else { _fileAllocationEnabled = false; } + // For now, only supported 'Accept-Features' is "metalink" used for + // transparent metalink. + // It would be good to put this value in Option so that user can tweak + // and add this list. + _acceptFeatures.push_back(FEATURE_METALINK); + initializePreDownloadHandler(); initializePostDownloadHandler(); } @@ -933,4 +941,24 @@ void RequestGroup::reportDownloadFinished() #endif // ENABLE_BITTORRENT } +const std::deque& RequestGroup::getAcceptFeatures() const +{ + return _acceptFeatures; +} + +void RequestGroup::addAcceptFeatureHeader(const std::string& feature) +{ + if(std::find(_acceptFeatures.begin(), _acceptFeatures.end(), feature) == _acceptFeatures.end()) { + _acceptFeatures.push_back(feature); + } +} + +void RequestGroup::removeAcceptFeatureHeader(const std::string& feature) +{ + std::deque::iterator i = std::find(_acceptFeatures.begin(), _acceptFeatures.end(), feature); + if(i != _acceptFeatures.end()) { + _acceptFeatures.erase(i); + } +} + } // namespace aria2 diff --git a/src/RequestGroup.h b/src/RequestGroup.h index eac026ad7..b6958ebd6 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -108,6 +108,8 @@ private: std::deque > _postDownloadHandlers; + std::deque _acceptFeatures; + const Option* _option; const Logger* _logger; @@ -334,6 +336,14 @@ public: void removeURIWhoseHostnameIs(const std::string& hostname); void reportDownloadFinished(); + + const std::deque& getAcceptFeatures() const; + + void addAcceptFeatureHeader(const std::string& feature); + + void removeAcceptFeatureHeader(const std::string& feature); + + static const std::string FEATURE_METALINK; }; typedef SharedHandle RequestGroupHandle; diff --git a/src/TaggedItem.cc b/src/TaggedItem.cc index 503e0b1f6..af12ee9ea 100644 --- a/src/TaggedItem.cc +++ b/src/TaggedItem.cc @@ -33,23 +33,12 @@ */ /* copyright --> */ #include "TaggedItem.h" +#include "a2functional.h" #include #include namespace aria2 { -class Concat { -private: - std::string _delim; -public: - Concat(const std::string& delim = ""):_delim(delim) {} - - std::string operator()(const std::string& s1, const std::string& s2) const - { - return s1+_delim+s2; - } -}; - std::string TaggedItem::toTagString() const { if(_tags.size()) { diff --git a/src/a2functional.h b/src/a2functional.h index a6ac6bded..09ec1a7c4 100644 --- a/src/a2functional.h +++ b/src/a2functional.h @@ -37,6 +37,7 @@ #include #include "SharedHandle.h" +#include namespace aria2 { @@ -138,6 +139,18 @@ public: } }; +class Concat { +private: + std::string _delim; +public: + Concat(const std::string& delim = ""):_delim(delim) {} + + std::string operator()(const std::string& s1, const std::string& s2) const + { + return s1+_delim+s2; + } +}; + } // namespace aria2 #endif // _D_A2_FUNCTIONAL_H_ diff --git a/test/HttpRequestTest.cc b/test/HttpRequestTest.cc index d514f3d4d..7400d0979 100644 --- a/test/HttpRequestTest.cc +++ b/test/HttpRequestTest.cc @@ -22,7 +22,7 @@ class HttpRequestTest : public CppUnit::TestFixture { CPPUNIT_TEST(testCreateProxyRequest); CPPUNIT_TEST(testIsRangeSatisfied); CPPUNIT_TEST(testUserAgent); - CPPUNIT_TEST(testUserHeaders); + CPPUNIT_TEST(testAddHeader); CPPUNIT_TEST_SUITE_END(); private: @@ -37,7 +37,7 @@ public: void testCreateProxyRequest(); void testIsRangeSatisfied(); void testUserAgent(); - void testUserHeaders(); + void testAddHeader(); }; @@ -596,14 +596,14 @@ void HttpRequestTest::testUserAgent() CPPUNIT_ASSERT_EQUAL(expectedTextForProxy, httpRequest.createProxyRequest()); } -void HttpRequestTest::testUserHeaders() +void HttpRequestTest::testAddHeader() { SharedHandle request(new Request()); request->setUrl("http://localhost/archives/aria2-1.0.0.tar.bz2"); HttpRequest httpRequest; httpRequest.setRequest(request); - httpRequest.setUserHeaders("X-ARIA2: v0.13\nX-ARIA2-DISTRIBUTE: enabled\n"); + httpRequest.addHeader("X-ARIA2: v0.13\nX-ARIA2-DISTRIBUTE: enabled\n"); std::string expectedText = "GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n" "User-Agent: aria2\r\n"