From f596de9eec5ea01a28b7ee5d632164d8b41221ec Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 5 May 2008 08:25:41 +0000 Subject: [PATCH] 2008-05-05 Tatsuhiro Tsujikawa Reimplemented transparent metalink using Accept request header. * src/HttpRequest.cc * src/HttpRequest.h * src/HttpRequestCommand.cc * src/Metalink2RequestGroup.cc * src/RequestGroup.cc * src/RequestGroup.h * test/HttpRequestTest.cc --- ChangeLog | 11 +++++++++++ src/HttpRequest.cc | 16 ++++++++++++++-- src/HttpRequest.h | 10 ++++++++++ src/HttpRequestCommand.cc | 11 +++++++---- src/Metalink2RequestGroup.cc | 8 ++++---- src/RequestGroup.cc | 26 ++++++++++++++++++++++---- src/RequestGroup.h | 10 +++++++++- test/HttpRequestTest.cc | 27 +++++++++++++++++++++++++++ 8 files changed, 104 insertions(+), 15 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8fd58c93f..a297833b3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +2008-05-05 Tatsuhiro Tsujikawa + + Reimplemented transparent metalink using Accept request header. + * src/HttpRequest.cc + * src/HttpRequest.h + * src/HttpRequestCommand.cc + * src/Metalink2RequestGroup.cc + * src/RequestGroup.cc + * src/RequestGroup.h + * test/HttpRequestTest.cc + 2008-04-27 Tatsuhiro Tsujikawa Bump up version number to 0.13.2a diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index 70e281da0..ab260a464 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -44,6 +44,7 @@ #include "prefs.h" #include "AuthConfigFactory.h" #include "AuthConfig.h" +#include "a2functional.h" #include namespace aria2 { @@ -145,11 +146,17 @@ std::string HttpRequest::createRequest() const } requestLine += std::string(" HTTP/1.1\r\n")+ - "User-Agent: "+userAgent+"\r\n"+ - "Accept: */*\r\n"+ /* */ + "User-Agent: "+userAgent+"\r\n"; + + requestLine += + std::accumulate(_acceptTypes.begin(), _acceptTypes.end(), + std::string("Accept: */*"), Concat(","))+"\r\n"; /* */ + + requestLine += "Host: "+getHostText(getHost(), getPort())+"\r\n"+ "Pragma: no-cache\r\n"+ "Cache-Control: no-cache\r\n"; + if(!request->isKeepAliveEnabled() && !request->isPipeliningEnabled()) { requestLine += "Connection: close\r\n"; } @@ -232,6 +239,11 @@ void HttpRequest::addHeader(const std::string& headersString) _headers.insert(_headers.end(), headers.begin(), headers.end()); } +void HttpRequest::addAcceptType(const std::string& type) +{ + _acceptTypes.push_back(type); +} + void HttpRequest::configure(const Option* option) { authEnabled = option->get(PREF_HTTP_AUTH_ENABLED) == V_TRUE; diff --git a/src/HttpRequest.h b/src/HttpRequest.h index ddf0ecfe1..58798c1dd 100644 --- a/src/HttpRequest.h +++ b/src/HttpRequest.h @@ -68,6 +68,8 @@ private: std::deque _headers; + std::deque _acceptTypes; + std::string getHostText(const std::string& host, uint16_t port) const; std::string getProxyAuthString() const; @@ -171,6 +173,14 @@ public: // accepts multiline headers, deliminated by LF void addHeader(const std::string& headers); + + void addAcceptType(const std::string& type); + + template + void addAcceptType(InputIterator first, InputIterator last) + { + _acceptTypes.insert(_acceptTypes.end(), first, last); + } }; typedef SharedHandle HttpRequestHandle; diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index e552007ce..803f876c8 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -71,7 +71,7 @@ createHttpRequest(const SharedHandle& req, const SharedHandle& segment, uint64_t totalLength, const Option* option, - const std::deque& acceptFeatures) + const RequestGroup* rg) { HttpRequestHandle httpRequest(new HttpRequest()); httpRequest->setUserAgent(option->get(PREF_USER_AGENT)); @@ -79,7 +79,8 @@ createHttpRequest(const SharedHandle& req, httpRequest->setSegment(segment); httpRequest->setEntityLength(totalLength); httpRequest->addHeader(option->get(PREF_HEADER)); - if(acceptFeatures.size()) { + if(rg->getAcceptFeatures().size()) { + const std::deque& acceptFeatures = rg->getAcceptFeatures(); std::string acceptFeaturesHeader = "Accept-Features: "+ Util::trim (std::accumulate(acceptFeatures.begin()+1, acceptFeatures.end(), @@ -88,6 +89,8 @@ createHttpRequest(const SharedHandle& req, ","); httpRequest->addHeader(acceptFeaturesHeader); } + httpRequest->addAcceptType(rg->getAcceptTypes().begin(), + rg->getAcceptTypes().end()); httpRequest->configure(option); return httpRequest; @@ -103,7 +106,7 @@ bool HttpRequestCommand::executeInternal() { HttpRequestHandle httpRequest (createHttpRequest(req, SharedHandle(), _requestGroup->getTotalLength(), e->option, - _requestGroup->getAcceptFeatures())); + _requestGroup)); _httpConnection->sendRequest(httpRequest); } else { for(Segments::iterator itr = _segments.begin(); itr != _segments.end(); ++itr) { @@ -112,7 +115,7 @@ bool HttpRequestCommand::executeInternal() { HttpRequestHandle httpRequest (createHttpRequest(req, segment, _requestGroup->getTotalLength(), e->option, - _requestGroup->getAcceptFeatures())); + _requestGroup)); _httpConnection->sendRequest(httpRequest); } } diff --git a/src/Metalink2RequestGroup.cc b/src/Metalink2RequestGroup.cc index 46dcc2bcc..e62c8c7f0 100644 --- a/src/Metalink2RequestGroup.cc +++ b/src/Metalink2RequestGroup.cc @@ -169,9 +169,9 @@ Metalink2RequestGroup::createRequestGroup(std::deque torrentRg->setDownloadContext(dctx); torrentRg->clearPreDowloadHandler(); torrentRg->clearPostDowloadHandler(); - // remove "metalink" from Accept-Feature list to avoid loop in tranparent + // remove "metalink" from Accept Type list to avoid loop in tranparent // metalink - torrentRg->removeAcceptFeatureHeader(RequestGroup::FEATURE_METALINK); + torrentRg->removeAcceptType(RequestGroup::ACCEPT_METALINK); // make it in-memory download SharedHandle preh(new MemoryBufferPreDownloadHandler()); { @@ -222,9 +222,9 @@ Metalink2RequestGroup::createRequestGroup(std::deque std::min(_option->getAsInt(PREF_METALINK_SERVERS), entry->maxConnections)); // In metalink, multi connection to a single host is not allowed by default. rg->setSingleHostMultiConnectionEnabled(!_option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)); - // remove "metalink" from Accept-Feature list to avoid loop in tranparent + // remove "metalink" from Accept Type list to avoid loop in tranparent // metalink - rg->removeAcceptFeatureHeader(RequestGroup::FEATURE_METALINK); + rg->removeAcceptType(RequestGroup::ACCEPT_METALINK); #ifdef ENABLE_BITTORRENT // Inject depenency between rg and torrentRg here if torrentRg.isNull() == false diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 5866cdf4e..82e8bc529 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -104,7 +104,7 @@ namespace aria2 { int32_t RequestGroup::_gidCounter = 0; -const std::string RequestGroup::FEATURE_METALINK = "metalink"; +const std::string RequestGroup::ACCEPT_METALINK = "application/metalink+xml"; RequestGroup::RequestGroup(const Option* option, const std::deque& uris): @@ -127,11 +127,11 @@ RequestGroup::RequestGroup(const Option* option, } else { _fileAllocationEnabled = false; } - // For now, only supported 'Accept-Features' is "metalink" used for - // transparent metalink. + // Add types to be sent as a Accept header value here. // It would be good to put this value in Option so that user can tweak // and add this list. - _acceptFeatures.push_back(FEATURE_METALINK); + // ACCEPT_METALINK is used for `transparent metalink'. + addAcceptType(ACCEPT_METALINK); initializePreDownloadHandler(); initializePostDownloadHandler(); @@ -969,4 +969,22 @@ void RequestGroup::removeAcceptFeatureHeader(const std::string& feature) } } +const std::deque& RequestGroup::getAcceptTypes() const +{ + return _acceptTypes; +} + +void RequestGroup::addAcceptType(const std::string& type) +{ + if(std::find(_acceptTypes.begin(), _acceptTypes.end(), type) == _acceptTypes.end()) { + _acceptTypes.push_back(type); + } +} + +void RequestGroup::removeAcceptType(const std::string& type) +{ + _acceptTypes.erase(std::remove(_acceptTypes.begin(), _acceptTypes.end(), type), + _acceptTypes.end()); +} + } // namespace aria2 diff --git a/src/RequestGroup.h b/src/RequestGroup.h index 657eb2c9e..6801c2435 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -110,6 +110,8 @@ private: std::deque _acceptFeatures; + std::deque _acceptTypes; + const Option* _option; Logger* _logger; @@ -343,7 +345,13 @@ public: void removeAcceptFeatureHeader(const std::string& feature); - static const std::string FEATURE_METALINK; + const std::deque& getAcceptTypes() const; + + void addAcceptType(const std::string& type); + + void removeAcceptType(const std::string& type); + + static const std::string ACCEPT_METALINK; }; typedef SharedHandle RequestGroupHandle; diff --git a/test/HttpRequestTest.cc b/test/HttpRequestTest.cc index dde436fea..19c044916 100644 --- a/test/HttpRequestTest.cc +++ b/test/HttpRequestTest.cc @@ -7,6 +7,7 @@ #include "Request.h" #include "CookieBox.h" #include "Option.h" +#include "array_fun.h" #include namespace aria2 { @@ -24,6 +25,7 @@ class HttpRequestTest : public CppUnit::TestFixture { CPPUNIT_TEST(testIsRangeSatisfied); CPPUNIT_TEST(testUserAgent); CPPUNIT_TEST(testAddHeader); + CPPUNIT_TEST(testAddAcceptType); CPPUNIT_TEST_SUITE_END(); private: @@ -40,6 +42,7 @@ public: void testIsRangeSatisfied(); void testUserAgent(); void testAddHeader(); + void testAddAcceptType(); }; @@ -641,5 +644,29 @@ void HttpRequestTest::testAddHeader() CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); } +void HttpRequestTest::testAddAcceptType() +{ + std::string acceptTypes[] = { "cream/custard", + "muffin/chocolate" }; + + SharedHandle request(new Request()); + request->setUrl("http://localhost/archives/aria2-1.0.0.tar.bz2"); + + HttpRequest httpRequest; + httpRequest.setRequest(request); + httpRequest.addAcceptType(&acceptTypes[0], &acceptTypes[arrayLength(acceptTypes)]); + + std::string expectedText = + "GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n" + "User-Agent: aria2\r\n" + "Accept: */*,cream/custard,muffin/chocolate\r\n" + "Host: localhost\r\n" + "Pragma: no-cache\r\n" + "Cache-Control: no-cache\r\n" + "Connection: close\r\n" + "\r\n"; + + CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); +} } // namespace aria2