From 01e1fdaf495342ae8735ec74eb57f912fa67d535 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 26 Jul 2012 21:07:10 +0900 Subject: [PATCH] Parse torrent file on the fly if --follow-torrent=mem is given In this changes, Bencode parser in bencode2 is removed and the request is delegated to BencodeParser. --- ...loadHandler.cc => AnonDiskWriterFactory.h} | 35 +-- ...skWriterFactory.cc => BencodeDiskWriter.h} | 21 +- ...skWriter.cc => BencodeDiskWriterFactory.h} | 47 +--- src/BtPostDownloadHandler.cc | 44 +++- src/ByteArrayDiskWriterFactory.h | 14 +- src/DHTMessageReceiver.cc | 2 +- src/DefaultBtAnnounce.cc | 2 +- src/DefaultExtensionMessageFactory.cc | 2 +- src/DownloadHandlerFactory.cc | 7 +- src/DownloadHandlerFactory.h | 9 +- src/GenericParser.h | 34 ++- src/HandshakeExtensionMessage.cc | 2 +- src/JsonDiskWriter.h | 47 +--- src/Makefile.am | 16 +- src/MemoryBencodePreDownloadHandler.h | 52 ++++ src/MemoryBufferPreDownloadHandler.h | 14 +- src/MemoryPreDownloadHandler.h | 64 +++++ src/UTPexExtensionMessage.cc | 2 +- src/ValueBaseDiskWriter.h | 111 +++++++++ src/ValueBaseStructParserStateMachine.cc | 7 +- src/ValueBaseStructParserStateMachine.h | 2 +- src/bencode2.cc | 226 ++---------------- src/bencode2.h | 26 +- src/bittorrent_helper.cc | 30 ++- src/bittorrent_helper.h | 7 + src/download_helper.cc | 66 +++-- src/download_helper.h | 10 + test/AnnounceListTest.cc | 27 +-- test/Bencode2Test.cc | 175 -------------- test/BittorrentHelperTest.cc | 18 +- 30 files changed, 510 insertions(+), 609 deletions(-) rename src/{MemoryBufferPreDownloadHandler.cc => AnonDiskWriterFactory.h} (71%) rename src/{ByteArrayDiskWriterFactory.cc => BencodeDiskWriter.h} (83%) rename src/{JsonDiskWriter.cc => BencodeDiskWriterFactory.h} (69%) create mode 100644 src/MemoryBencodePreDownloadHandler.h create mode 100644 src/MemoryPreDownloadHandler.h create mode 100644 src/ValueBaseDiskWriter.h diff --git a/src/MemoryBufferPreDownloadHandler.cc b/src/AnonDiskWriterFactory.h similarity index 71% rename from src/MemoryBufferPreDownloadHandler.cc rename to src/AnonDiskWriterFactory.h index bdf60bf8a..f92556d19 100644 --- a/src/MemoryBufferPreDownloadHandler.cc +++ b/src/AnonDiskWriterFactory.h @@ -2,7 +2,7 @@ /* * aria2 - The high speed download utility * - * Copyright (C) 2006 Tatsuhiro Tsujikawa + * Copyright (C) 2012 Tatsuhiro Tsujikawa * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -32,24 +32,27 @@ * files in the program, then also delete it here. */ /* copyright --> */ -#include "MemoryBufferPreDownloadHandler.h" -#include "RequestGroup.h" -#include "ByteArrayDiskWriterFactory.h" -#include "DownloadContext.h" +#ifndef D_ANON_DISK_WRITER_FACTORY_H +#define D_ANON_DISK_WRITER_FACTORY_H + +#include "DiskWriterFactory.h" namespace aria2 { -MemoryBufferPreDownloadHandler::MemoryBufferPreDownloadHandler() {} +// DiskwriterFactory class template to create DiskWriter derived +// object, ignoring filename. +template +class AnonDiskWriterFactory:public DiskWriterFactory { +public: + AnonDiskWriterFactory() {} + virtual ~AnonDiskWriterFactory() {} -MemoryBufferPreDownloadHandler::~MemoryBufferPreDownloadHandler() {} - -void MemoryBufferPreDownloadHandler::execute(RequestGroup* requestGroup) -{ - SharedHandle dwf(new ByteArrayDiskWriterFactory()); - requestGroup->setDiskWriterFactory(dwf); - requestGroup->setFileAllocationEnabled(false); - requestGroup->setPreLocalFileCheckEnabled(false); - requestGroup->markInMemoryDownload(); -} + virtual SharedHandle newDiskWriter(const std::string& filename) + { + return SharedHandle(new DiskWriterType()); + } +}; } // namespace aria2 + +#endif // D_ANON_DISK_WRITER_FACTORY_H diff --git a/src/ByteArrayDiskWriterFactory.cc b/src/BencodeDiskWriter.h similarity index 83% rename from src/ByteArrayDiskWriterFactory.cc rename to src/BencodeDiskWriter.h index 495488e4e..1554c7995 100644 --- a/src/ByteArrayDiskWriterFactory.cc +++ b/src/BencodeDiskWriter.h @@ -2,7 +2,7 @@ /* * aria2 - The high speed download utility * - * Copyright (C) 2006 Tatsuhiro Tsujikawa + * Copyright (C) 2012 Tatsuhiro Tsujikawa * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -32,15 +32,20 @@ * files in the program, then also delete it here. */ /* copyright --> */ -#include "ByteArrayDiskWriterFactory.h" -#include "ByteArrayDiskWriter.h" +#ifndef D_BENCODE_DISK_WRITER_H +#define D_BENCODE_DISK_WRITER_H + +#include "ValueBaseDiskWriter.h" +#include "BencodeParser.h" namespace aria2 { -DiskWriterHandle ByteArrayDiskWriterFactory::newDiskWriter -(const std::string& filename) -{ - return SharedHandle(new ByteArrayDiskWriter()); -} +namespace bittorrent { + +typedef ValueBaseDiskWriter BencodeDiskWriter; + +} // namespace bittorrent } // namespace aria2 + +#endif // D_BENCODE_DISK_WRITER_H diff --git a/src/JsonDiskWriter.cc b/src/BencodeDiskWriterFactory.h similarity index 69% rename from src/JsonDiskWriter.cc rename to src/BencodeDiskWriterFactory.h index b616e2bd5..a4a6038b9 100644 --- a/src/JsonDiskWriter.cc +++ b/src/BencodeDiskWriterFactory.h @@ -32,47 +32,20 @@ * files in the program, then also delete it here. */ /* copyright --> */ -#include "JsonDiskWriter.h" -#include "ValueBase.h" +#ifndef D_BENCODE_DISK_WRITER_FACTORY_H +#define D_BENCODE_DISK_WRITER_FACTORY_H + +#include "AnonDiskWriterFactory.h" +#include "BencodeDiskWriter.h" namespace aria2 { -namespace json { +namespace bittorrent { -JsonDiskWriter::JsonDiskWriter() - : parser_(&psm_) -{} +typedef AnonDiskWriterFactory BencodeDiskWriterFactory; -JsonDiskWriter::~JsonDiskWriter() -{} - -void JsonDiskWriter::initAndOpenFile(int64_t totalLength) -{ - parser_.reset(); -} - -void JsonDiskWriter::writeData(const unsigned char* data, size_t len, - int64_t offset) -{ - // Return value is ignored here but handled in finalize() - parser_.parseUpdate(reinterpret_cast(data), len); -} - -int JsonDiskWriter::finalize() -{ - return parser_.parseFinal(0, 0); -} - -SharedHandle JsonDiskWriter::getResult() const -{ - return psm_.getResult(); -} - -void JsonDiskWriter::reset() -{ - parser_.reset(); -} - -} // namespace json +} // namespace bittorrent } // namespace aria2 + +#endif // D_BENCODE_DISK_WRITER_FACTORY_H diff --git a/src/BtPostDownloadHandler.cc b/src/BtPostDownloadHandler.cc index 00ecea282..c3faccfc9 100644 --- a/src/BtPostDownloadHandler.cc +++ b/src/BtPostDownloadHandler.cc @@ -48,6 +48,10 @@ #include "DownloadContext.h" #include "download_helper.h" #include "fmt.h" +#include "ValueBaseBencodeParser.h" +#include "DiskWriter.h" +#include "AbstractSingleDiskAdaptor.h" +#include "BencodeDiskWriter.h" namespace aria2 { @@ -70,19 +74,41 @@ void BtPostDownloadHandler::getNextRequestGroups { A2_LOG_INFO(fmt("Generating RequestGroups for Torrent file %s", requestGroup->getFirstFilePath().c_str())); - std::string content; - try { - requestGroup->getPieceStorage()->getDiskAdaptor()->openExistingFile(); - content = util::toString(requestGroup->getPieceStorage()->getDiskAdaptor()); - requestGroup->getPieceStorage()->getDiskAdaptor()->closeFile(); - } catch(Exception& e) { - requestGroup->getPieceStorage()->getDiskAdaptor()->closeFile(); - throw; + SharedHandle torrent; + if(requestGroup->inMemoryDownload()) { + const SharedHandle& dw = + static_pointer_cast + (requestGroup->getPieceStorage()->getDiskAdaptor())->getDiskWriter(); + const SharedHandle& bdw = + static_pointer_cast(dw); + int error = bdw->finalize(); + if(error == 0) { + torrent = bdw->getResult(); + } + } else { + std::string content; + try { + requestGroup->getPieceStorage()->getDiskAdaptor()->openExistingFile(); + content = util::toString(requestGroup->getPieceStorage() + ->getDiskAdaptor()); + requestGroup->getPieceStorage()->getDiskAdaptor()->closeFile(); + } catch(Exception& e) { + requestGroup->getPieceStorage()->getDiskAdaptor()->closeFile(); + throw; + } + ssize_t error; + torrent = bittorrent::ValueBaseBencodeParser().parseFinal + (content.c_str(), content.size(), error); + } + if(!torrent) { + throw DL_ABORT_EX2("Could not parse BitTorrent metainfo", + error_code::BENCODE_PARSE_ERROR); } std::vector > newRgs; createRequestGroupForBitTorrent(newRgs, requestGroup->getOption(), std::vector(), - content); + "", + torrent); requestGroup->followedBy(newRgs.begin(), newRgs.end()); SharedHandle mi = createMetadataInfoFromFirstFileEntry(requestGroup->getDownloadContext()); diff --git a/src/ByteArrayDiskWriterFactory.h b/src/ByteArrayDiskWriterFactory.h index c770e7acd..5a8393712 100644 --- a/src/ByteArrayDiskWriterFactory.h +++ b/src/ByteArrayDiskWriterFactory.h @@ -2,7 +2,7 @@ /* * aria2 - The high speed download utility * - * Copyright (C) 2006 Tatsuhiro Tsujikawa + * Copyright (C) 2012 Tatsuhiro Tsujikawa * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -35,17 +35,13 @@ #ifndef D_BYTE_ARRAY_DISK_WRITER_FACTORY_H #define D_BYTE_ARRAY_DISK_WRITER_FACTORY_H -#include "DiskWriterFactory.h" +#include "AnonDiskWriterFactory.h" +#include "ByteArrayDiskWriter.h" namespace aria2 { -class ByteArrayDiskWriter; - -class ByteArrayDiskWriterFactory:public DiskWriterFactory -{ -public: - SharedHandle newDiskWriter(const std::string& filename); -}; +typedef AnonDiskWriterFactory +ByteArrayDiskWriterFactory; typedef SharedHandle ByteArrayDiskWriterFactoryHandle; diff --git a/src/DHTMessageReceiver.cc b/src/DHTMessageReceiver.cc index 8b2b98b17..e5558d9d1 100644 --- a/src/DHTMessageReceiver.cc +++ b/src/DHTMessageReceiver.cc @@ -76,7 +76,7 @@ SharedHandle DHTMessageReceiver::receiveMessage() return SharedHandle(); } bool isReply = false; - SharedHandle decoded = bencode2::decode(data, data+length); + SharedHandle decoded = bencode2::decode(data, length); const Dict* dict = downcast(decoded); if(dict) { const String* y = downcast(dict->get(DHTMessage::Y)); diff --git a/src/DefaultBtAnnounce.cc b/src/DefaultBtAnnounce.cc index 8f60285cb..1a76df15b 100644 --- a/src/DefaultBtAnnounce.cc +++ b/src/DefaultBtAnnounce.cc @@ -222,7 +222,7 @@ DefaultBtAnnounce::processAnnounceResponse(const unsigned char* trackerResponse, { A2_LOG_DEBUG("Now processing tracker response."); SharedHandle decodedValue = - bencode2::decode(trackerResponse, trackerResponse+trackerResponseLength); + bencode2::decode(trackerResponse, trackerResponseLength); const Dict* dict = downcast(decodedValue); if(!dict) { throw DL_ABORT_EX(MSG_NULL_TRACKER_RESPONSE); diff --git a/src/DefaultExtensionMessageFactory.cc b/src/DefaultExtensionMessageFactory.cc index 47fa26b3f..a86765c09 100644 --- a/src/DefaultExtensionMessageFactory.cc +++ b/src/DefaultExtensionMessageFactory.cc @@ -102,7 +102,7 @@ DefaultExtensionMessageFactory::createMessage(const unsigned char* data, size_t } size_t end; SharedHandle decoded = - bencode2::decode(data+1, data+length, end); + bencode2::decode(data+1, length - 1, end); const Dict* dict = downcast(decoded); if(!dict) { throw DL_ABORT_EX("Bad ut_metadata: dictionary not found"); diff --git a/src/DownloadHandlerFactory.cc b/src/DownloadHandlerFactory.cc index 985a90c7a..8fdb7a35e 100644 --- a/src/DownloadHandlerFactory.cc +++ b/src/DownloadHandlerFactory.cc @@ -54,7 +54,7 @@ DownloadHandlerFactory::metalinkPostDownloadHandler_; #ifdef ENABLE_BITTORRENT -SharedHandle +SharedHandle DownloadHandlerFactory::btPreDownloadHandler_; SharedHandle @@ -96,11 +96,12 @@ DownloadHandlerFactory::getMetalinkPostDownloadHandler() #ifdef ENABLE_BITTORRENT -SharedHandle +SharedHandle DownloadHandlerFactory::getBtPreDownloadHandler() { if(!btPreDownloadHandler_) { - btPreDownloadHandler_.reset(new MemoryBufferPreDownloadHandler()); + btPreDownloadHandler_.reset + (new bittorrent::MemoryBencodePreDownloadHandler()); RequestGroupCriteriaHandle criteria (new ContentTypeRequestGroupCriteria diff --git a/src/DownloadHandlerFactory.h b/src/DownloadHandlerFactory.h index 94550db5c..1ec81fd7f 100644 --- a/src/DownloadHandlerFactory.h +++ b/src/DownloadHandlerFactory.h @@ -37,10 +37,13 @@ #include "common.h" #include "SharedHandle.h" +#include "MemoryBufferPreDownloadHandler.h" +#ifdef ENABLE_BITTORRENT +# include "MemoryBencodePreDownloadHandler.h" +#endif // ENABLE_BITTORRENT namespace aria2 { -class MemoryBufferPreDownloadHandler; #ifdef ENABLE_METALINK class MetalinkPostDownloadHandler; #endif // ENABLE_METALINK @@ -61,7 +64,7 @@ private: #endif // ENABLE_METALINK #ifdef ENABLE_BITTORRENT - static SharedHandle + static SharedHandle btPreDownloadHandler_; static SharedHandle @@ -80,7 +83,7 @@ public: #endif // ENABLE_METALINK #ifdef ENABLE_BITTORRENT - static SharedHandle + static SharedHandle getBtPreDownloadHandler(); static SharedHandle diff --git a/src/GenericParser.h b/src/GenericParser.h index 77985f698..f988d58bf 100644 --- a/src/GenericParser.h +++ b/src/GenericParser.h @@ -37,6 +37,8 @@ #include "common.h" #include "SharedHandle.h" +#include "a2io.h" +#include "util.h" namespace aria2 { @@ -50,6 +52,9 @@ public: ~GenericParser() {} + typedef typename ParserStateMachine::ResultType ResultType; + typedef ParserStateMachine ParserStateMachineType; + // Parses |size| bytes of data |data| and returns the number of // bytes processed. On error, one of the negative error codes is // returned. @@ -63,13 +68,13 @@ public: // number of bytes processed (>= 0). On error, it will be one of the // negative error code. This function also resets underlying parser // facility and make it ready to reuse. - typename ParserStateMachine::ResultType + ResultType parseFinal(const char* data, size_t size, ssize_t& error) { - typename ParserStateMachine::ResultType res; + ResultType res; error = parser_.parseFinal(data, size); if(error < 0) { - res = ParserStateMachine::noResult; + res = ParserStateMachine::noResult(); } else { res = psm_.getResult(); } @@ -81,6 +86,29 @@ private: Parser parser_; }; +template +typename Parser::ResultType parseFile(Parser& parser, + const std::string& filename) +{ + int fd; + // TODO Overrode a2open(const char*,..) and a2open(const std::wstring&,..) + while((fd = a2open(utf8ToWChar(filename).c_str(), + O_BINARY | O_RDONLY, OPEN_MODE)) == -1 && fd != EINTR); + if(fd == -1) { + return Parser::ParserStateMachineType::noResult(); + } + char buf[4096]; + ssize_t nread; + ssize_t nproc; + while((nread = read(fd, buf, sizeof(buf))) > 0) { + nproc = parser.parseUpdate(buf, nread); + if(nproc < 0) { + break; + } + } + return parser.parseFinal(0, 0, nproc); +} + } // namespace aria2 #endif // D_GENERIC_PARSER_H diff --git a/src/HandshakeExtensionMessage.cc b/src/HandshakeExtensionMessage.cc index cdfc9d1da..4a9c065c8 100644 --- a/src/HandshakeExtensionMessage.cc +++ b/src/HandshakeExtensionMessage.cc @@ -168,7 +168,7 @@ HandshakeExtensionMessage::create(const unsigned char* data, size_t length) HandshakeExtensionMessageHandle msg(new HandshakeExtensionMessage()); A2_LOG_DEBUG(fmt("Creating HandshakeExtensionMessage from %s", util::percentEncode(data, length).c_str())); - SharedHandle decoded = bencode2::decode(data+1, data+length); + SharedHandle decoded = bencode2::decode(data+1, length - 1); const Dict* dict = downcast(decoded); if(!dict) { throw DL_ABORT_EX diff --git a/src/JsonDiskWriter.h b/src/JsonDiskWriter.h index 72a4a1c1f..70ba2660b 100644 --- a/src/JsonDiskWriter.h +++ b/src/JsonDiskWriter.h @@ -35,57 +35,14 @@ #ifndef D_JSON_DISK_WRITER_H #define D_JSON_DISK_WRITER_H -#include "DiskWriter.h" -#include "ValueBaseStructParserStateMachine.h" +#include "ValueBaseDiskWriter.h" #include "JsonParser.h" namespace aria2 { namespace json { -// DiskWriter backed with ValueBaseJsonParser. The written bytes are -// consumed by ValueBaseJsonParser. It is only capable of sequential -// write so offset argument in write() will be ignored. It also does -// not offer read(). -class JsonDiskWriter : public DiskWriter { -public: - JsonDiskWriter(); - - virtual ~JsonDiskWriter(); - - virtual void initAndOpenFile(int64_t totalLength = 0); - - virtual void openFile(int64_t totalLength = 0) - { - initAndOpenFile(totalLength); - } - - virtual void closeFile() {} - - virtual void openExistingFile(int64_t totalLength = 0) - { - initAndOpenFile(totalLength); - } - - virtual int64_t size() - { - return 0; - } - - virtual void writeData(const unsigned char* data, size_t len, int64_t offset); - - virtual ssize_t readData(unsigned char* data, size_t len, int64_t offset) - { - return 0; - } - - int finalize(); - SharedHandle getResult() const; - void reset(); -private: - ValueBaseStructParserStateMachine psm_; - JsonParser parser_; -}; +typedef ValueBaseDiskWriter JsonDiskWriter; } // namespace json diff --git a/src/Makefile.am b/src/Makefile.am index 25ec37239..8766237b0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -116,7 +116,8 @@ SRCS = Socket.h\ DownloadHandler.cc DownloadHandler.h\ DownloadHandlerConstants.cc DownloadHandlerConstants.h\ DownloadHandlerFactory.cc DownloadHandlerFactory.h\ - MemoryBufferPreDownloadHandler.cc MemoryBufferPreDownloadHandler.h\ + MemoryPreDownloadHandler.h\ + MemoryBufferPreDownloadHandler.h\ HaveEraseCommand.cc HaveEraseCommand.h\ Piece.cc Piece.h\ CheckIntegrityMan.h\ @@ -132,7 +133,7 @@ SRCS = Socket.h\ BtRegistry.cc BtRegistry.h\ MultiFileAllocationIterator.cc MultiFileAllocationIterator.h\ ByteArrayDiskWriter.cc ByteArrayDiskWriter.h\ - ByteArrayDiskWriterFactory.cc ByteArrayDiskWriterFactory.h\ + ByteArrayDiskWriterFactory.h\ DownloadContext.cc DownloadContext.h\ TimedHaltCommand.cc TimedHaltCommand.h\ CUIDCounter.cc CUIDCounter.h\ @@ -205,7 +206,7 @@ SRCS = Socket.h\ ValueBaseStructParserState.h\ ValueBaseStructParserStateImpl.cc ValueBaseStructParserStateImpl.h\ ValueBaseStructParserStateMachine.cc ValueBaseStructParserStateMachine.h\ - JsonDiskWriter.cc JsonDiskWriter.h\ + JsonDiskWriter.h\ HttpServerBodyCommand.cc HttpServerBodyCommand.h\ RpcRequest.cc RpcRequest.h\ RpcMethod.cc RpcMethod.h\ @@ -234,7 +235,9 @@ SRCS = Socket.h\ rpc_helper.cc rpc_helper.h\ WatchProcessCommand.cc WatchProcessCommand.h\ UnknownOptionException.cc UnknownOptionException.h\ - Notifier.cc Notifier.h + Notifier.cc Notifier.h\ + ValueBaseDiskWriter.h\ + AnonDiskWriterFactory.h if MINGW_BUILD SRCS += WinConsoleFile.cc WinConsoleFile.h @@ -510,7 +513,10 @@ SRCS += PeerAbstractCommand.cc PeerAbstractCommand.h\ LpdDispatchMessageCommand.cc LpdDispatchMessageCommand.h\ bencode2.cc bencode2.h\ BencodeParser.cc BencodeParser.h\ - ValueBaseBencodeParser.h + ValueBaseBencodeParser.h\ + BencodeDiskWriter.h\ + BencodeDiskWriterFactory.h\ + MemoryBencodePreDownloadHandler.h endif # ENABLE_BITTORRENT if ENABLE_METALINK diff --git a/src/MemoryBencodePreDownloadHandler.h b/src/MemoryBencodePreDownloadHandler.h new file mode 100644 index 000000000..98e6f2755 --- /dev/null +++ b/src/MemoryBencodePreDownloadHandler.h @@ -0,0 +1,52 @@ +/* */ +#ifndef D_MEMORY_BENCODE_PRE_DOWNLOAD_HANDLER_H +#define D_MEMORY_BENCODE_PRE_DOWNLOAD_HANDLER_H + +#include "MemoryPreDownloadHandler.h" +#include "BencodeDiskWriterFactory.h" + +namespace aria2 { + +namespace bittorrent { + +typedef MemoryPreDownloadHandler +MemoryBencodePreDownloadHandler; + +} // namespace bittorrent + +} // namespace aria2 + +#endif // D_MEMORY_BENCODE_PRE_DOWNLOAD_HANDLER_H diff --git a/src/MemoryBufferPreDownloadHandler.h b/src/MemoryBufferPreDownloadHandler.h index dda90b739..8e736a776 100644 --- a/src/MemoryBufferPreDownloadHandler.h +++ b/src/MemoryBufferPreDownloadHandler.h @@ -35,19 +35,13 @@ #ifndef D_MEMORY_BUFFER_PRE_DOWNLOAD_HANDLER_H #define D_MEMORY_BUFFER_PRE_DOWNLOAD_HANDLER_H -#include "PreDownloadHandler.h" +#include "MemoryPreDownloadHandler.h" +#include "ByteArrayDiskWriterFactory.h" namespace aria2 { -class MemoryBufferPreDownloadHandler:public PreDownloadHandler -{ -public: - MemoryBufferPreDownloadHandler(); - - virtual ~MemoryBufferPreDownloadHandler(); - - virtual void execute(RequestGroup* requestGroup); -}; +typedef MemoryPreDownloadHandler +MemoryBufferPreDownloadHandler; } // namespace aria2 diff --git a/src/MemoryPreDownloadHandler.h b/src/MemoryPreDownloadHandler.h new file mode 100644 index 000000000..c3fc8ca38 --- /dev/null +++ b/src/MemoryPreDownloadHandler.h @@ -0,0 +1,64 @@ +/* */ +#ifndef D_MEMORY_PRE_DOWNLOAD_HANDLER_H +#define D_MEMORY_PRE_DOWNLOAD_HANDLER_H + +#include "PreDownloadHandler.h" +#include "RequestGroup.h" + +namespace aria2 { + +template +class MemoryPreDownloadHandler:public PreDownloadHandler +{ +public: + MemoryPreDownloadHandler() {} + + virtual ~MemoryPreDownloadHandler() {} + + virtual void execute(RequestGroup* requestGroup) + { + SharedHandle dwf(new DiskWriterFactoryType()); + requestGroup->setDiskWriterFactory(dwf); + requestGroup->setFileAllocationEnabled(false); + requestGroup->setPreLocalFileCheckEnabled(false); + requestGroup->markInMemoryDownload(); + requestGroup->setNumConcurrentCommand(1); + } +}; + +} // namespace aria2 + +#endif // D_MEMORY_PRE_DOWNLOAD_HANDLER_H diff --git a/src/UTPexExtensionMessage.cc b/src/UTPexExtensionMessage.cc index 0e1446e0f..4b64be47d 100644 --- a/src/UTPexExtensionMessage.cc +++ b/src/UTPexExtensionMessage.cc @@ -191,7 +191,7 @@ UTPexExtensionMessage::create(const unsigned char* data, size_t len) } UTPexExtensionMessageHandle msg(new UTPexExtensionMessage(*data)); - SharedHandle decoded = bencode2::decode(data+1, data+len); + SharedHandle decoded = bencode2::decode(data+1, len - 1); const Dict* dict = downcast(decoded); if(dict) { const String* added = downcast(dict->get("added")); diff --git a/src/ValueBaseDiskWriter.h b/src/ValueBaseDiskWriter.h new file mode 100644 index 000000000..3f326c96a --- /dev/null +++ b/src/ValueBaseDiskWriter.h @@ -0,0 +1,111 @@ +/* */ +#ifndef D_VALUE_BASE_DISK_WRITER_H +#define D_VALUE_BASE_DISK_WRITER_H + +#include "DiskWriter.h" +#include "ValueBaseStructParserStateMachine.h" + +namespace aria2 { + +// DiskWriter backed with ValueBaseParser. The written bytes are +// consumed by ValueBaseParser. It is only capable of sequential write +// so offset argument in write() will be ignored. It also does not +// offer read(). +template +class ValueBaseDiskWriter : public DiskWriter { +public: + ValueBaseDiskWriter() + : parser_(&psm_) + {} + + virtual ~ValueBaseDiskWriter() + {} + + virtual void initAndOpenFile(int64_t totalLength = 0) + { + parser_.reset(); + } + + virtual void openFile(int64_t totalLength = 0) + { + initAndOpenFile(totalLength); + } + + virtual void closeFile() {} + + virtual void openExistingFile(int64_t totalLength = 0) + { + initAndOpenFile(totalLength); + } + + virtual int64_t size() + { + return 0; + } + + virtual void writeData(const unsigned char* data, size_t len, int64_t offset) + { + // Return value is ignored here but handled in finalize() + parser_.parseUpdate(reinterpret_cast(data), len); + } + + virtual ssize_t readData(unsigned char* data, size_t len, int64_t offset) + { + return 0; + } + + int finalize() + { + return parser_.parseFinal(0, 0); + } + + SharedHandle getResult() const + { + return psm_.getResult(); + } + + void reset() + { + parser_.reset(); + } +private: + ValueBaseStructParserStateMachine psm_; + ValueBaseParser parser_; +}; + +} // namespace aria2 + +#endif // D_VALUE_BASE_DISK_WRITER_H diff --git a/src/ValueBaseStructParserStateMachine.cc b/src/ValueBaseStructParserStateMachine.cc index 4a446a153..04d1033cf 100644 --- a/src/ValueBaseStructParserStateMachine.cc +++ b/src/ValueBaseStructParserStateMachine.cc @@ -65,8 +65,11 @@ NullValueBaseStructParserState* nullState = new NullValueBaseStructParserState(); } // namespace -const SharedHandle -ValueBaseStructParserStateMachine::noResult = ValueBase::none; +const SharedHandle& +ValueBaseStructParserStateMachine::noResult() +{ + return ValueBase::none; +} ValueBaseStructParserStateMachine::ValueBaseStructParserStateMachine() : ctrl_(new rpc::XmlRpcRequestParserController()) diff --git a/src/ValueBaseStructParserStateMachine.h b/src/ValueBaseStructParserStateMachine.h index 30143181e..5b5efa562 100644 --- a/src/ValueBaseStructParserStateMachine.h +++ b/src/ValueBaseStructParserStateMachine.h @@ -57,7 +57,7 @@ class ValueBaseStructParserState; class ValueBaseStructParserStateMachine : public StructParserStateMachine { public: typedef SharedHandle ResultType; - static const SharedHandle noResult; + static const SharedHandle& noResult(); struct NumberData { int64_t number; diff --git a/src/bencode2.cc b/src/bencode2.cc index 78071879c..2e5b2e6a1 100644 --- a/src/bencode2.cc +++ b/src/bencode2.cc @@ -41,229 +41,39 @@ #include "error_code.h" #include "BufferedFile.h" #include "util.h" +#include "ValueBaseBencodeParser.h" namespace aria2 { namespace bencode2 { -namespace { -template -std::pair, InputIterator> -decodeiter -(InputIterator first, - InputIterator last, - size_t depth); -} // namespace - -namespace { -template -std::pair, InputIterator> -decoderawstring(InputIterator first, InputIterator last) -{ - InputIterator i = first; - int32_t len; - for(; i != last && *i != ':'; ++i); - if(i == last || i == first || - !util::parseIntNoThrow(len, std::string(first, i)) || - len < 0) { - throw DL_ABORT_EX2("Bencode decoding failed:" - " A positive integer expected but none found.", - error_code::BENCODE_PARSE_ERROR); - } - ++i; - if(last-i < len) { - throw DL_ABORT_EX2 - (fmt("Bencode decoding failed:" - " Expected %d bytes of data, but only %d read.", - len, static_cast(last-i)), - error_code::BENCODE_PARSE_ERROR); - } - return std::make_pair(std::make_pair(i, i+len), i+len); -} -} // namespace - -namespace { -template -std::pair, InputIterator> -decodestring(InputIterator first, InputIterator last) -{ - std::pair, InputIterator> r = - decoderawstring(first, last); - return std::make_pair(String::g(r.first.first, r.first.second), r.second); -} -} // namespace - -namespace { -template -std::pair, InputIterator> -decodeinteger(InputIterator first, InputIterator last) -{ - InputIterator i = first; - for(; i != last && *i != 'e'; ++i); - Integer::ValueType iv; - if(i == last || !util::parseLLIntNoThrow(iv, std::string(first, i))) { - throw DL_ABORT_EX2("Bencode decoding failed:" - " Integer expected but none found", - error_code::BENCODE_PARSE_ERROR); - } - return std::make_pair(Integer::g(iv), ++i); -} -} // namespace - -namespace { -template -std::pair, InputIterator> -decodedict -(InputIterator first, - InputIterator last, - size_t depth) -{ - SharedHandle dict = Dict::g(); - while(first != last) { - if(*first == 'e') { - return std::make_pair(dict, ++first); - } else { - std::pair, InputIterator> keyp = - decoderawstring(first, last); - std::pair, InputIterator> r = - decodeiter(keyp.second, last, depth); - dict->put(std::string(keyp.first.first, keyp.first.second), r.first); - first = r.second; - } - } - throw DL_ABORT_EX2("Bencode decoding failed:" - " Unexpected EOF in dict context. 'e' expected.", - error_code::BENCODE_PARSE_ERROR); -} -} // namespace - -namespace { -template -std::pair, InputIterator> -decodelist -(InputIterator first, - InputIterator last, - size_t depth) -{ - SharedHandle list = List::g(); - while(first != last) { - if(*first == 'e') { - return std::make_pair(list, ++first); - } else { - std::pair, InputIterator> r = - decodeiter(first, last, depth); - list->append(r.first); - first = r.second; - } - } - throw DL_ABORT_EX2("Bencode decoding failed:" - " Unexpected EOF in list context. 'e' expected.", - error_code::BENCODE_PARSE_ERROR); -} -} // namespace - -namespace { -void checkDepth(size_t depth) -{ - if(depth >= MAX_STRUCTURE_DEPTH) { - throw DL_ABORT_EX2("Bencode decoding failed: Structure is too deep.", - error_code::BENCODE_PARSE_ERROR); - } -} -} // namespace - -namespace { -template -std::pair, InputIterator> -decodeiter -(InputIterator first, - InputIterator last, - size_t depth) -{ - checkDepth(depth); - if(first == last) { - throw DL_ABORT_EX2("Bencode decoding failed:" - " Unexpected EOF in term context." - " 'd', 'l', 'i' or digit is expected.", - error_code::BENCODE_PARSE_ERROR); - } - if(*first == 'd') { - return decodedict(++first, last, depth+1); - } else if(*first == 'l') { - return decodelist(++first, last, depth+1); - } else if(*first == 'i') { - return decodeinteger(++first, last); - } else { - return decodestring(first, last); - } -} -} // namespace - -namespace { -template -SharedHandle decodegen -(InputIterator first, - InputIterator last, - size_t& end) -{ - if(first == last) { - return SharedHandle(); - } - std::pair, InputIterator> p = - decodeiter(first, last, 0); - end = p.second-first; - return p.first; -} -} // namespace - -SharedHandle decode -(std::string::const_iterator first, - std::string::const_iterator last) +SharedHandle decode(const unsigned char* data, size_t len) { size_t end; - return decodegen(first, last, end); + return decode(data, len, end); } -SharedHandle decode -(std::string::const_iterator first, - std::string::const_iterator last, - size_t& end) -{ - return decodegen(first, last, end); -} - -SharedHandle decode -(const unsigned char* first, - const unsigned char* last) +SharedHandle decode(const std::string& data) { size_t end; - return decodegen(first, last, end); + return decode(reinterpret_cast(data.c_str()), + data.size(), end); } -SharedHandle decode -(const unsigned char* first, - const unsigned char* last, - size_t& end) +SharedHandle decode(const unsigned char* data, size_t len, + size_t& end) { - return decodegen(first, last, end); -} - -SharedHandle decodeFromFile(const std::string& filename) -{ - BufferedFile fp(filename, BufferedFile::READ); - if(fp) { - std::stringstream ss; - fp.transfer(ss); - fp.close(); - const std::string s = ss.str(); - size_t end; - return decodegen(s.begin(), s.end(), end); - } else { - throw DL_ABORT_EX2 - (fmt("Bencode decoding failed: Cannot open file '%s'.", - filename.c_str()), - error_code::BENCODE_PARSE_ERROR); + ssize_t error; + bittorrent::ValueBaseBencodeParser parser; + SharedHandle res = + parser.parseFinal(reinterpret_cast(data), len, error); + if(error < 0) { + throw DL_ABORT_EX2(fmt("Bencode decoding failed: error=%d", + static_cast(error)), + error_code::BENCODE_PARSE_ERROR); } + end = error; + return res; } std::string encode(const ValueBase* vlb) diff --git a/src/bencode2.h b/src/bencode2.h index bfc95de6d..7da4c2fdf 100644 --- a/src/bencode2.h +++ b/src/bencode2.h @@ -38,7 +38,6 @@ #include "common.h" #include -#include #include "ValueBase.h" @@ -46,28 +45,15 @@ namespace aria2 { namespace bencode2 { -const size_t MAX_STRUCTURE_DEPTH = 100; +// Decode the data whose length is len. +SharedHandle decode(const unsigned char* data, size_t len); -// Decode the data in [first, last). -SharedHandle decode -(std::string::const_iterator first, - std::string::const_iterator last); +SharedHandle decode(const std::string& data); -// Decode the data in [first, last). After decode is done +// Decode the data whose length is len. After decode is done // successfully, return the bencoded string length in end. -SharedHandle decode -(std::string::const_iterator first, - std::string::const_iterator last, - size_t& end); - -SharedHandle decode -(const unsigned char* first, - const unsigned char* last); - -SharedHandle decode -(const unsigned char* first, - const unsigned char* last, - size_t& end); +SharedHandle decode(const unsigned char* data, size_t len, + size_t& end); SharedHandle decodeFromFile(const std::string& filename); diff --git a/src/bittorrent_helper.cc b/src/bittorrent_helper.cc index b6fc89b70..ea7f1c5b2 100644 --- a/src/bittorrent_helper.cc +++ b/src/bittorrent_helper.cc @@ -61,6 +61,7 @@ #include "error_code.h" #include "array_fun.h" #include "DownloadFailureException.h" +#include "ValueBaseBencodeParser.h" namespace aria2 { @@ -523,8 +524,9 @@ void load(const std::string& torrentFile, const SharedHandle