diff --git a/Zip/include/Poco/Zip/AutoDetectStream.h b/Zip/include/Poco/Zip/AutoDetectStream.h index c89177c10..da14a74db 100644 --- a/Zip/include/Poco/Zip/AutoDetectStream.h +++ b/Zip/include/Poco/Zip/AutoDetectStream.h @@ -32,7 +32,7 @@ class Zip_API AutoDetectStreamBuf: public Poco::BufferedStreamBuf /// Data Descriptor signature. { public: - AutoDetectStreamBuf(std::istream& in, const std::string& prefix, const std::string& postfix, bool reposition, Poco::UInt32 start); + AutoDetectStreamBuf(std::istream& in, const std::string& prefix, const std::string& postfix, bool reposition, Poco::UInt32 start, bool needsZip64); /// Creates the AutoDetectStream. ~AutoDetectStreamBuf(); @@ -55,7 +55,8 @@ private: std::string _postfix; bool _reposition; Poco::UInt32 _start; - std::streamsize _length; + bool _needsZip64; + Poco::UInt64 _length; }; @@ -66,7 +67,7 @@ class Zip_API AutoDetectIOS: public virtual std::ios /// order of the stream buffer and base classes. { public: - AutoDetectIOS(std::istream& istr, const std::string& prefix, const std::string& postfix, bool reposition, Poco::UInt32 start); + AutoDetectIOS(std::istream& istr, const std::string& prefix, const std::string& postfix, bool reposition, Poco::UInt32 start, bool needsZip64); /// Creates the basic stream and connects it /// to the given input stream. @@ -86,7 +87,7 @@ class Zip_API AutoDetectInputStream: public AutoDetectIOS, public std::istream /// Data Descriptor signature. { public: - AutoDetectInputStream(std::istream& istr, const std::string& prefix = std::string(), const std::string& postfix = std::string(), bool reposition = false, Poco::UInt32 start = 0); + AutoDetectInputStream(std::istream& istr, const std::string& prefix = std::string(), const std::string& postfix = std::string(), bool reposition = false, Poco::UInt32 start = 0, bool needsZip64 = false); /// Creates the AutoDetectInputStream and connects it to the underlying stream. ~AutoDetectInputStream(); diff --git a/Zip/include/Poco/Zip/Compress.h b/Zip/include/Poco/Zip/Compress.h index 7f0397eeb..b71418d97 100644 --- a/Zip/include/Poco/Zip/Compress.h +++ b/Zip/include/Poco/Zip/Compress.h @@ -36,9 +36,11 @@ class Zip_API Compress public: Poco::FIFOEvent EDone; - Compress(std::ostream& out, bool seekableOut); + Compress(std::ostream& out, bool seekableOut, bool forceZip64 = false); /// seekableOut determines how we write the zip, setting it to true is recommended for local files (smaller zip file), /// if you are compressing directly to a network, you MUST set it to false + /// If forceZip64 is set true then the file header is allocated with zip64 extension so that it can be updated after the file data is written + /// if seekableOut is true in case the compressed or uncompressed size exceeds 32 bits. ~Compress(); @@ -104,14 +106,16 @@ private: /// copys an already compressed ZipEntry from in private: - std::set _storeExtensions; - std::ostream& _out; - bool _seekableOut; - ZipArchive::FileHeaders _files; - ZipArchive::FileInfos _infos; - ZipArchive::DirectoryInfos _dirs; - Poco::UInt32 _offset; - std::string _comment; + std::set _storeExtensions; + std::ostream& _out; + bool _seekableOut; + bool _forceZip64; + ZipArchive::FileHeaders _files; + ZipArchive::FileInfos _infos; + ZipArchive::DirectoryInfos _dirs; + ZipArchive::DirectoryInfos64 _dirs64; + Poco::UInt64 _offset; + std::string _comment; friend class Keep; friend class Rename; @@ -121,6 +125,8 @@ private: // // inlines // + + inline void Compress::setZipComment(const std::string& comment) { _comment = comment; diff --git a/Zip/include/Poco/Zip/Zip.h b/Zip/include/Poco/Zip/Zip.h index 7ce4e38a4..1f1d781f3 100644 --- a/Zip/include/Poco/Zip/Zip.h +++ b/Zip/include/Poco/Zip/Zip.h @@ -28,7 +28,7 @@ // from a DLL simpler. All files within this DLL are compiled with the Zip_EXPORTS // symbol defined on the command line. this symbol should not be defined on any project // that uses this DLL. This way any other project whose source files include this file see -// Zip_API functions as being imported from a DLL, wheras this DLL sees symbols +// Zip_API functions as being imported from a DLL, whereas this DLL sees symbols // defined with this macro as being exported. // #if defined(_WIN32) && defined(POCO_DLL) diff --git a/Zip/include/Poco/Zip/ZipArchive.h b/Zip/include/Poco/Zip/ZipArchive.h index e07812a40..8951d59e9 100644 --- a/Zip/include/Poco/Zip/ZipArchive.h +++ b/Zip/include/Poco/Zip/ZipArchive.h @@ -41,6 +41,7 @@ public: typedef std::map FileHeaders; typedef std::map FileInfos; typedef std::map DirectoryInfos; + typedef std::map DirectoryInfos64; ZipArchive(std::istream& in); /// Creates the ZipArchive from a file. Note that the in stream will be in state failed after the constructor is finished @@ -66,7 +67,7 @@ public: private: void parse(std::istream& in, ParseCallback& pc); - ZipArchive(const FileHeaders& entries, const FileInfos& infos, const DirectoryInfos& dirs ); + ZipArchive(const FileHeaders& entries, const FileInfos& infos, const DirectoryInfos& dirs, const DirectoryInfos64& dirs64 ); private: FileHeaders _entries; @@ -75,6 +76,8 @@ private: /// Info generated by parsing the directory block of the zip file DirectoryInfos _disks; /// Stores directory info for all found disks + DirectoryInfos64 _disks64; + /// Stores directory info for all found disks static const std::string EMPTY_COMMENT; diff --git a/Zip/include/Poco/Zip/ZipArchiveInfo.h b/Zip/include/Poco/Zip/ZipArchiveInfo.h index 2fa0c04e2..3605e6040 100644 --- a/Zip/include/Poco/Zip/ZipArchiveInfo.h +++ b/Zip/include/Poco/Zip/ZipArchiveInfo.h @@ -68,15 +68,19 @@ public: /// Sets the optional Zip comment. void setNumberOfEntries(Poco::UInt16 val); - /// Returns the number of entries on this disk + /// Sets the number of entries on this disk void setTotalNumberOfEntries(Poco::UInt16 val); - /// Returns the total number of entries on all disks + /// Sets the total number of entries on all disks void setCentralDirectorySize(Poco::UInt32 val); - /// Returns the size of the central directory in bytes + /// Sets the size of the central directory in bytes - void setHeaderOffset(Poco::UInt32 val); + void setCentralDirectoryOffset(Poco::UInt32 val); + /// Sets the offset of the central directory from beginning of first disk + + void setHeaderOffset(std::streamoff val); + /// Sets the offset of the header in relation to the begin of this disk std::string createHeader() const; /// Creates a header @@ -113,6 +117,122 @@ private: }; +class Zip_API ZipArchiveInfo64 + /// A ZipArchiveInfo64 stores central directory info +{ +public: + static const char HEADER[ZipCommon::HEADER_SIZE]; + static const char LOCATOR_HEADER[ZipCommon::HEADER_SIZE]; + + ZipArchiveInfo64(); + /// Default constructor, everything set to zero or empty + + ZipArchiveInfo64(std::istream& in, bool assumeHeaderRead); + /// Creates the ZipArchiveInfo64 by parsing the input stream. + /// If assumeHeaderRead is true we assume that the first 4 bytes were already read outside. + + ~ZipArchiveInfo64(); + /// Destroys the ZipArchiveInfo64. + + void getVersionMadeBy(int& major, int& minor); + /// The ZIP version used to create the file + + void getRequiredVersion(int& major, int& minor); + /// The minimum version required to extract the data + + Poco::UInt32 getDiskNumber() const; + /// Get the number of the disk where this header can be found + + Poco::UInt32 getFirstDiskForDirectoryHeader() const; + /// Returns the number of the disk that contains the start of the directory header + + Poco::UInt64 getNumberOfEntries() const; + /// Returns the number of entries on this disk + + Poco::UInt64 getTotalNumberOfEntries() const; + /// Returns the total number of entries on all disks + + Poco::UInt64 getCentralDirectorySize() const; + /// Returns the size of the central directory in bytes + + std::streamoff getCentralDirectoryOffset() const; + /// Returns the offset of the central directory from beginning of first disk + + std::streamoff getHeaderOffset() const; + /// Returns the offset of the header in relation to the begin of this disk + + void setNumberOfEntries(Poco::UInt64 val); + /// Sets the number of entries on this disk + + void setTotalNumberOfEntries(Poco::UInt64 val); + /// Sets the total number of entries on all disks + + void setCentralDirectorySize(Poco::UInt64 val); + /// Set the size of the central directory in bytes + + void setCentralDirectoryOffset(Poco::UInt64 val); + /// Returns the offset of the central directory from beginning of first disk + + void setHeaderOffset(std::streamoff val); + /// Sets the offset of the header in relation to the begin of this disk + + void setTotalNumberOfDisks(Poco::UInt32 val); + /// Sets the offset of the central directory from beginning of first disk + + std::string createHeader() const; + /// Creates a header + +private: + void parse(std::istream& inp, bool assumeHeaderRead); + void setRequiredVersion(int major, int minor); + +private: + enum + { + HEADER_POS = 0, + RECORDSIZE_POS = HEADER_POS + ZipCommon::HEADER_SIZE, + RECORDSIZE_SIZE = 8, + VERSIONMADEBY_POS = RECORDSIZE_POS + RECORDSIZE_SIZE, + VERSIONMADEBY_SIZE = 2, + VERSION_NEEDED_POS = VERSIONMADEBY_POS + VERSIONMADEBY_SIZE, + VERSION_NEEDED_SIZE = 2, + NUMBEROFTHISDISK_POS = VERSION_NEEDED_POS + VERSION_NEEDED_SIZE, + NUMBEROFTHISDISK_SIZE = 4, + NUMBEROFCENTRALDIRDISK_POS = NUMBEROFTHISDISK_POS + NUMBEROFTHISDISK_SIZE, + NUMBEROFCENTRALDIRDISK_SIZE = 4, + NUMENTRIESTHISDISK_POS = NUMBEROFCENTRALDIRDISK_POS + NUMBEROFCENTRALDIRDISK_SIZE, + NUMENTRIESTHISDISK_SIZE = 8, + TOTALNUMENTRIES_POS = NUMENTRIESTHISDISK_POS + NUMENTRIESTHISDISK_SIZE, + TOTALNUMENTRIES_SIZE = 8, + CENTRALDIRSIZE_POS = TOTALNUMENTRIES_POS + TOTALNUMENTRIES_SIZE, + CENTRALDIRSIZE_SIZE = 8, + CENTRALDIRSTARTOFFSET_POS = CENTRALDIRSIZE_POS + CENTRALDIRSIZE_SIZE, + CENTRALDIRSTARTOFFSET_SIZE = 8, + FULL_HEADER_SIZE = 56, + + LOCATOR_HEADER_POS = 0, + NUMBEROFENDOFCENTRALDIRDISK_POS = LOCATOR_HEADER_POS + ZipCommon::HEADER_SIZE, + NUMBEROFENDOFCENTRALDIRDISK_SIZE = 4, + ENDOFCENTRALDIROFFSET_POS = NUMBEROFENDOFCENTRALDIRDISK_POS + NUMBEROFENDOFCENTRALDIRDISK_SIZE, + ENDOFCENTRALDIROFFSET_SIZE = 8, + TOTALNUMBEROFENDDISKS_POS = ENDOFCENTRALDIROFFSET_POS + ENDOFCENTRALDIROFFSET_SIZE, + TOTALNUMBEROFENDDISKS_SIZE = 4, + + FULL_LOCATOR_SIZE = 20 + }; + + char _rawInfo[FULL_HEADER_SIZE]; + std::string _extraField; + char _locInfo[FULL_LOCATOR_SIZE]; + std::streamoff _startPos; +}; + + +// +// inlines +// + + inline Poco::UInt16 ZipArchiveInfo::getDiskNumber() const { return ZipUtil::get16BitValue(_rawInfo, NUMBEROFTHISDISK_POS); @@ -179,11 +299,105 @@ inline void ZipArchiveInfo::setCentralDirectorySize(Poco::UInt32 val) } -inline void ZipArchiveInfo::setHeaderOffset(Poco::UInt32 val) +inline void ZipArchiveInfo::setCentralDirectoryOffset(Poco::UInt32 val) { ZipUtil::set32BitValue(val, _rawInfo, CENTRALDIRSTARTOFFSET_POS); } +inline void ZipArchiveInfo::setHeaderOffset(std::streamoff val) +{ + _startPos = val; +} + + +inline Poco::UInt32 ZipArchiveInfo64::getDiskNumber() const +{ + return ZipUtil::get32BitValue(_rawInfo, NUMBEROFTHISDISK_POS); +} + + +inline Poco::UInt32 ZipArchiveInfo64::getFirstDiskForDirectoryHeader() const +{ + return ZipUtil::get32BitValue(_rawInfo, NUMBEROFCENTRALDIRDISK_POS); +} + + +inline Poco::UInt64 ZipArchiveInfo64::getNumberOfEntries() const +{ + return ZipUtil::get64BitValue(_rawInfo, NUMENTRIESTHISDISK_POS); +} + + +inline Poco::UInt64 ZipArchiveInfo64::getTotalNumberOfEntries() const +{ + return ZipUtil::get64BitValue(_rawInfo, TOTALNUMENTRIES_POS); +} + + +inline Poco::UInt64 ZipArchiveInfo64::getCentralDirectorySize() const +{ + return ZipUtil::get64BitValue(_rawInfo, CENTRALDIRSIZE_POS); +} + + +inline std::streamoff ZipArchiveInfo64::getCentralDirectoryOffset() const +{ + return _startPos; +} + + +inline std::streamoff ZipArchiveInfo64::getHeaderOffset() const +{ + return _startPos; +} + + +inline void ZipArchiveInfo64::setRequiredVersion(int major, int minor) +{ + poco_assert (minor < 10); + poco_assert (major < 24); + Poco::UInt8 val = static_cast(major)*10+static_cast(minor); + _rawInfo[VERSIONMADEBY_POS] = static_cast(val); + _rawInfo[VERSION_NEEDED_POS] = static_cast(val); +} + + +inline void ZipArchiveInfo64::setNumberOfEntries(Poco::UInt64 val) +{ + ZipUtil::set64BitValue(val, _rawInfo, NUMENTRIESTHISDISK_POS); +} + + +inline void ZipArchiveInfo64::setTotalNumberOfEntries(Poco::UInt64 val) +{ + ZipUtil::set64BitValue(val, _rawInfo, TOTALNUMENTRIES_POS); +} + + +inline void ZipArchiveInfo64::setCentralDirectorySize(Poco::UInt64 val) +{ + ZipUtil::set64BitValue(val, _rawInfo, CENTRALDIRSIZE_POS); +} + + +inline void ZipArchiveInfo64::setCentralDirectoryOffset(Poco::UInt64 val) +{ + ZipUtil::set64BitValue(val, _rawInfo, CENTRALDIRSTARTOFFSET_POS); +} + + +inline void ZipArchiveInfo64::setHeaderOffset(std::streamoff val) +{ + _startPos = val; + ZipUtil::set64BitValue(val, _locInfo, ENDOFCENTRALDIROFFSET_POS); +} + + +inline void ZipArchiveInfo64::setTotalNumberOfDisks(Poco::UInt32 val) +{ + ZipUtil::set32BitValue(val, _locInfo, TOTALNUMBEROFENDDISKS_POS); +} + } } // namespace Poco::Zip diff --git a/Zip/include/Poco/Zip/ZipCommon.h b/Zip/include/Poco/Zip/ZipCommon.h index 47c70ba8c..9fd45afa1 100644 --- a/Zip/include/Poco/Zip/ZipCommon.h +++ b/Zip/include/Poco/Zip/ZipCommon.h @@ -34,6 +34,10 @@ public: HEADER_SIZE = 4 }; + static const Poco::UInt16 ZIP64_EXTRA_ID = 0x1; // Extra data id tag for Zip64 data (in extension for ZipLocalFileHeader and ZipFileInfo) + static const Poco::UInt16 ZIP64_MAGIC_SHORT = 0xFFFF; + static const Poco::UInt32 ZIP64_MAGIC = 0xFFFFFFFF; + enum CompressionMethod { CM_STORE = 0, diff --git a/Zip/include/Poco/Zip/ZipDataInfo.h b/Zip/include/Poco/Zip/ZipDataInfo.h index 7eee188ec..1d412b1d9 100644 --- a/Zip/include/Poco/Zip/ZipDataInfo.h +++ b/Zip/include/Poco/Zip/ZipDataInfo.h @@ -78,6 +78,62 @@ private: }; +class Zip_API ZipDataInfo64 + /// A ZipDataInfo64 stores a Zip data descriptor for a Zip64 file +{ +public: + static const char HEADER[ZipCommon::HEADER_SIZE]; + + ZipDataInfo64(); + /// Creates a header with all fields (except the header field) set to 0 + + ZipDataInfo64(std::istream& in, bool assumeHeaderRead); + /// Creates the ZipDataInfo64. + + ~ZipDataInfo64(); + /// Destroys the ZipDataInfo64. + + bool isValid() const; + + Poco::UInt32 getCRC32() const; + + void setCRC32(Poco::UInt32 crc); + + Poco::UInt64 getCompressedSize() const; + + void setCompressedSize(Poco::UInt64 size); + + Poco::UInt64 getUncompressedSize() const; + + void setUncompressedSize(Poco::UInt64 size); + + static Poco::UInt32 getFullHeaderSize(); + + const char* getRawHeader() const; + +private: + enum + { + HEADER_POS = 0, + CRC32_POS = HEADER_POS + ZipCommon::HEADER_SIZE, + CRC32_SIZE = 4, + COMPRESSED_POS = CRC32_POS + CRC32_SIZE, + COMPRESSED_SIZE = 8, + UNCOMPRESSED_POS = COMPRESSED_POS + COMPRESSED_SIZE, + UNCOMPRESSED_SIZE = 8, + FULLHEADER_SIZE = UNCOMPRESSED_POS + UNCOMPRESSED_SIZE + }; + + char _rawInfo[FULLHEADER_SIZE]; + bool _valid; +}; + + +// +// inlines +// + + inline const char* ZipDataInfo::getRawHeader() const { return _rawInfo; @@ -132,6 +188,60 @@ inline Poco::UInt32 ZipDataInfo::getFullHeaderSize() } +inline const char* ZipDataInfo64::getRawHeader() const +{ + return _rawInfo; +} + + +inline bool ZipDataInfo64::isValid() const +{ + return _valid; +} + + +inline Poco::UInt32 ZipDataInfo64::getCRC32() const +{ + return ZipUtil::get32BitValue(_rawInfo, CRC32_POS); +} + + +inline void ZipDataInfo64::setCRC32(Poco::UInt32 crc) +{ + return ZipUtil::set32BitValue(crc, _rawInfo, CRC32_POS); +} + + +inline Poco::UInt64 ZipDataInfo64::getCompressedSize() const +{ + return ZipUtil::get64BitValue(_rawInfo, COMPRESSED_POS); +} + + +inline void ZipDataInfo64::setCompressedSize(Poco::UInt64 size) +{ + return ZipUtil::set64BitValue(size, _rawInfo, COMPRESSED_POS); +} + + +inline Poco::UInt64 ZipDataInfo64::getUncompressedSize() const +{ + return ZipUtil::get64BitValue(_rawInfo, UNCOMPRESSED_POS); +} + + +inline void ZipDataInfo64::setUncompressedSize(Poco::UInt64 size) +{ + return ZipUtil::set64BitValue(size, _rawInfo, UNCOMPRESSED_POS); +} + + +inline Poco::UInt32 ZipDataInfo64::getFullHeaderSize() +{ + return FULLHEADER_SIZE; +} + + } } // namespace Poco::Zip diff --git a/Zip/include/Poco/Zip/ZipFileInfo.h b/Zip/include/Poco/Zip/ZipFileInfo.h index 406c26665..eb106235c 100644 --- a/Zip/include/Poco/Zip/ZipFileInfo.h +++ b/Zip/include/Poco/Zip/ZipFileInfo.h @@ -46,9 +46,6 @@ public: ~ZipFileInfo(); /// Destroys the ZipFileInfo. - Poco::UInt32 getRelativeOffsetOfLocalHeader() const; - /// Where on the disk starts the localheader. Combined with the disk number gives the exact location of the header - ZipCommon::CompressionMethod getCompressionMethod() const; bool isEncrypted() const; @@ -60,9 +57,12 @@ public: Poco::UInt32 getHeaderSize() const; /// Returns the total size of the header including filename + other additional fields - Poco::UInt32 getCompressedSize() const; + Poco::UInt64 getCompressedSize() const; - Poco::UInt32 getUncompressedSize() const; + Poco::UInt64 getUncompressedSize() const; + + Poco::UInt64 getOffset() const; + /// Where on the disk starts the localheader. Combined with the disk number gives the exact location of the header const std::string& getFileName() const; @@ -92,14 +92,19 @@ public: std::string createHeader() const; - void setOffset(Poco::UInt32 val); + void setOffset(Poco::UInt64 val); + + bool needsZip64() const; + + void setZip64Data(); private: + void setCRC(Poco::UInt32 val); - void setCompressedSize(Poco::UInt32 val); + void setCompressedSize(Poco::UInt64 val); - void setUncompressedSize(Poco::UInt32 val); + void setUncompressedSize(Poco::UInt64 val); void setCompressionMethod(ZipCommon::CompressionMethod cm); @@ -129,6 +134,8 @@ private: Poco::UInt32 getUncompressedSizeFromHeader() const; + Poco::UInt32 getOffsetFromHeader() const; + Poco::UInt16 getFileNameLength() const; Poco::UInt16 getExtraFieldLength() const; @@ -175,7 +182,17 @@ private: EXTERNALFILE_ATTR_SIZE = 4, RELATIVEOFFSETLOCALHEADER_POS = EXTERNALFILE_ATTR_POS + EXTERNALFILE_ATTR_SIZE, RELATIVEOFFSETLOCALHEADER_SIZE = 4, - FULLHEADER_SIZE = 46 + FULLHEADER_SIZE = 46, + + EXTRA_DATA_TAG_SIZE = 2, + EXTRA_DATA_TAG_POS = 0, + EXTRA_DATA_SIZE_SIZE = 2, + EXTRA_DATA_SIZE_POS = EXTRA_DATA_TAG_POS + EXTRA_DATA_TAG_SIZE, + EXTRA_DATA_POS = EXTRA_DATA_SIZE_POS + EXTRA_DATA_SIZE_SIZE, + EXTRA_DATA_UNCOMPRESSED_SIZE_SIZE = 8, + EXTRA_DATA_COMPRESSED_SIZE_SIZE = 8, + EXTRA_DATA_OFFSET_SIZE = 8, + FULLEXTRA_DATA_SIZE = 28 }; enum @@ -186,8 +203,9 @@ private: char _rawInfo[FULLHEADER_SIZE]; Poco::UInt32 _crc32; - Poco::UInt32 _compressedSize; - Poco::UInt32 _uncompressedSize; + Poco::UInt64 _compressedSize; + Poco::UInt64 _uncompressedSize; + Poco::UInt64 _localHeaderOffset; std::string _fileName; Poco::DateTime _lastModifiedAt; std::string _extraField; @@ -195,12 +213,6 @@ private: }; -inline Poco::UInt32 ZipFileInfo::getRelativeOffsetOfLocalHeader() const -{ - return ZipUtil::get32BitValue(_rawInfo, RELATIVEOFFSETLOCALHEADER_POS); -} - - inline Poco::UInt32 ZipFileInfo::getCRCFromHeader() const { return ZipUtil::get32BitValue(_rawInfo, CRC32_POS); @@ -218,6 +230,11 @@ inline Poco::UInt32 ZipFileInfo::getUncompressedSizeFromHeader() const return ZipUtil::get32BitValue(_rawInfo, UNCOMPRESSED_SIZE_POS); } +inline Poco::UInt32 ZipFileInfo::getOffsetFromHeader() const +{ + return ZipUtil::get32BitValue(_rawInfo, RELATIVEOFFSETLOCALHEADER_POS); +} + inline void ZipFileInfo::parseDateTime() { @@ -244,19 +261,25 @@ inline const Poco::DateTime& ZipFileInfo::lastModifiedAt() const } +inline Poco::UInt64 ZipFileInfo::getOffset() const +{ + return _localHeaderOffset; +} + + inline Poco::UInt32 ZipFileInfo::getCRC() const { return _crc32; } -inline Poco::UInt32 ZipFileInfo::getCompressedSize() const +inline Poco::UInt64 ZipFileInfo::getCompressedSize() const { return _compressedSize; } -inline Poco::UInt32 ZipFileInfo::getUncompressedSize() const +inline Poco::UInt64 ZipFileInfo::getUncompressedSize() const { return _uncompressedSize; } @@ -361,6 +384,39 @@ inline Poco::UInt32 ZipFileInfo::getHeaderSize() const } +inline bool ZipFileInfo::needsZip64() const +{ + return _localHeaderOffset >= ZipCommon::ZIP64_MAGIC || _compressedSize >= ZipCommon::ZIP64_MAGIC || _uncompressedSize >= ZipCommon::ZIP64_MAGIC; +} + + +inline void ZipFileInfo::setZip64Data() +{ + if (needsZip64()) + { + setRequiredVersion(4, 5); + char data[FULLEXTRA_DATA_SIZE]; + ZipUtil::set16BitValue(ZipCommon::ZIP64_EXTRA_ID, data, EXTRA_DATA_TAG_POS); + Poco::UInt16 pos = EXTRA_DATA_POS; + if (_uncompressedSize >= ZipCommon::ZIP64_MAGIC) + { + ZipUtil::set64BitValue(_uncompressedSize, data, pos); pos += 8; + } + if (_compressedSize >= ZipCommon::ZIP64_MAGIC) + { + ZipUtil::set64BitValue(_compressedSize, data, pos); pos += 8; + } + if (_localHeaderOffset >= ZipCommon::ZIP64_MAGIC) + { + ZipUtil::set64BitValue(_localHeaderOffset, data, pos); pos += 8; + } + ZipUtil::set16BitValue(pos - EXTRA_DATA_POS, data, EXTRA_DATA_SIZE_POS); + _extraField = std::string(data, pos); + ZipUtil::set16BitValue(pos, _rawInfo, EXTRAFIELD_LENGTH_POS); + } +} + + inline void ZipFileInfo::setCRC(Poco::UInt32 val) { _crc32 = val; @@ -368,23 +424,24 @@ inline void ZipFileInfo::setCRC(Poco::UInt32 val) } -inline void ZipFileInfo::setOffset(Poco::UInt32 val) +inline void ZipFileInfo::setOffset(Poco::UInt64 val) { - ZipUtil::set32BitValue(val, _rawInfo, RELATIVEOFFSETLOCALHEADER_POS); + _localHeaderOffset = val; + ZipUtil::set32BitValue(val >= ZipCommon::ZIP64_MAGIC ? ZipCommon::ZIP64_MAGIC : static_cast(val), _rawInfo, RELATIVEOFFSETLOCALHEADER_POS); } -inline void ZipFileInfo::setCompressedSize(Poco::UInt32 val) +inline void ZipFileInfo::setCompressedSize(Poco::UInt64 val) { _compressedSize = val; - ZipUtil::set32BitValue(val, _rawInfo, COMPRESSED_SIZE_POS); + ZipUtil::set32BitValue(val >= ZipCommon::ZIP64_MAGIC ? ZipCommon::ZIP64_MAGIC : static_cast(val), _rawInfo, COMPRESSED_SIZE_POS); } -inline void ZipFileInfo::setUncompressedSize(Poco::UInt32 val) +inline void ZipFileInfo::setUncompressedSize(Poco::UInt64 val) { _uncompressedSize = val; - ZipUtil::set32BitValue(val, _rawInfo, UNCOMPRESSED_SIZE_POS); + ZipUtil::set32BitValue(val >= ZipCommon::ZIP64_MAGIC ? ZipCommon::ZIP64_MAGIC : static_cast(val), _rawInfo, UNCOMPRESSED_SIZE_POS); } @@ -421,7 +478,7 @@ inline void ZipFileInfo::setRequiredVersion(int major, int minor) { poco_assert (minor < 10); poco_assert (major < 24); - Poco::UInt8 val = static_cast(major)*10 + static_cast(minor); + Poco::UInt8 val = static_cast(major)*10+static_cast(minor); _rawInfo[VERSIONMADEBY_POS] = static_cast(val); _rawInfo[VERSION_NEEDED_POS] = static_cast(val); } diff --git a/Zip/include/Poco/Zip/ZipLocalFileHeader.h b/Zip/include/Poco/Zip/ZipLocalFileHeader.h index 86880e02b..ddcf19fcf 100644 --- a/Zip/include/Poco/Zip/ZipLocalFileHeader.h +++ b/Zip/include/Poco/Zip/ZipLocalFileHeader.h @@ -39,9 +39,10 @@ class Zip_API ZipLocalFileHeader public: static const char HEADER[ZipCommon::HEADER_SIZE]; - ZipLocalFileHeader(const Poco::Path& fileName, const Poco::DateTime& lastModifiedAt, ZipCommon::CompressionMethod cm, ZipCommon::CompressionLevel cl); + ZipLocalFileHeader(const Poco::Path& fileName, const Poco::DateTime& lastModifiedAt, ZipCommon::CompressionMethod cm, ZipCommon::CompressionLevel cl, bool forceZip64 = false); /// Creates a zip file header from an absoluteFile. fileName is the name of the file in the zip, outputIsSeekable determines if we write /// CRC and file sizes to the LocalFileHeader or after data compression into a ZipDataInfo + /// If forceZip64 is set true then the file header is allocated with zip64 extension. ZipLocalFileHeader(std::istream& inp, bool assumeHeaderRead, ParseCallback& callback); /// Creates the ZipLocalFileHeader by parsing the input stream. @@ -88,15 +89,15 @@ public: Poco::UInt32 getCRC() const; - Poco::UInt32 getCompressedSize() const; + Poco::UInt64 getCompressedSize() const; - Poco::UInt32 getUncompressedSize() const; + Poco::UInt64 getUncompressedSize() const; void setCRC(Poco::UInt32 val); - void setCompressedSize(Poco::UInt32 val); + void setCompressedSize(Poco::UInt64 val); - void setUncompressedSize(Poco::UInt32 val); + void setUncompressedSize(Poco::UInt64 val); const std::string& getFileName() const; @@ -116,6 +117,10 @@ public: void setFileName(const std::string& fileName, bool isDirectory); + bool needsZip64() const; + + void setZip64Data(); + std::string createHeader() const; /// Creates a header @@ -168,17 +173,27 @@ private: LASTMODEFILEDATE_POS = LASTMODEFILETIME_POS + LASTMODEFILETIME_SIZE, CRC32_SIZE = 4, CRC32_POS = LASTMODEFILEDATE_POS + LASTMODEFILEDATE_SIZE, - COMPRESSEDSIZE_SIZE = 4, - COMPRESSEDSIZE_POS = CRC32_POS + CRC32_SIZE, - UNCOMPRESSEDSIZE_SIZE = 4, - UNCOMPRESSEDSIZE_POS = COMPRESSEDSIZE_POS + COMPRESSEDSIZE_SIZE, - FILELENGTH_SIZE = 2, - FILELENGTH_POS = UNCOMPRESSEDSIZE_POS + UNCOMPRESSEDSIZE_SIZE, - EXTRAFIELD_LENGTH = 2, - EXTRAFIELD_POS = FILELENGTH_POS + FILELENGTH_SIZE, - FULLHEADER_SIZE = 30 + COMPRESSED_SIZE_SIZE = 4, + COMPRESSED_SIZE_POS = CRC32_POS + CRC32_SIZE, + UNCOMPRESSED_SIZE_SIZE = 4, + UNCOMPRESSED_SIZE_POS = COMPRESSED_SIZE_POS + COMPRESSED_SIZE_SIZE, + FILE_LENGTH_SIZE = 2, + FILE_LENGTH_POS = UNCOMPRESSED_SIZE_POS + UNCOMPRESSED_SIZE_SIZE, + EXTRA_FIELD_LENGTH = 2, + EXTRA_FIELD_POS = FILE_LENGTH_POS + FILE_LENGTH_SIZE, + FULLHEADER_SIZE = 30, + + EXTRA_DATA_TAG_SIZE = 2, + EXTRA_DATA_TAG_POS = 0, + EXTRA_DATA_SIZE_SIZE = 2, + EXTRA_DATA_SIZE_POS = EXTRA_DATA_TAG_POS + EXTRA_DATA_TAG_SIZE, + EXTRA_DATA_POS = EXTRA_DATA_SIZE_POS + EXTRA_DATA_SIZE_SIZE, + EXTRA_DATA_UNCOMPRESSED_SIZE_SIZE = 8, + EXTRA_DATA_COMPRESSED_SIZE_SIZE = 8, + FULLEXTRA_DATA_SIZE = 20 }; + bool _forceZip64; char _rawHeader[FULLHEADER_SIZE]; std::streamoff _startPos; std::streamoff _endPos; @@ -186,8 +201,8 @@ private: Poco::DateTime _lastModifiedAt; std::string _extraField; Poco::UInt32 _crc32; - Poco::UInt32 _compressedSize; - Poco::UInt32 _uncompressedSize; + Poco::UInt64 _compressedSize; + Poco::UInt64 _uncompressedSize; friend class ZipStreamBuf; }; @@ -195,13 +210,13 @@ private: inline void ZipLocalFileHeader::setFileNameLength(Poco::UInt16 size) { - ZipUtil::set16BitValue(size, _rawHeader, FILELENGTH_POS); + ZipUtil::set16BitValue(size, _rawHeader, FILE_LENGTH_POS); } inline void ZipLocalFileHeader::setExtraFieldSize(Poco::UInt16 size) { - ZipUtil::set16BitValue(size, _rawHeader, EXTRAFIELD_POS); + ZipUtil::set16BitValue(size, _rawHeader, EXTRA_FIELD_POS); } @@ -236,6 +251,28 @@ inline void ZipLocalFileHeader::getRequiredVersion(int& major, int& minor) } +inline bool ZipLocalFileHeader::needsZip64() const +{ + return _forceZip64 || _startPos >= ZipCommon::ZIP64_MAGIC || _compressedSize >= ZipCommon::ZIP64_MAGIC || _uncompressedSize >= ZipCommon::ZIP64_MAGIC; +} + + +inline void ZipLocalFileHeader::setZip64Data() +{ + setRequiredVersion(4, 5); + char data[FULLEXTRA_DATA_SIZE]; + ZipUtil::set16BitValue(ZipCommon::ZIP64_EXTRA_ID, data, EXTRA_DATA_TAG_POS); + Poco::UInt16 pos = EXTRA_DATA_POS; + ZipUtil::set64BitValue(_uncompressedSize, data, pos); pos += 8; + ZipUtil::set32BitValue(ZipCommon::ZIP64_MAGIC, _rawHeader, UNCOMPRESSED_SIZE_POS); + ZipUtil::set64BitValue(_compressedSize, data, pos); pos += 8; + ZipUtil::set32BitValue(ZipCommon::ZIP64_MAGIC, _rawHeader, COMPRESSED_SIZE_POS); + ZipUtil::set16BitValue(pos - EXTRA_DATA_POS, data, EXTRA_DATA_SIZE_POS); + _extraField = std::string(data, pos); + ZipUtil::set16BitValue(pos, _rawHeader, EXTRA_FIELD_POS); +} + + inline void ZipLocalFileHeader::setRequiredVersion(int major, int minor) { poco_assert (minor < 10); @@ -243,15 +280,16 @@ inline void ZipLocalFileHeader::setRequiredVersion(int major, int minor) _rawHeader[VERSION_POS] = static_cast(static_cast(major)*10+static_cast(minor)); } + inline Poco::UInt16 ZipLocalFileHeader::getFileNameLength() const { - return ZipUtil::get16BitValue(_rawHeader, FILELENGTH_POS); + return ZipUtil::get16BitValue(_rawHeader, FILE_LENGTH_POS); } inline Poco::UInt16 ZipLocalFileHeader::getExtraFieldLength() const { - return ZipUtil::get16BitValue(_rawHeader, EXTRAFIELD_POS); + return ZipUtil::get16BitValue(_rawHeader, EXTRA_FIELD_POS); } @@ -360,13 +398,13 @@ inline Poco::UInt32 ZipLocalFileHeader::getCRC() const } -inline Poco::UInt32 ZipLocalFileHeader::getCompressedSize() const +inline Poco::UInt64 ZipLocalFileHeader::getCompressedSize() const { return _compressedSize; } -inline Poco::UInt32 ZipLocalFileHeader::getUncompressedSize() const +inline Poco::UInt64 ZipLocalFileHeader::getUncompressedSize() const { return _uncompressedSize; } @@ -379,17 +417,17 @@ inline void ZipLocalFileHeader::setCRC(Poco::UInt32 val) } -inline void ZipLocalFileHeader::setCompressedSize(Poco::UInt32 val) +inline void ZipLocalFileHeader::setCompressedSize(Poco::UInt64 val) { _compressedSize = val; - ZipUtil::set32BitValue(val, _rawHeader, COMPRESSEDSIZE_POS); + ZipUtil::set32BitValue(val >= ZipCommon::ZIP64_MAGIC ? ZipCommon::ZIP64_MAGIC : static_cast(val), _rawHeader, COMPRESSED_SIZE_POS); } -inline void ZipLocalFileHeader::setUncompressedSize(Poco::UInt32 val) +inline void ZipLocalFileHeader::setUncompressedSize(Poco::UInt64 val) { _uncompressedSize = val; - ZipUtil::set32BitValue(val, _rawHeader, UNCOMPRESSEDSIZE_POS); + ZipUtil::set32BitValue(val >= ZipCommon::ZIP64_MAGIC ? ZipCommon::ZIP64_MAGIC : static_cast(val), _rawHeader, UNCOMPRESSED_SIZE_POS); } @@ -401,13 +439,13 @@ inline Poco::UInt32 ZipLocalFileHeader::getCRCFromHeader() const inline Poco::UInt32 ZipLocalFileHeader::getCompressedSizeFromHeader() const { - return ZipUtil::get32BitValue(_rawHeader, COMPRESSEDSIZE_POS); + return ZipUtil::get32BitValue(_rawHeader, COMPRESSED_SIZE_POS); } inline Poco::UInt32 ZipLocalFileHeader::getUncompressedSizeFromHeader() const { - return ZipUtil::get32BitValue(_rawHeader, UNCOMPRESSEDSIZE_POS); + return ZipUtil::get32BitValue(_rawHeader, UNCOMPRESSED_SIZE_POS); } diff --git a/Zip/include/Poco/Zip/ZipStream.h b/Zip/include/Poco/Zip/ZipStream.h index cdae9c1dd..bd26daa48 100644 --- a/Zip/include/Poco/Zip/ZipStream.h +++ b/Zip/include/Poco/Zip/ZipStream.h @@ -48,7 +48,7 @@ public: virtual ~ZipStreamBuf(); /// Destroys the ZipStreamBuf. - void close(); + void close(Poco::UInt64& extraDataSize); /// Informs a writing outputstream that writing is done for this stream bool crcValid() const; @@ -77,7 +77,7 @@ private: Poco::UInt32 _expectedCrc32; bool _checkCRC; /// Note: we do not check crc if we decompress a streaming zip file and the crc is stored in the directory header - Poco::UInt32 _bytesWritten; + Poco::UInt64 _bytesWritten; ZipLocalFileHeader* _pHeader; }; @@ -138,7 +138,7 @@ public: ~ZipOutputStream(); /// Destroys the ZipOutputStream. - void close(); + void close(Poco::UInt64& extraDataSize); /// Must be called for ZipOutputStreams! }; diff --git a/Zip/include/Poco/Zip/ZipUtil.h b/Zip/include/Poco/Zip/ZipUtil.h index d399dcb5e..0721237d9 100644 --- a/Zip/include/Poco/Zip/ZipUtil.h +++ b/Zip/include/Poco/Zip/ZipUtil.h @@ -29,7 +29,7 @@ namespace Poco { namespace Zip { -class ZipUtil +class Zip_API ZipUtil /// A utility class used for parsing header information inside of zip files { public: @@ -37,10 +37,14 @@ public: static Poco::UInt32 get32BitValue(const char* pVal, const Poco::UInt32 pos); + static Poco::UInt64 get64BitValue(const char* pVal, const Poco::UInt32 pos); + static void set16BitValue(const Poco::UInt16 val, char* pVal, const Poco::UInt32 pos); static void set32BitValue(const Poco::UInt32 val, char* pVal, const Poco::UInt32 pos); + static void set64BitValue(const Poco::UInt64 val, char* pVal, const Poco::UInt32 pos); + static Poco::DateTime parseDateTime(const char* pVal, const Poco::UInt32 timePos, const Poco::UInt32 datePos); static void setDateTime(const Poco::DateTime& dt, char* pVal, const Poco::UInt32 timePos, const Poco::UInt32 datePos); @@ -76,6 +80,14 @@ inline Poco::UInt32 ZipUtil::get32BitValue(const char* pVal, const Poco::UInt32 } +inline Poco::UInt64 ZipUtil::get64BitValue(const char* pVal, const Poco::UInt32 pos) +{ + Poco::UInt64 val = ZipUtil::get32BitValue(pVal, pos+4); + val = (val << 32) | ZipUtil::get32BitValue(pVal, pos); + return val; +} + + inline void ZipUtil::set16BitValue(const Poco::UInt16 val, char* pVal, const Poco::UInt32 pos) { pVal[pos] = static_cast(val); @@ -92,6 +104,19 @@ inline void ZipUtil::set32BitValue(const Poco::UInt32 val, char* pVal, const Poc } +inline void ZipUtil::set64BitValue(const Poco::UInt64 val, char* pVal, const Poco::UInt32 pos) +{ + pVal[pos] = static_cast(val); + pVal[pos+1] = static_cast(val>>8); + pVal[pos+2] = static_cast(val>>16); + pVal[pos+3] = static_cast(val>>24); + pVal[pos+4] = static_cast(val>>32); + pVal[pos+5] = static_cast(val>>40); + pVal[pos+6] = static_cast(val>>48); + pVal[pos+7] = static_cast(val>>56); +} + + } } // namespace Poco::Zip diff --git a/Zip/src/AutoDetectStream.cpp b/Zip/src/AutoDetectStream.cpp index 61c395d80..a14d10505 100644 --- a/Zip/src/AutoDetectStream.cpp +++ b/Zip/src/AutoDetectStream.cpp @@ -25,7 +25,7 @@ namespace Poco { namespace Zip { -AutoDetectStreamBuf::AutoDetectStreamBuf(std::istream& in, const std::string& pre, const std::string& post, bool reposition, Poco::UInt32 start): +AutoDetectStreamBuf::AutoDetectStreamBuf(std::istream& in, const std::string& pre, const std::string& post, bool reposition, Poco::UInt32 start, bool needsZip64): Poco::BufferedStreamBuf(STREAM_BUFFER_SIZE, std::ios::in), _pIstr(&in), _eofDetected(false), @@ -34,6 +34,7 @@ AutoDetectStreamBuf::AutoDetectStreamBuf(std::istream& in, const std::string& pr _postfix(post), _reposition(reposition), _start(start), + _needsZip64(needsZip64), _length(0) { } @@ -61,7 +62,7 @@ int AutoDetectStreamBuf::readFromDevice(char* buffer, std::streamsize length) std::streamsize n = (_prefix.size() > length) ? length : static_cast(_prefix.size()); std::memcpy(buffer, _prefix.data(), n); _prefix.erase(0, n); - return n; + return static_cast(n); } if (_eofDetected) @@ -71,7 +72,7 @@ int AutoDetectStreamBuf::readFromDevice(char* buffer, std::streamsize length) std::streamsize n = (_postfix.size() > length) ? length : static_cast(_postfix.size()); std::memcpy(buffer, _postfix.data(), n); _postfix.erase(0, n); - return n; + return static_cast(n); } else return -1; } @@ -81,7 +82,7 @@ int AutoDetectStreamBuf::readFromDevice(char* buffer, std::streamsize length) std::streamsize offset = 0; static std::istream::int_type eof = std::istream::traits_type::eof(); while (_pIstr->good() && !_pIstr->eof() && (offset + 4) < length) - { + { std::istream::int_type c = _pIstr->get(); if (c != eof) { @@ -112,37 +113,64 @@ int AutoDetectStreamBuf::readFromDevice(char* buffer, std::streamsize length) { if (ZipDataInfo::HEADER[3] == c) { - ZipDataInfo dataInfo(*_pIstr, true); - if (!_pIstr->good()) throw Poco::IOException("Failed to read data descriptor"); - - if (dataInfo.getCompressedSize() == _length + offset) + std::streamsize dataInfoSize = 0; + if (_needsZip64) { - _pIstr->seekg(-static_cast(dataInfo.getFullHeaderSize()), std::ios::cur); - if (!_pIstr->good()) throw Poco::IOException("Failed to seek on input stream"); + ZipDataInfo64 dataInfo(*_pIstr, true); + if (!_pIstr->good()) throw Poco::IOException("Failed to read data descriptor"); - _eofDetected = true; - _length += offset; - - if (offset == 0 && !_postfix.empty()) + dataInfoSize = dataInfo.getFullHeaderSize(); + if (dataInfo.getCompressedSize() == _length + offset) { - offset = (_postfix.size() > length) ? length : static_cast(_postfix.size()); - std::memcpy(buffer, _postfix.data(), offset); - _postfix.erase(0, offset); - } + _pIstr->seekg(-static_cast(dataInfoSize), std::ios::cur); + if (!_pIstr->good()) throw Poco::IOException("Failed to seek on input stream"); + + _eofDetected = true; + _length += offset; - return offset; + if (offset == 0 && !_postfix.empty()) + { + offset = (_postfix.size() > length) ? length : static_cast(_postfix.size()); + std::memcpy(buffer, _postfix.data(), offset); + _postfix.erase(0, offset); + } + + return static_cast(offset); + } } else { - _pIstr->seekg(-static_cast(dataInfo.getFullHeaderSize() - 4), std::ios::cur); - if (!_pIstr->good()) throw Poco::IOException("Failed to seek on input stream"); + ZipDataInfo dataInfo(*_pIstr, true); + if (!_pIstr->good()) throw Poco::IOException("Failed to read data descriptor"); - buffer[offset++] = ZipDataInfo::HEADER[0]; - buffer[offset++] = ZipDataInfo::HEADER[1]; - buffer[offset++] = ZipDataInfo::HEADER[2]; - buffer[offset++] = ZipDataInfo::HEADER[3]; - _matchCnt = 0; + dataInfoSize = dataInfo.getFullHeaderSize(); + if (dataInfo.getCompressedSize() == _length + offset) + { + _pIstr->seekg(-static_cast(dataInfoSize), std::ios::cur); + if (!_pIstr->good()) throw Poco::IOException("Failed to seek on input stream"); + + _eofDetected = true; + _length += offset; + + if (offset == 0 && !_postfix.empty()) + { + offset = (_postfix.size() > length) ? length : static_cast(_postfix.size()); + std::memcpy(buffer, _postfix.data(), offset); + _postfix.erase(0, offset); + } + + return static_cast(offset); + } } + + _pIstr->seekg(-static_cast(dataInfoSize - 4), std::ios::cur); + if (!_pIstr->good()) throw Poco::IOException("Failed to seek on input stream"); + + buffer[offset++] = ZipDataInfo::HEADER[0]; + buffer[offset++] = ZipDataInfo::HEADER[1]; + buffer[offset++] = ZipDataInfo::HEADER[2]; + buffer[offset++] = ZipDataInfo::HEADER[3]; + _matchCnt = 0; } else { @@ -150,14 +178,14 @@ int AutoDetectStreamBuf::readFromDevice(char* buffer, std::streamsize length) buffer[offset++] = ZipDataInfo::HEADER[1]; buffer[offset++] = ZipDataInfo::HEADER[2]; buffer[offset++] = c; - _matchCnt = 0; + _matchCnt = 0; } } } } _length += offset; - return offset; + return static_cast(offset); } @@ -168,8 +196,8 @@ int AutoDetectStreamBuf::writeToDevice(const char* buffer, std::streamsize lengt } -AutoDetectIOS::AutoDetectIOS(std::istream& istr, const std::string& pre, const std::string& post, bool reposition, Poco::UInt32 start): - _buf(istr, pre, post, reposition, start) +AutoDetectIOS::AutoDetectIOS(std::istream& istr, const std::string& pre, const std::string& post, bool reposition, Poco::UInt32 start, bool needsZip64): + _buf(istr, pre, post, reposition, start, needsZip64) { poco_ios_init(&_buf); } @@ -186,8 +214,8 @@ AutoDetectStreamBuf* AutoDetectIOS::rdbuf() } -AutoDetectInputStream::AutoDetectInputStream(std::istream& istr, const std::string& pre, const std::string& post, bool reposition, Poco::UInt32 start): - AutoDetectIOS(istr, pre, post, reposition, start), +AutoDetectInputStream::AutoDetectInputStream(std::istream& istr, const std::string& pre, const std::string& post, bool reposition, Poco::UInt32 start, bool needsZip64): + AutoDetectIOS(istr, pre, post, reposition, start, needsZip64), std::istream(&_buf) { } diff --git a/Zip/src/Compress.cpp b/Zip/src/Compress.cpp index 58c7eea97..7558bd9ca 100644 --- a/Zip/src/Compress.cpp +++ b/Zip/src/Compress.cpp @@ -3,12 +3,12 @@ // // Library: Zip // Package: Zip -// Module: Compress +// Module: Compress // // Copyright (c) 2007, Applied Informatics Software Engineering GmbH. // and Contributors. // -// SPDX-License-Identifier: BSL-1.0 +// SPDX-License-Identifier: BSL-1.0 // @@ -28,9 +28,10 @@ namespace Poco { namespace Zip { -Compress::Compress(std::ostream& out, bool seekableOut): +Compress::Compress(std::ostream& out, bool seekableOut, bool forceZip64): _out(out), _seekableOut(seekableOut), + _forceZip64(forceZip64), _files(), _infos(), _dirs(), @@ -66,8 +67,6 @@ void Compress::addEntry(std::istream& in, const Poco::DateTime& lastModifiedAt, std::string fn = ZipUtil::validZipEntryFileName(fileName); - if (_files.size() >= 65535) - throw ZipException("Maximum number of entries for a ZIP file reached: 65535"); if (!in.good()) throw ZipException("Invalid input stream"); @@ -83,7 +82,7 @@ void Compress::addEntry(std::istream& in, const Poco::DateTime& lastModifiedAt, } std::streamoff localHeaderOffset = _offset; - ZipLocalFileHeader hdr(fileName, lastModifiedAt, cm, cl); + ZipLocalFileHeader hdr(fileName, lastModifiedAt, cm, cl, _forceZip64); hdr.setStartPos(localHeaderOffset); ZipOutputStream zipOut(_out, hdr, _seekableOut); @@ -92,15 +91,15 @@ void Compress::addEntry(std::istream& in, const Poco::DateTime& lastModifiedAt, zipOut.put(static_cast(firstChar)); Poco::StreamCopier::copyStream(in, zipOut); } - zipOut.close(); - hdr.setStartPos(localHeaderOffset); // reset again now that compressed Size is known + Poco::UInt64 extraDataSize; + zipOut.close(extraDataSize); _offset = hdr.getEndPos(); - if (hdr.searchCRCAndSizesAfterData()) - _offset += ZipDataInfo::getFullHeaderSize(); + _offset += extraDataSize; _files.insert(std::make_pair(fileName.toString(Poco::Path::PATH_UNIX), hdr)); if (!_out) throw Poco::IOException("Bad output stream"); ZipFileInfo nfo(hdr); nfo.setOffset(localHeaderOffset); + nfo.setZip64Data(); _infos.insert(std::make_pair(fileName.toString(Poco::Path::PATH_UNIX), nfo)); EDone.notify(this, hdr); } @@ -108,30 +107,30 @@ void Compress::addEntry(std::istream& in, const Poco::DateTime& lastModifiedAt, void Compress::addFileRaw(std::istream& in, const ZipLocalFileHeader& h, const Poco::Path& fileName) { + if (!in.good()) + throw ZipException("Invalid input stream"); + std::string fn = ZipUtil::validZipEntryFileName(fileName); //bypass the header of the input stream and point to the first byte of the data payload in.seekg(h.getDataStartPos(), std::ios_base::beg); if (!in.good()) throw Poco::IOException("Failed to seek on input stream"); - if (_files.size() >= 65535) - throw ZipException("Maximum number of entries for a ZIP file reached: 65535"); - if (!in.good()) - throw ZipException("Invalid input stream"); - std::streamoff localHeaderOffset = _offset; ZipLocalFileHeader hdr(h); hdr.setFileName(fn, h.isDirectory()); hdr.setStartPos(localHeaderOffset); + if (hdr.needsZip64()) + hdr.setZip64Data(); //bypass zipoutputstream //write the header directly std::string header = hdr.createHeader(); _out.write(header.c_str(), static_cast(header.size())); // now fwd the payload to _out in chunks of size CHUNKSIZE - Poco::UInt32 totalSize = hdr.getCompressedSize(); + Poco::UInt64 totalSize = hdr.getCompressedSize(); if (totalSize > 0) { Poco::Buffer buffer(COMPRESS_CHUNK_SIZE); - Poco::UInt32 remaining = totalSize; + Poco::UInt64 remaining = totalSize; while (remaining > 0) { if (remaining > COMPRESS_CHUNK_SIZE) @@ -152,20 +151,39 @@ void Compress::addFileRaw(std::istream& in, const ZipLocalFileHeader& h, const P } } } + hdr.setStartPos(localHeaderOffset); // This resets EndPos now that compressed Size is known + _offset = hdr.getEndPos(); //write optional block afterwards if (hdr.searchCRCAndSizesAfterData()) { - ZipDataInfo info(in, false); - _out.write(info.getRawHeader(), static_cast(info.getFullHeaderSize())); + if (hdr.needsZip64()) + { + ZipDataInfo64 info(in, false); + _out.write(info.getRawHeader(), static_cast(info.getFullHeaderSize())); + _offset += ZipDataInfo::getFullHeaderSize(); + } + else + { + ZipDataInfo info(in, false); + _out.write(info.getRawHeader(), static_cast(info.getFullHeaderSize())); + _offset += ZipDataInfo::getFullHeaderSize(); + } } - hdr.setStartPos(localHeaderOffset); // reset again now that compressed Size is known - _offset = hdr.getEndPos(); - if (hdr.searchCRCAndSizesAfterData()) - _offset += ZipDataInfo::getFullHeaderSize(); + else + { + if (hdr.hasExtraField()) // Update sizes in header extension. + hdr.setZip64Data(); + _out.seekp(hdr.getStartPos(), std::ios_base::beg); + std::string header = hdr.createHeader(); + _out.write(header.c_str(), static_cast(header.size())); + _out.seekp(0, std::ios_base::end); + } + _files.insert(std::make_pair(fileName.toString(Poco::Path::PATH_UNIX), hdr)); if (!_out) throw Poco::IOException("Bad output stream"); ZipFileInfo nfo(hdr); nfo.setOffset(localHeaderOffset); + nfo.setZip64Data(); _infos.insert(std::make_pair(fileName.toString(Poco::Path::PATH_UNIX), nfo)); EDone.notify(this, hdr); } @@ -205,8 +223,6 @@ void Compress::addDirectory(const Poco::Path& entryName, const Poco::DateTime& l std::string fileStr = entryName.toString(Poco::Path::PATH_UNIX); if (_files.find(fileStr) != _files.end()) return; // ignore duplicate add - if (_files.size() >= 65535) - throw ZipException("Maximum number of entries for a ZIP file reached: 65535"); if (fileStr == "/") throw ZipException("Illegal entry name /"); if (fileStr.empty()) @@ -225,15 +241,17 @@ void Compress::addDirectory(const Poco::Path& entryName, const Poco::DateTime& l ZipLocalFileHeader hdr(entryName, lastModifiedAt, cm, cl); hdr.setStartPos(localHeaderOffset); ZipOutputStream zipOut(_out, hdr, _seekableOut); - zipOut.close(); + Poco::UInt64 extraDataSize; + zipOut.close(extraDataSize); hdr.setStartPos(localHeaderOffset); // reset again now that compressed Size is known _offset = hdr.getEndPos(); if (hdr.searchCRCAndSizesAfterData()) - _offset += ZipDataInfo::getFullHeaderSize(); + _offset += extraDataSize; _files.insert(std::make_pair(entryName.toString(Poco::Path::PATH_UNIX), hdr)); if (!_out) throw Poco::IOException("Bad output stream"); ZipFileInfo nfo(hdr); nfo.setOffset(localHeaderOffset); + nfo.setZip64Data(); _infos.insert(std::make_pair(entryName.toString(Poco::Path::PATH_UNIX), nfo)); EDone.notify(this, hdr); } @@ -293,33 +311,58 @@ void Compress::addRecursive(const Poco::Path& entry, ZipCommon::CompressionMetho ZipArchive Compress::close() { - if (!_dirs.empty()) - return ZipArchive(_files, _infos, _dirs); + if (!_dirs.empty() || ! _dirs64.empty()) + return ZipArchive(_files, _infos, _dirs, _dirs64); poco_assert (_infos.size() == _files.size()); - poco_assert (_files.size() < 65536); - Poco::UInt32 centralDirStart = _offset; - Poco::UInt32 centralDirSize = 0; + Poco::UInt64 centralDirSize64 = 0; + Poco::UInt64 centralDirStart64 = _offset; // write all infos ZipArchive::FileInfos::const_iterator it = _infos.begin(); ZipArchive::FileInfos::const_iterator itEnd = _infos.end(); + bool needZip64 = _forceZip64; + needZip64 = needZip64 || _files.size() >= ZipCommon::ZIP64_MAGIC_SHORT || centralDirStart64 >= ZipCommon::ZIP64_MAGIC; for (; it != itEnd; ++it) { const ZipFileInfo& nfo = it->second; + needZip64 = needZip64 || nfo.needsZip64(); + std::string info(nfo.createHeader()); _out.write(info.c_str(), static_cast(info.size())); Poco::UInt32 entrySize = static_cast(info.size()); - centralDirSize += entrySize; + centralDirSize64 += entrySize; _offset += entrySize; } if (!_out) throw Poco::IOException("Bad output stream"); - Poco::UInt16 numEntries = static_cast(_infos.size()); + Poco::UInt64 numEntries64 = _infos.size(); + needZip64 = needZip64 || _offset >= ZipCommon::ZIP64_MAGIC; + if (needZip64) + { + ZipArchiveInfo64 central; + central.setCentralDirectorySize(centralDirSize64); + central.setCentralDirectoryOffset(centralDirStart64); + central.setNumberOfEntries(numEntries64); + central.setTotalNumberOfEntries(numEntries64); + central.setHeaderOffset(_offset); + central.setTotalNumberOfDisks(1); + std::string centr(central.createHeader()); + _out.write(centr.c_str(), static_cast(centr.size())); + _out.flush(); + _offset += centr.size(); + _dirs64.insert(std::make_pair(0, central)); + } + + Poco::UInt16 numEntries = numEntries64 >= ZipCommon::ZIP64_MAGIC_SHORT ? ZipCommon::ZIP64_MAGIC_SHORT : static_cast(numEntries64); + Poco::UInt32 centralDirStart = centralDirStart64 >= ZipCommon::ZIP64_MAGIC ? ZipCommon::ZIP64_MAGIC : static_cast(centralDirStart64); + Poco::UInt32 centralDirSize = centralDirSize64 >= ZipCommon::ZIP64_MAGIC ? ZipCommon::ZIP64_MAGIC : static_cast(centralDirSize64); + Poco::UInt32 offset = _offset >= ZipCommon::ZIP64_MAGIC ? ZipCommon::ZIP64_MAGIC : static_cast(_offset); ZipArchiveInfo central; central.setCentralDirectorySize(centralDirSize); + central.setCentralDirectoryOffset(centralDirStart); central.setNumberOfEntries(numEntries); central.setTotalNumberOfEntries(numEntries); - central.setHeaderOffset(centralDirStart); + central.setHeaderOffset(offset); if (!_comment.empty() && _comment.size() <= 65535) { central.setZipComment(_comment); @@ -327,8 +370,9 @@ ZipArchive Compress::close() std::string centr(central.createHeader()); _out.write(centr.c_str(), static_cast(centr.size())); _out.flush(); + _offset += centr.size(); _dirs.insert(std::make_pair(0, central)); - return ZipArchive(_files, _infos, _dirs); + return ZipArchive(_files, _infos, _dirs, _dirs64); } diff --git a/Zip/src/PartialStream.cpp b/Zip/src/PartialStream.cpp index 7721912ba..c0282537d 100644 --- a/Zip/src/PartialStream.cpp +++ b/Zip/src/PartialStream.cpp @@ -74,7 +74,7 @@ int PartialStreamBuf::readFromDevice(char* buffer, std::streamsize length) std::streamsize tmp = (_prefix.size() > length)? length: static_cast(_prefix.size()); std::memcpy(buffer, _prefix.c_str(), tmp); _prefix = _prefix.substr(tmp); - return tmp; + return static_cast(tmp); } if (_numBytes == 0) @@ -84,7 +84,7 @@ int PartialStreamBuf::readFromDevice(char* buffer, std::streamsize length) std::streamsize tmp = (_postfix.size() > length)? length: static_cast(_postfix.size()); std::memcpy(buffer, _postfix.c_str(), tmp); _postfix = _postfix.substr(tmp); - return tmp; + return static_cast(tmp); } else return -1; @@ -99,7 +99,7 @@ int PartialStreamBuf::readFromDevice(char* buffer, std::streamsize length) _pIstr->read(buffer, length); std::streamsize bytesRead = _pIstr->gcount(); _numBytes -= bytesRead; - return bytesRead; + return static_cast(bytesRead); } @@ -121,7 +121,7 @@ int PartialStreamBuf::writeToDevice(const char* buffer, std::streamsize length) { _ignoreStart -= length; // fake return values - return length; + return static_cast(length); } else { @@ -136,10 +136,10 @@ int PartialStreamBuf::writeToDevice(const char* buffer, std::streamsize length) cnt += static_cast(_ignoreStart); _ignoreStart = 0; poco_assert (cnt < length); - _bufferOffset = length - cnt; + _bufferOffset = static_cast(length - cnt); std::memcpy(_buffer.begin(), buffer + cnt, _bufferOffset); - return length; + return static_cast(length); } } if (_buffer.size() > 0) @@ -148,7 +148,8 @@ int PartialStreamBuf::writeToDevice(const char* buffer, std::streamsize length) // thus first fill the buffer with the last n bytes of the msg // how much of the already cached data do we need to write? - Poco::Int32 cache = _bufferOffset + length - _buffer.size(); + Poco::Int32 cache = static_cast(_bufferOffset + + static_cast(length) - static_cast(_buffer.size())); if (cache > 0) { if (cache > _bufferOffset) @@ -184,7 +185,7 @@ int PartialStreamBuf::writeToDevice(const char* buffer, std::streamsize length) } if (_pOstr->good()) - return length; + return static_cast(length); throw Poco::IOException("Failed to write to output stream"); } @@ -192,7 +193,7 @@ int PartialStreamBuf::writeToDevice(const char* buffer, std::streamsize length) void PartialStreamBuf::close() { - // DONT write data from _buffer! + // DON'T write data from _buffer! } @@ -219,8 +220,8 @@ PartialStreamBuf* PartialIOS::rdbuf() } -PartialInputStream::PartialInputStream(std::istream& istr, std::ios::pos_type start, std::ios::pos_type end, bool initStream, const std::string& pre, const std::string& post): - PartialIOS(istr, start, end, pre, post, initStream), +PartialInputStream::PartialInputStream(std::istream& istr, std::ios::pos_type start, std::ios::pos_type end, bool initStream, const std::string& pre, const std::string& post): + PartialIOS(istr, start, end, pre, post, initStream), std::istream(&_buf) { } @@ -232,7 +233,7 @@ PartialInputStream::~PartialInputStream() PartialOutputStream::PartialOutputStream(std::ostream& ostr, std::size_t start, std::size_t end, bool initStream): - PartialIOS(ostr, start, end, initStream), + PartialIOS(ostr, start, end, initStream), std::ostream(&_buf) { } diff --git a/Zip/src/ZipArchive.cpp b/Zip/src/ZipArchive.cpp index 7611ba02a..f8c6db1a8 100644 --- a/Zip/src/ZipArchive.cpp +++ b/Zip/src/ZipArchive.cpp @@ -3,12 +3,12 @@ // // Library: Zip // Package: Zip -// Module: ZipArchive +// Module: ZipArchive // // Copyright (c) 2007, Applied Informatics Software Engineering GmbH. // and Contributors. // -// SPDX-License-Identifier: BSL-1.0 +// SPDX-License-Identifier: BSL-1.0 // @@ -28,7 +28,8 @@ const std::string ZipArchive::EMPTY_COMMENT; ZipArchive::ZipArchive(std::istream& in): _entries(), _infos(), - _disks() + _disks(), + _disks64() { poco_assert_dbg (in); SkipCallback skip; @@ -36,10 +37,11 @@ ZipArchive::ZipArchive(std::istream& in): } -ZipArchive::ZipArchive(const FileHeaders& entries, const FileInfos& infos, const DirectoryInfos& dirs): +ZipArchive::ZipArchive(const FileHeaders& entries, const FileInfos& infos, const DirectoryInfos& dirs, const DirectoryInfos64& dirs64): _entries(entries), _infos(infos), - _disks(dirs) + _disks(dirs), + _disks64(dirs64) { } @@ -47,7 +49,8 @@ ZipArchive::ZipArchive(const FileHeaders& entries, const FileInfos& infos, const ZipArchive::ZipArchive(std::istream& in, ParseCallback& pc): _entries(), _infos(), - _disks() + _disks(), + _disks64() { poco_assert_dbg (in); parse(in, pc); @@ -79,7 +82,7 @@ void ZipArchive::parse(std::istream& in, ParseCallback& pc) FileHeaders::iterator it = _entries.find(info.getFileName()); if (it != _entries.end()) { - it->second.setStartPos(info.getRelativeOffsetOfLocalHeader()); + it->second.setStartPos(info.getOffset()); } poco_assert (_infos.insert(std::make_pair(info.getFileName(), info)).second); } @@ -88,9 +91,14 @@ void ZipArchive::parse(std::istream& in, ParseCallback& pc) ZipArchiveInfo nfo(in, true); poco_assert (_disks.insert(std::make_pair(nfo.getDiskNumber(), nfo)).second); } + else if (std::memcmp(header, ZipArchiveInfo64::HEADER, ZipCommon::HEADER_SIZE) == 0) + { + ZipArchiveInfo64 nfo(in, true); + poco_assert (_disks64.insert(std::make_pair(nfo.getDiskNumber(), nfo)).second); + } else { - if (_disks.empty()) + if (_disks.empty() && _disks64.empty()) throw Poco::IllegalStateException("Illegal header in zip file"); else throw Poco::IllegalStateException("Garbage after directory header"); @@ -104,9 +112,17 @@ const std::string& ZipArchive::getZipComment() const // It seems that only the "first" disk is populated (look at Compress::close()), so getting the first ZipArchiveInfo DirectoryInfos::const_iterator it = _disks.begin(); if (it != _disks.end()) + { return it->second.getZipComment(); + } else - return EMPTY_COMMENT; + { + DirectoryInfos64::const_iterator it64 = _disks64.begin(); + if (it64 != _disks64.end()) + return it->second.getZipComment(); + else + return EMPTY_COMMENT; + } } diff --git a/Zip/src/ZipArchiveInfo.cpp b/Zip/src/ZipArchiveInfo.cpp index 1b152301c..9523ca2fd 100644 --- a/Zip/src/ZipArchiveInfo.cpp +++ b/Zip/src/ZipArchiveInfo.cpp @@ -36,6 +36,7 @@ ZipArchiveInfo::ZipArchiveInfo(std::istream& in, bool assumeHeaderRead): parse(in, assumeHeaderRead); } + ZipArchiveInfo::ZipArchiveInfo(): _rawInfo(), _startPos(0), @@ -100,4 +101,90 @@ void ZipArchiveInfo::setZipComment(const std::string& comment) } +const char ZipArchiveInfo64::HEADER[ZipCommon::HEADER_SIZE] = {'\x50', '\x4b', '\x06', '\x06'}; +const char ZipArchiveInfo64::LOCATOR_HEADER[ZipCommon::HEADER_SIZE] = {'\x50', '\x4b', '\x06', '\x07'}; + + +ZipArchiveInfo64::ZipArchiveInfo64(std::istream& in, bool assumeHeaderRead): + _rawInfo(), + _startPos(in.tellg()) +{ + if (assumeHeaderRead) + _startPos -= ZipCommon::HEADER_SIZE; + parse(in, assumeHeaderRead); +} + + +ZipArchiveInfo64::ZipArchiveInfo64(): + _rawInfo(), + _startPos(0) +{ + std::memset(_rawInfo, 0, FULL_HEADER_SIZE); + std::memcpy(_rawInfo, HEADER, ZipCommon::HEADER_SIZE); + ZipUtil::set64BitValue(FULL_HEADER_SIZE - (RECORDSIZE_POS + RECORDSIZE_SIZE), _rawInfo, RECORDSIZE_POS); + std::memset(_locInfo, 0, FULL_LOCATOR_SIZE); + std::memcpy(_locInfo, LOCATOR_HEADER, ZipCommon::HEADER_SIZE); + setRequiredVersion(4, 5); +} + + +ZipArchiveInfo64::~ZipArchiveInfo64() +{ +} + + +void ZipArchiveInfo64::parse(std::istream& inp, bool assumeHeaderRead) +{ + if (!assumeHeaderRead) + { + inp.read(_rawInfo, ZipCommon::HEADER_SIZE); + if (inp.gcount() != ZipCommon::HEADER_SIZE) + throw Poco::IOException("Failed to read archive info header"); + if (std::memcmp(_rawInfo, HEADER, ZipCommon::HEADER_SIZE) != 0) + throw Poco::DataFormatException("Bad archive info header"); + } + else + { + std::memcpy(_rawInfo, HEADER, ZipCommon::HEADER_SIZE); + } + + std::memset(_rawInfo + ZipCommon::HEADER_SIZE, 0, FULL_HEADER_SIZE - ZipCommon::HEADER_SIZE); + + // read the rest of the header + Poco::UInt64 offset = RECORDSIZE_POS; + inp.read(_rawInfo + ZipCommon::HEADER_SIZE, RECORDSIZE_SIZE); + offset += RECORDSIZE_SIZE; + Poco::UInt64 len = ZipUtil::get64BitValue(_rawInfo, RECORDSIZE_POS); + if (len <= FULL_HEADER_SIZE - offset) + { + inp.read(_rawInfo + offset, len); + ZipUtil::set64BitValue(FULL_HEADER_SIZE - offset, _rawInfo, RECORDSIZE_POS); + } + else + { + inp.read(_rawInfo + offset, FULL_HEADER_SIZE - offset); + len -= (FULL_HEADER_SIZE - offset); + Poco::Buffer xtra(len); + inp.read(xtra.begin(), len); + _extraField = std::string(xtra.begin(), len); + ZipUtil::set64BitValue(FULL_HEADER_SIZE + len - offset, _rawInfo, RECORDSIZE_POS); + } + inp.read(_locInfo, FULL_LOCATOR_SIZE); + if (inp.gcount() != FULL_LOCATOR_SIZE) + throw Poco::IOException("Failed to read locator"); + if (std::memcmp(_locInfo, LOCATOR_HEADER, ZipCommon::HEADER_SIZE) != 0) + throw Poco::DataFormatException("Bad locator header"); + +} + + +std::string ZipArchiveInfo64::createHeader() const +{ + std::string result(_rawInfo, FULL_HEADER_SIZE); + result.append(_extraField); + result.append(_locInfo, FULL_LOCATOR_SIZE); + return result; +} + + } } // namespace Poco::Zip diff --git a/Zip/src/ZipDataInfo.cpp b/Zip/src/ZipDataInfo.cpp index bb03acd9c..3a8522b37 100644 --- a/Zip/src/ZipDataInfo.cpp +++ b/Zip/src/ZipDataInfo.cpp @@ -62,4 +62,45 @@ ZipDataInfo::~ZipDataInfo() } +const char ZipDataInfo64::HEADER[ZipCommon::HEADER_SIZE] = {'\x50', '\x4b', '\x07', '\x08'}; + + +ZipDataInfo64::ZipDataInfo64(): + _rawInfo(), + _valid(true) +{ + std::memcpy(_rawInfo, HEADER, ZipCommon::HEADER_SIZE); + std::memset(_rawInfo+ZipCommon::HEADER_SIZE, 0, FULLHEADER_SIZE - ZipCommon::HEADER_SIZE); + _valid = true; +} + + +ZipDataInfo64::ZipDataInfo64(std::istream& in, bool assumeHeaderRead): + _rawInfo(), + _valid(false) +{ + if (assumeHeaderRead) + { + std::memcpy(_rawInfo, HEADER, ZipCommon::HEADER_SIZE); + } + else + { + in.read(_rawInfo, ZipCommon::HEADER_SIZE); + if (in.gcount() != ZipCommon::HEADER_SIZE) + throw Poco::IOException("Failed to read data info header"); + if (std::memcmp(_rawInfo, HEADER, ZipCommon::HEADER_SIZE) != 0) + throw Poco::DataFormatException("Bad data info header"); + } + + // now copy the rest of the header + in.read(_rawInfo+ZipCommon::HEADER_SIZE, FULLHEADER_SIZE - ZipCommon::HEADER_SIZE); + _valid = (!in.eof() && in.good()); +} + + +ZipDataInfo64::~ZipDataInfo64() +{ +} + + } } // namespace Poco::Zip diff --git a/Zip/src/ZipFileInfo.cpp b/Zip/src/ZipFileInfo.cpp index 487852dc2..68c0af69a 100644 --- a/Zip/src/ZipFileInfo.cpp +++ b/Zip/src/ZipFileInfo.cpp @@ -31,6 +31,7 @@ ZipFileInfo::ZipFileInfo(const ZipLocalFileHeader& header): _crc32(0), _compressedSize(0), _uncompressedSize(0), + _localHeaderOffset(0), _fileName(), _lastModifiedAt(), _extraField() @@ -63,6 +64,7 @@ ZipFileInfo::ZipFileInfo(std::istream& in, bool assumeHeaderRead): _crc32(0), _compressedSize(0), _uncompressedSize(0), + _localHeaderOffset(0), _fileName(), _lastModifiedAt(), _extraField() @@ -98,6 +100,7 @@ void ZipFileInfo::parse(std::istream& inp, bool assumeHeaderRead) _crc32 = getCRCFromHeader(); _compressedSize = getCompressedSizeFromHeader(); _uncompressedSize = getUncompressedSizeFromHeader(); + _localHeaderOffset = getOffsetFromHeader(); parseDateTime(); Poco::UInt16 len = getFileNameLength(); if (len > 0) @@ -114,6 +117,40 @@ void ZipFileInfo::parse(std::istream& inp, bool assumeHeaderRead) Poco::Buffer xtra(len); inp.read(xtra.begin(), len); _extraField = std::string(xtra.begin(), len); + char* ptr = xtra.begin(); + while (ptr <= xtra.begin() + len - 4) + { + Poco::UInt16 id = ZipUtil::get16BitValue(ptr, 0); + ptr += 2; + Poco::UInt16 size = ZipUtil::get16BitValue(ptr, 0); + ptr += 2; + if (id == ZipCommon::ZIP64_EXTRA_ID) + { + poco_assert(size >= 8); + if (getUncompressedSizeFromHeader() == ZipCommon::ZIP64_MAGIC) + { + setUncompressedSize(ZipUtil::get64BitValue(ptr, 0)); + size -= 8; + ptr += 8; + } + if (size >= 8 && getCompressedSizeFromHeader() == ZipCommon::ZIP64_MAGIC) + { + setCompressedSize(ZipUtil::get64BitValue(ptr, 0)); + size -= 8; + ptr += 8; + } + if (size >= 8 && getOffsetFromHeader() == ZipCommon::ZIP64_MAGIC) + { + setOffset(ZipUtil::get64BitValue(ptr, 0)); + size -= 8; + ptr += 8; + } + } + else + { + ptr += size; + } + } } } len = getFileCommentLength(); diff --git a/Zip/src/ZipLocalFileHeader.cpp b/Zip/src/ZipLocalFileHeader.cpp index 71e13d9e5..d46ec8f98 100644 --- a/Zip/src/ZipLocalFileHeader.cpp +++ b/Zip/src/ZipLocalFileHeader.cpp @@ -8,7 +8,7 @@ // Copyright (c) 2007, Applied Informatics Software Engineering GmbH. // and Contributors. // -// SPDX-License-Identifier: BSL-1.0 +// SPDX-License-Identifier: BSL-1.0 // @@ -28,66 +28,81 @@ namespace Zip { const char ZipLocalFileHeader::HEADER[ZipCommon::HEADER_SIZE] = {'\x50', '\x4b', '\x03', '\x04'}; -ZipLocalFileHeader::ZipLocalFileHeader(const Poco::Path& fileName, - const Poco::DateTime& lastModifiedAt, - ZipCommon::CompressionMethod cm, - ZipCommon::CompressionLevel cl): - _rawHeader(), - _startPos(-1), - _endPos(-1), - _fileName(), - _lastModifiedAt(), - _extraField(), - _crc32(0), - _compressedSize(0), - _uncompressedSize(0) +ZipLocalFileHeader::ZipLocalFileHeader(const Poco::Path& fileName, + const Poco::DateTime& lastModifiedAt, + ZipCommon::CompressionMethod cm, + ZipCommon::CompressionLevel cl, + bool forceZip64): + _forceZip64(forceZip64), + _rawHeader(), + _startPos(-1), + _endPos(-1), + _fileName(), + _lastModifiedAt(), + _extraField(), + _crc32(0), + _compressedSize(0), + _uncompressedSize(0) { - std::memcpy(_rawHeader, HEADER, ZipCommon::HEADER_SIZE); - std::memset(_rawHeader+ZipCommon::HEADER_SIZE, 0, FULLHEADER_SIZE - ZipCommon::HEADER_SIZE); - setHostSystem(ZipCommon::HS_FAT); - setEncryption(false); - setExtraFieldSize(0); - setLastModifiedAt(lastModifiedAt); - init(fileName, cm, cl); + std::memcpy(_rawHeader, HEADER, ZipCommon::HEADER_SIZE); + std::memset(_rawHeader+ZipCommon::HEADER_SIZE, 0, FULLHEADER_SIZE - ZipCommon::HEADER_SIZE); + setHostSystem(ZipCommon::HS_FAT); + setEncryption(false); + setExtraFieldSize(0); + setLastModifiedAt(lastModifiedAt); + init(fileName, cm, cl); } ZipLocalFileHeader::ZipLocalFileHeader(std::istream& inp, bool assumeHeaderRead, ParseCallback& callback): - _rawHeader(), - _startPos(inp.tellg()), - _endPos(-1), - _fileName(), - _lastModifiedAt(), - _extraField(), - _crc32(0), - _compressedSize(0), - _uncompressedSize(0) + _forceZip64(false), + _rawHeader(), + _startPos(inp.tellg()), + _endPos(-1), + _fileName(), + _lastModifiedAt(), + _extraField(), + _crc32(0), + _compressedSize(0), + _uncompressedSize(0) { - poco_assert_dbg( (EXTRAFIELD_POS+EXTRAFIELD_LENGTH) == FULLHEADER_SIZE); + poco_assert_dbg( (EXTRA_FIELD_POS+EXTRA_FIELD_LENGTH) == FULLHEADER_SIZE); - if (assumeHeaderRead) - _startPos -= ZipCommon::HEADER_SIZE; + if (assumeHeaderRead) + _startPos -= ZipCommon::HEADER_SIZE; - parse(inp, assumeHeaderRead); + parse(inp, assumeHeaderRead); - bool ok = callback.handleZipEntry(inp, *this); + bool ok = callback.handleZipEntry(inp, *this); - if (ok) - { - if (searchCRCAndSizesAfterData()) - { - ZipDataInfo nfo(inp, false); - setCRC(nfo.getCRC32()); - setCompressedSize(nfo.getCompressedSize()); - setUncompressedSize(nfo.getUncompressedSize()); - } - } - else - { - poco_assert_dbg(!searchCRCAndSizesAfterData()); - ZipUtil::sync(inp); - } - _endPos = _startPos + getHeaderSize() + _compressedSize; // exclude the data block! + if (ok) + { + if (searchCRCAndSizesAfterData()) + { + char header[ZipCommon::HEADER_SIZE]={'\x00', '\x00', '\x00', '\x00'}; + inp.read(header, ZipCommon::HEADER_SIZE); + if (_forceZip64) + { + ZipDataInfo64 nfo(inp, true); + setCRC(nfo.getCRC32()); + setCompressedSize(nfo.getCompressedSize()); + setUncompressedSize(nfo.getUncompressedSize()); + } + else + { + ZipDataInfo nfo(inp, true); + setCRC(nfo.getCRC32()); + setCompressedSize(nfo.getCompressedSize()); + setUncompressedSize(nfo.getUncompressedSize()); + } + } + } + else + { + poco_assert_dbg(!searchCRCAndSizesAfterData()); + ZipUtil::sync(inp); + } + _endPos = _startPos + getHeaderSize() + _compressedSize; // exclude the data block! } @@ -98,49 +113,79 @@ ZipLocalFileHeader::~ZipLocalFileHeader() void ZipLocalFileHeader::parse(std::istream& inp, bool assumeHeaderRead) { - if (!assumeHeaderRead) - { - inp.read(_rawHeader, ZipCommon::HEADER_SIZE); + if (!assumeHeaderRead) + { + inp.read(_rawHeader, ZipCommon::HEADER_SIZE); if (inp.gcount() != ZipCommon::HEADER_SIZE) throw Poco::IOException("Failed to read local file header"); if (std::memcmp(_rawHeader, HEADER, ZipCommon::HEADER_SIZE) != 0) throw Poco::DataFormatException("Bad local file header"); - } - else - { - std::memcpy(_rawHeader, HEADER, ZipCommon::HEADER_SIZE); - } + } + else + { + std::memcpy(_rawHeader, HEADER, ZipCommon::HEADER_SIZE); + } - // read the rest of the header - inp.read(_rawHeader + ZipCommon::HEADER_SIZE, FULLHEADER_SIZE - ZipCommon::HEADER_SIZE); - if (!(_rawHeader[VERSION_POS + 1]>= ZipCommon::HS_FAT && _rawHeader[VERSION_POS + 1] < ZipCommon::HS_UNUSED)) - throw Poco::DataFormatException("Bad local file header", "invalid version"); - if (ZipUtil::get16BitValue(_rawHeader, COMPR_METHOD_POS) >= ZipCommon::CM_UNUSED) - throw Poco::DataFormatException("Bad local file header", "invalid compression method"); - parseDateTime(); - Poco::UInt16 len = getFileNameLength(); - if (len > 0) - { - Poco::Buffer buf(len); - inp.read(buf.begin(), len); - _fileName = std::string(buf.begin(), len); + // read the rest of the header + inp.read(_rawHeader + ZipCommon::HEADER_SIZE, FULLHEADER_SIZE - ZipCommon::HEADER_SIZE); + poco_assert (_rawHeader[VERSION_POS + 1]>= ZipCommon::HS_FAT && _rawHeader[VERSION_POS + 1] < ZipCommon::HS_UNUSED); + poco_assert (getMajorVersionNumber() <= 4); // Allow for Zip64 version 4.5 + poco_assert (ZipUtil::get16BitValue(_rawHeader, COMPR_METHOD_POS) < ZipCommon::CM_UNUSED); + parseDateTime(); + Poco::UInt16 len = getFileNameLength(); + if (len > 0) + { + Poco::Buffer buf(len); + inp.read(buf.begin(), len); + _fileName = std::string(buf.begin(), len); } - if (hasExtraField()) - { - len = getExtraFieldLength(); - if (len > 0) - { + + if (!searchCRCAndSizesAfterData()) + { + _crc32 = getCRCFromHeader(); + _compressedSize = getCompressedSizeFromHeader(); + _uncompressedSize = getUncompressedSizeFromHeader(); + } + + if (hasExtraField()) + { + len = getExtraFieldLength(); + if (len > 0) + { Poco::Buffer xtra(len); inp.read(xtra.begin(), len); _extraField = std::string(xtra.begin(), len); - } - } - if (!searchCRCAndSizesAfterData()) - { - _crc32 = getCRCFromHeader(); - _compressedSize = getCompressedSizeFromHeader(); - _uncompressedSize = getUncompressedSizeFromHeader(); - } + char* ptr = xtra.begin(); + while (ptr <= xtra.begin() + len - 4) + { + Poco::UInt16 id = ZipUtil::get16BitValue(ptr, 0); + ptr += 2; + Poco::UInt16 size = ZipUtil::get16BitValue(ptr, 0); + ptr += 2; + if (id == ZipCommon::ZIP64_EXTRA_ID) + { + _forceZip64 = true; + poco_assert(size >= 8); + if (getUncompressedSizeFromHeader() == ZipCommon::ZIP64_MAGIC) + { + setUncompressedSize(ZipUtil::get64BitValue(ptr, 0)); + size -= 8; + ptr += 8; + } + if (size >= 8 && getCompressedSizeFromHeader() == ZipCommon::ZIP64_MAGIC) + { + setCompressedSize(ZipUtil::get64BitValue(ptr, 0)); + size -= 8; + ptr += 8; + } + } + else + { + ptr += size; + } + } + } + } } @@ -157,61 +202,61 @@ bool ZipLocalFileHeader::searchCRCAndSizesAfterData() const void ZipLocalFileHeader::setFileName(const std::string& fileName, bool isDirectory) { - poco_assert (!fileName.empty()); - Poco::Path aPath(fileName); + poco_assert (!fileName.empty()); + Poco::Path aPath(fileName); - if (isDirectory) - { - aPath.makeDirectory(); - setCRC(0); - setCompressedSize(0); - setUncompressedSize(0); - setCompressionMethod(ZipCommon::CM_STORE); - setCompressionLevel(ZipCommon::CL_NORMAL); - } - else - { - aPath.makeFile(); - } - _fileName = aPath.toString(Poco::Path::PATH_UNIX); - if (_fileName[0] == '/') - _fileName = _fileName.substr(1); - if (isDirectory) - { - poco_assert_dbg (_fileName[_fileName.size()-1] == '/'); - } - setFileNameLength(static_cast(_fileName.size())); + if (isDirectory) + { + aPath.makeDirectory(); + setCRC(0); + setCompressedSize(0); + setUncompressedSize(0); + setCompressionMethod(ZipCommon::CM_STORE); + setCompressionLevel(ZipCommon::CL_NORMAL); + } + else + { + aPath.makeFile(); + } + _fileName = aPath.toString(Poco::Path::PATH_UNIX); + if (_fileName[0] == '/') + _fileName = _fileName.substr(1); + if (isDirectory) + { + poco_assert_dbg (_fileName[_fileName.size()-1] == '/'); + } + setFileNameLength(static_cast(_fileName.size())); } -void ZipLocalFileHeader::init( const Poco::Path& fName, - ZipCommon::CompressionMethod cm, - ZipCommon::CompressionLevel cl) +void ZipLocalFileHeader::init(const Poco::Path& fName, ZipCommon::CompressionMethod cm, ZipCommon::CompressionLevel cl) { - poco_assert (_fileName.empty()); - setSearchCRCAndSizesAfterData(false); - Poco::Path fileName(fName); - fileName.setDevice(""); // clear device! - setFileName(fileName.toString(Poco::Path::PATH_UNIX), fileName.isDirectory()); - setRequiredVersion(2, 0); - if (fileName.isFile()) - { - setCompressionMethod(cm); - setCompressionLevel(cl); - } - else - setCompressionMethod(ZipCommon::CM_STORE); - + poco_assert (_fileName.empty()); + setSearchCRCAndSizesAfterData(false); + Poco::Path fileName(fName); + fileName.setDevice(""); // clear device! + setFileName(fileName.toString(Poco::Path::PATH_UNIX), fileName.isDirectory()); + setRequiredVersion(2, 0); + if (fileName.isFile()) + { + setCompressionMethod(cm); + setCompressionLevel(cl); + } + else + setCompressionMethod(ZipCommon::CM_STORE); + if (_forceZip64) + setZip64Data(); + _rawHeader[GENERAL_PURPOSE_POS+1] |= 0x08; // Set "language encoding flag" to indicate that filenames and paths are in UTF-8. } std::string ZipLocalFileHeader::createHeader() const { - std::string result(_rawHeader, FULLHEADER_SIZE); - result.append(_fileName); - result.append(_extraField); - return result; + std::string result(_rawHeader, FULLHEADER_SIZE); + result.append(_fileName); + result.append(_extraField); + return result; } diff --git a/Zip/src/ZipStream.cpp b/Zip/src/ZipStream.cpp index 9f33b03bb..33678c980 100644 --- a/Zip/src/ZipStream.cpp +++ b/Zip/src/ZipStream.cpp @@ -61,7 +61,7 @@ ZipStreamBuf::ZipStreamBuf(std::istream& istr, const ZipLocalFileHeader& fileEnt std::string crc(4, ' '); if (fileEntry.searchCRCAndSizesAfterData()) { - _ptrHelper = new AutoDetectInputStream(istr, init, crc, reposition, start); + _ptrHelper = new AutoDetectInputStream(istr, init, crc, reposition, static_cast(start), fileEntry.needsZip64()); } else { @@ -73,7 +73,7 @@ ZipStreamBuf::ZipStreamBuf(std::istream& istr, const ZipLocalFileHeader& fileEnt { if (fileEntry.searchCRCAndSizesAfterData()) { - _ptrBuf = new AutoDetectInputStream(istr, "", "", reposition, start); + _ptrBuf = new AutoDetectInputStream(istr, "", "", reposition, static_cast(start), fileEntry.needsZip64()); } else { @@ -122,7 +122,7 @@ ZipStreamBuf::ZipStreamBuf(std::ostream& ostr, ZipLocalFileHeader& fileEntry, bo else if (fileEntry.getCompressionLevel() == ZipCommon::CL_MAXIMUM) level = Z_BEST_COMPRESSION; // ignore the zlib init string which is of size 2 and also ignore the 4 byte adler32 value at the end of the stream! - _ptrOHelper = new PartialOutputStream(*_pOstr, 2, 4, false); + _ptrOHelper = new PartialOutputStream(*_pOstr, 2, 4, false); _ptrOBuf = new Poco::DeflatingOutputStream(*_ptrOHelper, DeflatingStreamBuf::STREAM_ZLIB, level); } else if (fileEntry.getCompressionMethod() == ZipCommon::CM_STORE) @@ -133,6 +133,8 @@ ZipStreamBuf::ZipStreamBuf(std::ostream& ostr, ZipLocalFileHeader& fileEntry, bo else throw Poco::NotImplementedException("Unsupported compression method"); // now write the header to the ostr! + if (fileEntry.needsZip64()) + fileEntry.setZip64Data(); std::string header = fileEntry.createHeader(); ostr.write(header.c_str(), static_cast(header.size())); } @@ -153,7 +155,7 @@ int ZipStreamBuf::readFromDevice(char* buffer, std::streamsize length) { if (!_ptrBuf) return 0; // directory entry _ptrBuf->read(buffer, length); - int cnt = _ptrBuf->gcount(); + int cnt = static_cast(_ptrBuf->gcount()); if (cnt > 0) { _crc32.update(buffer, cnt); @@ -172,6 +174,7 @@ int ZipStreamBuf::readFromDevice(char* buffer, std::streamsize length) // now push back the header to the stream, so that the ZipLocalFileHeader can read it Poco::Int32 size = static_cast(nfo.getFullHeaderSize()); _expectedCrc32 = nfo.getCRC32(); + const char* rawHeader = nfo.getRawHeader(); _pIstr->seekg(-size, std::ios::cur); if (!_pIstr->good()) throw Poco::IOException("Failed to seek on input stream"); if (!crcValid()) @@ -190,13 +193,14 @@ int ZipStreamBuf::writeToDevice(const char* buffer, std::streamsize length) return 0; _bytesWritten += length; _ptrOBuf->write(buffer, length); - _crc32.update(buffer, length); - return length; + _crc32.update(buffer, static_cast(length)); + return static_cast(length); } -void ZipStreamBuf::close() +void ZipStreamBuf::close(Poco::UInt64& extraDataSize) { + extraDataSize = 0; if (_ptrOBuf && _pHeader) { _ptrOBuf->flush(); @@ -209,15 +213,15 @@ void ZipStreamBuf::close() _ptrOHelper->close(); } _ptrOBuf = 0; - poco_assert (*_pOstr); + if (!*_pOstr) throw Poco::IOException("Bad output stream"); + // write an extra datablock if required // or fix the crc entries - poco_check_ptr(_pHeader); _pHeader->setCRC(_crc32.checksum()); _pHeader->setUncompressedSize(_bytesWritten); _pHeader->setCompressedSize(_ptrOHelper->bytesWritten()); - if (_bytesWritten == 0) + if (_bytesWritten == 0) { poco_assert (_ptrOHelper->bytesWritten() == 0); // Empty files must use CM_STORE, otherwise unzipping will fail @@ -228,20 +232,36 @@ void ZipStreamBuf::close() if (_pHeader->searchCRCAndSizesAfterData()) { - ZipDataInfo info; - info.setCRC32(_crc32.checksum()); - info.setUncompressedSize(_bytesWritten); - info.setCompressedSize(static_cast(_ptrOHelper->bytesWritten())); - _pOstr->write(info.getRawHeader(), static_cast(info.getFullHeaderSize())); + if (_pHeader->needsZip64()) + { + ZipDataInfo64 info; + info.setCRC32(_crc32.checksum()); + info.setUncompressedSize(_bytesWritten); + info.setCompressedSize(_ptrOHelper->bytesWritten()); + extraDataSize = info.getFullHeaderSize(); + _pOstr->write(info.getRawHeader(), static_cast(extraDataSize)); + } + else + { + ZipDataInfo info; + info.setCRC32(_crc32.checksum()); + info.setUncompressedSize(static_cast(_bytesWritten)); + info.setCompressedSize(static_cast(_ptrOHelper->bytesWritten())); + extraDataSize = info.getFullHeaderSize(); + _pOstr->write(info.getRawHeader(), static_cast(extraDataSize)); + } } else { _pOstr->seekp(_pHeader->getStartPos(), std::ios_base::beg); - poco_assert (*_pOstr); + if (!*_pOstr) throw Poco::IOException("Bad output stream"); + + if (_pHeader->hasExtraField()) // Update sizes in header extension. + _pHeader->setZip64Data(); std::string header = _pHeader->createHeader(); _pOstr->write(header.c_str(), static_cast(header.size())); _pOstr->seekp(0, std::ios_base::end); - poco_assert (*_pOstr); + if (!*_pOstr) throw Poco::IOException("Bad output stream"); } _pHeader = 0; } @@ -304,10 +324,10 @@ ZipOutputStream::~ZipOutputStream() } -void ZipOutputStream::close() +void ZipOutputStream::close(Poco::UInt64& extraDataSize) { flush(); - _buf.close(); + _buf.close(extraDataSize); } diff --git a/Zip/testsuite/src/CompressTest.cpp b/Zip/testsuite/src/CompressTest.cpp index 8541bcccf..ddcda6790 100644 --- a/Zip/testsuite/src/CompressTest.cpp +++ b/Zip/testsuite/src/CompressTest.cpp @@ -4,19 +4,23 @@ // Copyright (c) 2007, Applied Informatics Software Engineering GmbH. // and Contributors. // -// SPDX-License-Identifier: BSL-1.0 +// SPDX-License-Identifier: BSL-1.0 // #include "CompressTest.h" #include "ZipTest.h" +#include "Poco/Buffer.h" #include "Poco/Zip/Compress.h" #include "Poco/Zip/ZipManipulator.h" #include "Poco/File.h" #include "Poco/FileStream.h" #include "CppUnit/TestCaller.h" #include "CppUnit/TestSuite.h" +#include #include +#undef min +#include using namespace Poco::Zip; @@ -35,7 +39,7 @@ CompressTest::~CompressTest() void CompressTest::testSingleFile() { std::ofstream out("appinf.zip", std::ios::binary); - Poco::Path theFile(ZipTest::getTestFile("test.zip")); + Poco::Path theFile(ZipTest::getTestFile("data", "test.zip")); Compress c(out, true); c.addFile(theFile, theFile.getFileName()); ZipArchive a(c.close()); @@ -70,14 +74,14 @@ void CompressTest::testManipulator() { { std::ofstream out("appinf.zip", std::ios::binary); - Poco::Path theFile(ZipTest::getTestFile("test.zip")); + Poco::Path theFile(ZipTest::getTestFile("data", "test.zip")); Compress c(out, true); c.addFile(theFile, theFile.getFileName()); ZipArchive a(c.close()); } ZipManipulator zm("appinf.zip", true); zm.renameFile("test.zip", "renamedtest.zip"); - zm.addFile("doc/othertest.zip", ZipTest::getTestFile("test.zip")); + zm.addFile("doc/othertest.zip", ZipTest::getTestFile("data", "test.zip")); ZipArchive archive=zm.commit(); assert (archive.findHeader("doc/othertest.zip") != archive.headerEnd()); } @@ -87,14 +91,14 @@ void CompressTest::testManipulatorDel() { { std::ofstream out("appinf.zip", std::ios::binary); - Poco::Path theFile(ZipTest::getTestFile("test.zip")); + Poco::Path theFile(ZipTest::getTestFile("data", "test.zip")); Compress c(out, true); c.addFile(theFile, theFile.getFileName()); ZipArchive a(c.close()); } ZipManipulator zm("appinf.zip", true); zm.deleteFile("test.zip"); - zm.addFile("doc/data.zip", ZipTest::getTestFile("data.zip")); + zm.addFile("doc/data.zip", ZipTest::getTestFile("data", "data.zip")); ZipArchive archive=zm.commit(); assert (archive.findHeader("test.zip") == archive.headerEnd()); assert (archive.findHeader("doc/data.zip") != archive.headerEnd()); @@ -105,13 +109,13 @@ void CompressTest::testManipulatorReplace() { { std::ofstream out("appinf.zip", std::ios::binary); - Poco::Path theFile(ZipTest::getTestFile("test.zip")); + Poco::Path theFile(ZipTest::getTestFile("data", "test.zip")); Compress c(out, true); c.addFile(theFile, theFile.getFileName()); ZipArchive a(c.close()); } ZipManipulator zm("appinf.zip", true); - zm.replaceFile("test.zip", ZipTest::getTestFile("doc.zip")); + zm.replaceFile("test.zip", ZipTest::getTestFile("data", "doc.zip")); ZipArchive archive=zm.commit(); assert (archive.findHeader("test.zip") != archive.headerEnd()); @@ -123,7 +127,7 @@ void CompressTest::testSetZipComment() { std::string comment("Testing...123..."); std::ofstream out("comment.zip", std::ios::binary); - Poco::Path theFile(ZipTest::getTestFile("test.zip")); + Poco::Path theFile(ZipTest::getTestFile("data", "test.zip")); Compress c(out, true); c.addFile(theFile, theFile.getFileName()); c.setZipComment(comment); @@ -132,6 +136,60 @@ void CompressTest::testSetZipComment() } +void CompressTest::createDataFile(const std::string& path, Poco::UInt64 size) +{ + std::ofstream out(path.c_str(), std::ios::binary | std::ios::trunc); + assert( ! out.fail() ); + Poco::Buffer buffer(MB); + for(int i = 0; size != 0; i++) { + std::memset(buffer.begin(), i, buffer.size()); + Poco::UInt64 bytesToWrite = std::min(size, static_cast(buffer.size())); + out.write(buffer.begin(), bytesToWrite); + assert( ! out.fail() ); + size -= bytesToWrite; + } + out.flush(); + assert( ! out.fail() ); + out.close(); + assert( ! out.fail() ); +} + + +void CompressTest::testZip64() +{ + std::cout << std::endl; + std::map files; + files["data1.bin"] = static_cast(KB)*4096+1; + files["data2.bin"] = static_cast(KB)*16; + files["data3.bin"] = static_cast(KB)*4096-1; + + for(std::map::const_iterator it = files.begin(); it != files.end(); it++) + { + std::cout << '\t' << "createDataFile(" << it->first << ", " << it->second << ");" << std::endl; + createDataFile(it->first, it->second); + } + std::ofstream out("zip64.zip", std::ios::binary | std::ios::trunc); + Compress c(out, true, true); + for(std::map::const_iterator it = files.begin(); it != files.end(); it++) + { + const std::string& path = it->first; + std::cout << '\t' << "addFile(" << path << ");" << std::endl; + c.addFile(path, path, ZipCommon::CM_STORE); + } + ZipArchive a(c.close()); + for(std::map::const_iterator it = files.begin(); it != files.end(); it++) + { + const std::string& path = it->first; + Poco::UInt64 size = it->second; + ZipArchive::FileHeaders::const_iterator it2 = a.findHeader(path); + assert (it2 != a.headerEnd()); + const Poco::Zip::ZipLocalFileHeader& file = it2->second; + assert(file.getUncompressedSize() == size); + assert(file.getCompressedSize() == size); + } +} + + void CompressTest::setUp() { } @@ -152,6 +210,7 @@ CppUnit::Test* CompressTest::suite() CppUnit_addTest(pSuite, CompressTest, testManipulatorDel); CppUnit_addTest(pSuite, CompressTest, testManipulatorReplace); CppUnit_addTest(pSuite, CompressTest, testSetZipComment); + CppUnit_addTest(pSuite, CompressTest, testZip64); return pSuite; } diff --git a/Zip/testsuite/src/CompressTest.h b/Zip/testsuite/src/CompressTest.h index fd912d4d0..375cc6d81 100644 --- a/Zip/testsuite/src/CompressTest.h +++ b/Zip/testsuite/src/CompressTest.h @@ -31,6 +31,11 @@ public: void testManipulatorReplace(); void testSetZipComment(); + static const Poco::UInt64 KB = 1024; + static const Poco::UInt64 MB = 1024*KB; + void createDataFile(const std::string& path, Poco::UInt64 size); + void testZip64(); + void setUp(); void tearDown(); diff --git a/Zip/testsuite/src/ZipTest.cpp b/Zip/testsuite/src/ZipTest.cpp index 08803c247..64ebdcaa6 100644 --- a/Zip/testsuite/src/ZipTest.cpp +++ b/Zip/testsuite/src/ZipTest.cpp @@ -4,7 +4,7 @@ // Copyright (c) 2007, Applied Informatics Software Engineering GmbH. // and Contributors. // -// SPDX-License-Identifier: BSL-1.0 +// SPDX-License-Identifier: BSL-1.0 // @@ -21,8 +21,12 @@ #include "Poco/Path.h" #include "Poco/Delegate.h" #include "Poco/StreamCopier.h" +#include "Poco/Environment.h" #include "CppUnit/TestCaller.h" #include "CppUnit/TestSuite.h" +#undef min +#include +#include #include #include @@ -42,7 +46,7 @@ ZipTest::~ZipTest() void ZipTest::testSkipSingleFile() { - std::string testFile = getTestFile("test.zip"); + std::string testFile = getTestFile("data", "test.zip"); std::ifstream inp(testFile.c_str(), std::ios::binary); assert (inp.good()); SkipCallback skip; @@ -56,15 +60,15 @@ void ZipTest::testSkipSingleFile() ZipCommon::CompressionMethod cm = hdr.getCompressionMethod(); assert (!hdr.isEncrypted()); Poco::DateTime aDate = hdr.lastModifiedAt(); - Poco::UInt32 cS = hdr.getCompressedSize(); - Poco::UInt32 uS = hdr.getUncompressedSize(); + Poco::UInt64 cS = hdr.getCompressedSize(); + Poco::UInt64 uS = hdr.getUncompressedSize(); const std::string& fileName = hdr.getFileName(); } void ZipTest::testDecompressSingleFile() { - std::string testFile = getTestFile("test.zip"); + std::string testFile = getTestFile("data", "test.zip"); std::ifstream inp(testFile.c_str(), std::ios::binary); assert (inp.good()); ZipArchive arch(inp); @@ -77,9 +81,24 @@ void ZipTest::testDecompressSingleFile() } +void ZipTest::testDecompressSingleFileInDir() +{ + std::string testFile = getTestFile("data","test.zip"); + std::ifstream inp(testFile.c_str(), std::ios::binary); + assert (inp.good()); + ZipArchive arch(inp); + ZipArchive::FileHeaders::const_iterator it = arch.findHeader("testdir/testfile.txt"); + assert (it != arch.headerEnd()); + ZipInputStream zipin (inp, it->second); + std::ostringstream out(std::ios::binary); + Poco::StreamCopier::copyStream(zipin, out); + assert(!out.str().empty()); +} + + void ZipTest::testCrcAndSizeAfterData() { - std::string testFile = getTestFile("data.zip"); + std::string testFile = getTestFile("data", "data.zip"); std::ifstream inp(testFile.c_str(), std::ios::binary); assert (inp.good()); Decompress dec(inp, Poco::Path()); @@ -93,7 +112,7 @@ void ZipTest::testCrcAndSizeAfterData() void ZipTest::testCrcAndSizeAfterDataWithArchive() { - std::string testFile = getTestFile("data.zip"); + std::string testFile = getTestFile("data", "data.zip"); std::ifstream inp(testFile.c_str(), std::ios::binary); assert (inp.good()); Poco::Zip::ZipArchive zip(inp); @@ -113,30 +132,34 @@ void ZipTest::testCrcAndSizeAfterDataWithArchive() } -std::string ZipTest::getTestFile(const std::string& testFile) +std::string ZipTest::getTestFile(const std::string& directory, const std::string& file) { - Poco::Path root; - root.makeAbsolute(); - Poco::Path result; - while (!Poco::Path::find(root.toString(), "data", result)) + std::ostringstream ostr; + ostr << directory << '/' << file; + std::string validDir(ostr.str()); + Poco::Path pathPattern(validDir); + if (Poco::File(pathPattern).exists()) { - root.makeParent(); - if (root.toString().empty() || root.toString() == "/") - throw Poco::FileNotFoundException("Didn't find data subdir"); + return validDir; } - result.makeDirectory(); - result.setFileName(testFile); - Poco::File aFile(result.toString()); - if (!aFile.exists() || (aFile.exists() && !aFile.isFile())) - throw Poco::FileNotFoundException("Didn't find " + testFile); - - return result.toString(); + + ostr.str(""); + ostr << "/Zip/testsuite/" << directory << '/' << file; + validDir = Poco::Environment::get("POCO_BASE") + ostr.str(); + pathPattern = validDir; + + if (!Poco::File(pathPattern).exists()) + { + std::cout << "Can't find " << validDir << std::endl; + throw Poco::NotFoundException("cannot locate directory containing valid Zip test files"); + } + return validDir; } void ZipTest::testDecompress() { - std::string testFile = getTestFile("test.zip"); + std::string testFile = getTestFile("data", "test.zip"); std::ifstream inp(testFile.c_str(), std::ios::binary); assert (inp.good()); Decompress dec(inp, Poco::Path()); @@ -150,7 +173,7 @@ void ZipTest::testDecompress() void ZipTest::testDecompressFlat() { - std::string testFile = getTestFile("test.zip"); + std::string testFile = getTestFile("data", "test.zip"); std::ifstream inp(testFile.c_str(), std::ios::binary); assert (inp.good()); Decompress dec(inp, Poco::Path(), true); @@ -162,6 +185,51 @@ void ZipTest::testDecompressFlat() } +void ZipTest::verifyDataFile(const std::string& path, Poco::UInt64 size) +{ + std::ifstream in(path.c_str(), std::ios::binary); + assert( ! in.fail() ); + Poco::Buffer buffer1(MB); + Poco::Buffer buffer2(MB); + for (int i = 0; size != 0; i++) + { + std::memset(buffer1.begin(), i, buffer1.size()); + std::memset(buffer2.begin(), 0, buffer2.size()); + Poco::UInt64 bytesToRead = std::min(size, static_cast(buffer2.size())); + in.read(buffer2.begin(), bytesToRead); + assert(!in.fail() ); + assert(std::memcmp(buffer1.begin(), buffer2.begin(), static_cast(bytesToRead)) == 0); + size -= bytesToRead; + } + char c; + in.read(&c, 1); + assert ( in.eof() ); +} + + +void ZipTest::testDecompressZip64() +{ + std::map files; + files["data1.bin"] = static_cast(KB)*4096+1; + files["data2.bin"] = static_cast(KB)*16; + files["data3.bin"] = static_cast(KB)*4096-1; + + for(std::map::const_iterator it = files.begin(); it != files.end(); it++) + { + Poco::File file(it->first); + if(file.exists()) + file.remove(); + } + std::ifstream in("zip64.zip", std::ios::binary); + Decompress c(in, "."); + c.decompressAllFiles(); + for(std::map::const_iterator it = files.begin(); it != files.end(); it++) + { + verifyDataFile(it->first, it->second); + } +} + + void ZipTest::onDecompressError(const void* pSender, std::pair& info) { ++_errCnt; @@ -185,9 +253,12 @@ CppUnit::Test* ZipTest::suite() CppUnit_addTest(pSuite, ZipTest, testSkipSingleFile); CppUnit_addTest(pSuite, ZipTest, testDecompressSingleFile); + CppUnit_addTest(pSuite, ZipTest, testDecompressSingleFileInDir); CppUnit_addTest(pSuite, ZipTest, testDecompress); CppUnit_addTest(pSuite, ZipTest, testDecompressFlat); CppUnit_addTest(pSuite, ZipTest, testCrcAndSizeAfterData); CppUnit_addTest(pSuite, ZipTest, testCrcAndSizeAfterDataWithArchive); + CppUnit_addTest(pSuite, ZipTest, testDecompressZip64); + return pSuite; } diff --git a/Zip/testsuite/src/ZipTest.h b/Zip/testsuite/src/ZipTest.h index 42207d54f..1e005895d 100644 --- a/Zip/testsuite/src/ZipTest.h +++ b/Zip/testsuite/src/ZipTest.h @@ -6,7 +6,7 @@ // Copyright (c) 2007, Applied Informatics Software Engineering GmbH. // and Contributors. // -// SPDX-License-Identifier: BSL-1.0 +// SPDX-License-Identifier: BSL-1.0 // @@ -27,18 +27,24 @@ public: void testSkipSingleFile(); void testDecompressSingleFile(); + void testDecompressSingleFileInDir(); void testDecompress(); void testCrcAndSizeAfterData(); void testCrcAndSizeAfterDataWithArchive(); void testDecompressFlat(); + static const Poco::UInt64 KB = 1024; + static const Poco::UInt64 MB = 1024*KB; + void verifyDataFile(const std::string& path, Poco::UInt64 size); + void testDecompressZip64(); + void setUp(); void tearDown(); static CppUnit::Test* suite(); - static std::string getTestFile(const std::string& testFile); + static std::string getTestFile(const std::string& directory, const std::string& type); private: void onDecompressError(const void* pSender, std::pair& info); diff --git a/Zip/testsuite/src/ZipTestSuite.cpp b/Zip/testsuite/src/ZipTestSuite.cpp index 6fca8359b..3846e0564 100644 --- a/Zip/testsuite/src/ZipTestSuite.cpp +++ b/Zip/testsuite/src/ZipTestSuite.cpp @@ -18,9 +18,9 @@ CppUnit::Test* ZipTestSuite::suite() { CppUnit::TestSuite* pSuite = new CppUnit::TestSuite("ZipTestSuite"); + pSuite->addTest(CompressTest::suite()); pSuite->addTest(ZipTest::suite()); pSuite->addTest(PartialStreamTest::suite()); - pSuite->addTest(CompressTest::suite()); return pSuite; }