//----------------------------------- //--- 010 Editor v2.0 Binary Template // // File: ZIPTemplateAdv.bt // Author: SweetScape Software // Revision: 2.4 // Purpose: Defines a template for // parsing ZIP files. Handles more // complex ZIP data such as ZIP64 // that the base ZIPTemplate.bt cannot // read. These changes may be merged into // the main ZIPTemplate.bt file. // // Changes: // 2.1 (SweetScape): // - Added write function for ZIPFILERECORD structure. // 2.2 (S.Gibson, KPMG LLP): // - Fix for entry comment field // - Fix for parsing data descriptors // 2.3 (DCeres): // - Added write function for VERECORD structure. // - Added write function for ZIP64ENDLOCATORRECORD structure. // - Added write function for ZIP64ENDLOCATOR structure. // - Added write function for EXTRAFIELD structure. // 2.4 (George Woods): // - Handle zero compressed size in header with ZIPDATADESCR header // after compressed data. //----------------------------------- // Define structures used in ZIP files typedef enum { S_ZIPFILERECORD = 0x04034b50, S_ZIPDATADESCR = 0x08074b50, S_ZIPDIRENTRY = 0x02014b50, S_ZIPDIGITALSIG = 0x05054b50, S_ZIP64ENDLOCATORRECORD = 0x06064b50, S_ZIP64ENDLOCATOR = 0x07064b50, S_ZIPENDLOCATOR = 0x06054b50 } SignatureTYPE ; typedef enum { OS_FAT = 0, OS_AMIGA = 1, OS_VMS = 2, // VAX/VMS OS_Unix = 3, OS_VM_CMS = 4, OS_Atari = 5, // what if it's a minix filesystem? [cjh] OS_HPFS = 6, // filesystem used by OS/2 (and NT 3.x) OS_Mac = 7, OS_Z_System = 8, OS_CPM = 9, OS_TOPS20 = 10, // pkzip 2.50 NTFS OS_NTFS = 11, // filesystem used by Windows NT OS_QDOS = 12, // SMS/QDOS OS_Acorn = 13, // Archimedes Acorn RISC OS OS_VFAT = 14, // filesystem used by Windows 95, NT OS_MVS = 15, OS_BeOS = 16, // hybrid POSIX/database filesystem OS_Tandem = 17, OS_OS400 = 18, OS_OSX = 19 } HOSTOSTYPE; typedef byte VERSIONTYPE ; string read_VERSIONTYPE (local VERSIONTYPE &af) { local string s = ""; SPrintf (s, "%1.1f", (float)af / 10); return s; } typedef struct{ VERSIONTYPE Version; HOSTOSTYPE HostOS; } VERECORD ; string read_VERECORD (local VERECORD &af) { local string s = ""; SPrintf (s, "Ver %1.1f, ", (float)af.Version / 10); s += EnumToString( af.HostOS); return s; } //enum used for compression format typedef enum { COMP_STORED = 0, COMP_SHRUNK = 1, COMP_REDUCED1 = 2, COMP_REDUCED2 = 3, COMP_REDUCED3 = 4, COMP_REDUCED4 = 5, COMP_IMPLODED = 6, COMP_TOKEN = 7, COMP_DEFLATE = 8, // Deflate - standard ZIP codec, used from the start, also found in regular ZIP files COMP_DEFLATE64 = 9, COMP_PKImploding = 10, COMP_BZip2 = 12, // BZIP2 - Newer than deflate, with better compression and slightly slower speed. COMP_LZMA = 14, // LZMA - advanced ZIPX codec, taken from open source 7-zip format COMP_Terse = 18, COMP_Lz77 = 19, COMP_Jpeg = 0x60, // Jpeg - Codec added by WinZip for specific support of jpeg images ( http://www.winzip.com/wz_jpg_comp.pdf ) COMP_WavPack = 0x61, // WavPack - Codec for compressing specifically wav files. ( http://www.wavpack.com ) COMP_PPMd = 0x62, // PPMd - context modeling based codec, also featured in new ZIP standard. We use it in our Optimized method for text file compression. ( http://www.compression.ru/ds/ ) COMP_WzAES = 0x63 // WZAES encryption methods } COMPTYPE; typedef enum { FLAG_Encrypted = 0x0001, //Bit 0: If set, indicates that the file is encrypted. FLAG_CompressionFlagBit1 = 0x0002, FLAG_CompressionFlagBit2 = 0x0004, FLAG_DescriptorUsedMask = 0x0008, FLAG_Reserved1 = 0x0010, FLAG_Reserved2 = 0x0020, FLAG_StrongEncrypted = 0x0040, //Bit 6: Strong encryption FLAG_CurrentlyUnused1 = 0x0080, FLAG_CurrentlyUnused2 = 0x0100, FLAG_CurrentlyUnused3 = 0x0200, FLAG_CurrentlyUnused4 = 0x0400, FLAG_Utf8 = 0x0800, // Bit 11: filename and comment encoded using UTF-8 FLAG_ReservedPKWARE1 = 0x1000, FLAG_CDEncrypted = 0x2000, // Bit 13: Used when encrypting the Central Directory to indicate selected data values in the Local Header are masked to hide their actual values. FLAG_ReservedPKWARE2 = 0x4000, FLAG_ReservedPKWARE3 = 0x8000, } FLAGTYPE ; string read_FLAGTYPE (local FLAGTYPE &af) { local string s = ""; local int commaNeeded = 0; local FLAGTYPE i = 1; SPrintf (s, "%d: ", af); while (i < FLAG_ReservedPKWARE3) { if (af & i) { if (commaNeeded) { s += ", "; } s += EnumToString(i); commaNeeded = 1; } i = i << 1; } return s; } typedef enum { EH_Zip64 = 0x0001, //Zip64 extended information extra field EH_AVInfo = 0x0007, //AV Info EH_ExtLanguage = 0x0008, //Reserved for extended language encoding data (PFS) EH_OS2 = 0x0009, //OS/2 EH_NTFS = 0x000a, //NTFS EH_OpenVMS = 0x000c, //OpenVMS EH_UNIX = 0x000d, //UNIX EH_fileStream = 0x000e, //Reserved for file stream and fork descriptors EH_PatchDescriptor = 0x000f, //Patch Descriptor EH_PKCS7X509 = 0x0014, //PKCS#7 Store for X.509 Certificates EH_X509IDSignature = 0x0015, //X.509 Certificate ID and Signature for individual file EH_X509IDCD = 0x0016, //X.509 Certificate ID for Central Directory EH_StrongEncryption = 0x0017, //Strong Encryption Header EH_RecordManagement = 0x0018, //Record Management Controls EH_PKCS7List = 0x0019, //PKCS#7 Encryption Recipient Certificate List EH_Attributes = 0x0065, //IBM S/390 (Z390), AS/400 (I400) attributes uncompressed EH_ReservedAttributes = 0x0066, //Reserved for IBM S/390 (Z390), AS/400 (I400) attributes - compressed EH_POSZIP4690 = 0x4690, //POSZIP 4690 (reserved) EH_Mac = 0x07c8, //Macintosh EH_ZipItMac1 = 0x2605, //ZipIt Macintosh EH_ZipItMac2 = 0x2705, //ZipIt Macintosh 1.3.5+ EH_ZipItMac3 = 0x2805, //ZipIt Macintosh 1.3.5+ EH_InfoZIPMac = 0x334d, //Info-ZIP Macintosh EH_Acorn = 0x4341, //Acorn/SparkFS EH_WinNTSecurity = 0x4453, //Windows NT security descriptor (binary ACL) EH_VM_CMS = 0x4704, //VM/CMS EH_MVS = 0x470f, //MVS EH_FWKCS = 0x4b46, //FWKCS MD5 (see below) EH_OS2AccessList = 0x4c41, //OS/2 access control list (text ACL) EH_InfoZIPOpenVMS = 0x4d49, //Info-ZIP OpenVMS EH_Xceed = 0x4f4c, //Xceed original location extra field EH_AOSVS = 0x5356, //AOS/VS (ACL) EH_extTimestamp = 0x5455, //extended timestamp EH_XceedUnicode = 0x554e, //Xceed unicode extra field EH_InfoZIPUNIX = 0x5855, //Info-ZIP UNIX (original, also OS/2, NT, etc) EH_InfoZIPUnicodeComment = 0x6375, //Info-ZIP Unicode Comment Extra Field EH_BeOS = 0x6542, //BeOS/BeBox EH_InfoZIPUnicodePath = 0x7075, //Info-ZIP Unicode Path Extra Field EH_ASiUNIX = 0x756e, //ASi UNIX EH_InfoZIPUNIXNew = 0x7855, //Info-ZIP UNIX (16-bit UID/GID info) EH_InfoZIPUNIXNew3rd = 0x7875, //Info-ZIP UNIX 3rd generation (generic UID/GID, ...) EH_WinGrowth = 0xa220, //Microsoft Open Packaging Growth Hint EH_SMSQDOS = 0xfd4a, //SMS/QDOS EH_WzAES = 0x9901, // } HEADERFLAG; typedef enum { AlgID_DES = 0x6601, //- DES AlgID_RC2OLD = 0x6602, //- RC2 (version needed to extract < 5.2) AlgID_3DES168 = 0x6603, //- 3DES 168 AlgID_3DES112 = 0x6609, //- 3DES 112 AlgID_AES128 = 0x660E, //- AES 128 AlgID_AES192 = 0x660F, //- AES 192 AlgID_AES256 = 0x6610, //- AES 256 AlgID_RC2 = 0x6702, //- RC2 (version needed to extract >= 5.2) AlgID_Blowfish = 0x6720, //- Blowfish AlgID_Twofish = 0x6721, //- Twofish AlgID_RC4 = 0x6801, //- RC4 AlgID_Unknown = 0xFFFF, //- Unknown algorithm } ALGFLAG; typedef enum { AES128 = 0x01, //128-bit encryption key AES192 = 0x02, //192-bit encryption key AES256 = 0x03, //256-bit encryption key } AESMODE; typedef enum { pfPassword = 0x0001, //- Password is required to decrypt pfCertificates = 0x0002, //- Certificates only pfPasswordCertificates = 0x0003, //- Password or certificate required to decrypt } PRCFLAG; typedef struct { HEADERFLAG efHeaderID; ushort efDataSize; Printf("%d", efHeaderID); switch (efHeaderID) { case EH_Zip64: uint64 efOriginalSize; uint64 efCompressedSize; //uint64 efHeaderOffset; //uint efDiskNumberStart; break; case EH_InfoZIPUnicodePath: byte efVersion; uint efNameCRC32; if(efDataSize > 0 ) char efUnicodeName[efDataSize - 5]; break; case EH_NTFS: int Reserved; //4 bytes Reserved for future use local int len = efDataSize - 4; while (len > 0) { ushort Tag; //2 bytes NTFS attribute tag value #1 ushort Size; //2 bytes Size of attribute #1, in bytes if (Tag == 0x001) { FILETIME Mtime; //8 bytes File last modification time FILETIME Atime; //8 bytes File last access time FILETIME Ctime; //8 bytes File creation time } else byte Data[Size]; //(var.) Size1 Attribute #1 data len -= Size + 4; } break; case EH_StrongEncryption: ushort Format; //2 bytes Format definition for this record ALGFLAG AlgID; //2 bytes Encryption algorithm identifier ushort Bitlen; //2 bytes Bit length of encryption key PRCFLAG Flags; //2 bytes Processing flags if (efDataSize > 8) byte CertData[efDataSize - 8]; //TSize-8 Certificate decryption extra field data break; case EH_WzAES: ushort version; //2 bytes Integer version number specific to the zip vendor char VendorID[2]; //2 bytes 2-character vendor ID AESMODE Strength; //1 bytes Integer mode value indicating AES encryption strength COMPTYPE deCompression; //2 bytes The actual compression method used to compress the file break; default: if(efDataSize > 0 ) char efData[ efDataSize ]; break; } } EXTRAFIELD ; string read_EXTRAFIELD (local EXTRAFIELD &af) { return EnumToString(af.efHeaderID); } typedef struct { HEADERFLAG efHeaderID; uint efDataSize; Printf("%d", efHeaderID); switch (efHeaderID) { case EH_Zip64: uint64 efOriginalSize; uint64 efCompressedSize; //uint64 efHeaderOffset; //uint efDiskNumberStart; break; case EH_InfoZIPUnicodePath: byte efVersion; uint efNameCRC32; if(efDataSize > 0 ) char efUnicodeName[efDataSize - 5]; break; default: if(efDataSize > 0 ) char efData[ efDataSize ]; break; } } EXTRA64FIELD; typedef enum { FA_READONLY = 0x00000001, FA_HIDDEN = 0x00000002, FA_SYSTEM = 0x00000004, FA_DIRECTORY = 0x00000010, FA_ARCHIVE = 0x00000020, FA_DEVICE = 0x00000040, FA_NORMAL = 0x00000080, FA_TEMPORARY = 0x00000100, FA_SPARSE_FILE = 0x00000200, FA_REPARSE_POINT = 0x00000400, FA_COMPRESSED = 0x00000800, FA_OFFLINE = 0x00001000, FA_NOT_CONTENT_INDEXED = 0x00002000, FA_ENCRYPTED = 0x00004000, FA_VIRTUAL = 0x00010000, kIFMT = 0170000 << 16, /* Unix file type mask */ kIFDIR = 0040000 << 16, /* Unix directory */ kIFREG = 0100000 << 16, /* Unix regular file */ kIFSOCK = 0140000 << 16, /* Unix socket (BSD, not SysV or Amiga) */ kIFLNK = 0120000 << 16, /* Unix symbolic link (not SysV, Amiga) */ kIFBLK = 0060000 << 16, /* Unix block special (not Amiga) */ kIFCHR = 0020000 << 16, /* Unix character special (not Amiga) */ kIFIFO = 0010000 << 16, /* Unix fifo (BCC, not MSC or Amiga) */ kISUID = 04000 << 16, /* Unix set user id on execution */ kISGID = 02000 << 16, /* Unix set group id on execution */ kISVTX = 01000 << 16, /* Unix directory permissions control */ kIRWXU = 00700 << 16, /* Unix read, write, execute: owner */ kIRUSR = 00400 << 16, /* Unix read permission: owner */ kIWUSR = 00200 << 16, /* Unix write permission: owner */ kIXUSR = 00100 << 16, /* Unix execute permission: owner */ kIRWXG = 00070 << 16, /* Unix read, write, execute: group */ kIRGRP = 00040 << 16, /* Unix read permission: group */ kIWGRP = 00020 << 16, /* Unix write permission: group */ kIXGRP = 00010 << 16, /* Unix execute permission: group */ kIRWXO = 00007 << 16, /* Unix read, write, execute: other */ kIROTH = 00004 << 16, /* Unix read permission: other */ kIWOTH = 00002 << 16, /* Unix write permission: other */ kIXOTH = 00001 << 16 /* Unix execute permission: other */ } FILEATTRIBUTE ; string read_FILEATTRIBUTE (local FILEATTRIBUTE &af) { local string s = ""; local int commaNeeded = 0; local FILEATTRIBUTE i = 1; SPrintf (s, "0x%X: ", af); while (i < 0xFFFFFF - 2) { if (af & i) { if (commaNeeded) { s += ", "; } s += EnumToString(i); commaNeeded = 1; } i = i << 1; } return s; } // Defintes the Data descriptor typedef struct { SignatureTYPE ddSignature; //0x08074b50 uint ddCRC ; uint ddCompressedSize; uint ddUncompressedSize; } ZIPDATADESCR; // Defines a file record typedef struct { // Header for the file SignatureTYPE frSignature; //0x04034b50 VERECORD frVersion; FLAGTYPE frFlags; COMPTYPE frCompression; DOSTIME frFileTime; DOSDATE frFileDate; uint frCrc ; uint frCompressedSize; uint frUncompressedSize; ushort frFileNameLength; ushort frExtraFieldLength; if( frFileNameLength > 0 ) char frFileName[ frFileNameLength ]; local int len = frExtraFieldLength; while (len > 0) { EXTRAFIELD frExtraField; len -= frExtraField.efDataSize + 4; } // Compressed data SetBackColor( cNone ); if ((frFlags & FLAG_Encrypted) && ( frFlags & FLAG_StrongEncrypted)) { struct { ushort IVSize; byte IVData[IVSize]; uint Size; ushort Format; ALGFLAG AlgID; ushort BitLen; ushort Flags; ushort ErdSize; byte ErdData[ErdSize]; uint Reserved; ushort VSize; byte VData[VSize - 4]; uint VCRC32; } StrongEncryptedHeader; char frData[ frCompressedSize - StrongEncryptedHeader.IVSize - StrongEncryptedHeader.Size - 6]; } else if ((frFlags & FLAG_Encrypted) && ( frCompression == COMP_WzAES )) { local int lenSalt = 0; if (frExtraField.efHeaderID == EH_WzAES) { switch (frExtraField.Strength) { case AES128: lenSalt = 8; break; case AES192: lenSalt = 12; break; case AES256: lenSalt = 16; break; } } uchar SaltValue[lenSalt]; ushort PassVerification; uchar frData[ frCompressedSize - 12 - lenSalt]; uchar AuthenticationCode[ 10]; } else if( (frCompressedSize > 0) && (frCompressedSize < 0xFFFFFFFF)) { uchar frData[ frCompressedSize ]; } else if (frCompressedSize == 0 && (frFlags & FLAG_DescriptorUsedMask)) { // If bit 3 (0x08) of the flags field is set, then the CRC-32 and file sizes were not known when the header was written. // Instead there is an additional header ZIPDATADESCR appended after the compressed data. // We look for the next signature of the appended header here. local int64 posCurrent = FTell(); local int64 posNext = FindFirst(S_ZIPDATADESCR,true,false,false,0.0,1,posCurrent); if (posNext >= posCurrent) { uchar frData[posNext - posCurrent]; SetBackColor( cLtGreen ); ZIPDATADESCR dataDescr; } } } ZIPFILERECORD ; // Defines an entry in the directory table typedef struct { SignatureTYPE deSignature; //0x02014b50 VERECORD deVersionMadeBy; VERECORD deVersionToExtract; FLAGTYPE deFlags; COMPTYPE deCompression; DOSTIME deFileTime; DOSDATE deFileDate; uint deCrc ; uint deCompressedSize; uint deUncompressedSize; ushort deFileNameLength; ushort deExtraFieldLength; ushort deFileCommentLength; ushort deDiskNumberStart; ushort deInternalAttributes; FILEATTRIBUTE deExternalAttributes; uint deHeaderOffset; if( deFileNameLength > 0 ) char deFileName[ deFileNameLength ]; local int len = deExtraFieldLength; while (len > 0) { EXTRAFIELD deExtraField; len -= deExtraField.efDataSize + 4; } if( deFileCommentLength > 0 ) uchar deFileComment[ deFileCommentLength ]; } ZIPDIRENTRY ; // Defines the digital signature typedef struct { SignatureTYPE dsSignature; //0x05054b50 ushort dsDataLength; if( dsDataLength > 0 ) uchar dsData[ dsDataLength ]; } ZIPDIGITALSIG; // Zip64 end of central directory record typedef struct { SignatureTYPE elr64Signature; //0x06064b50 int64 elr64DirectoryRecordSize; if (elr64DirectoryRecordSize > 1) VERECORD elr64VersionMadeBy; if (elr64DirectoryRecordSize > 2) VERECORD elr64VersionToExtract; if (elr64DirectoryRecordSize > 4) uint el64DiskNumber; if (elr64DirectoryRecordSize > 8) uint el64StartDiskNumber; if (elr64DirectoryRecordSize > 12) int64 el64EntriesOnDisk; if (elr64DirectoryRecordSize > 20) int64 el64EntriesInDirectory; if (elr64DirectoryRecordSize > 28) int64 el64DirectorySize; if (elr64DirectoryRecordSize > 36) int64 el64DirectoryOffset; if (elr64DirectoryRecordSize > 44) { char DataSect [elr64DirectoryRecordSize - 44]; // local int len = elr64DirectoryRecordSize - 44; // while (len > 0) // { // EXTRA64FIELD frExtraField; // len -= frExtraField.efDataSize + 4; // } } } ZIP64ENDLOCATORRECORD; //Zip64 end of central directory locator typedef struct { SignatureTYPE elSignature; //0x07064b50 uint elStartDiskNumber; int64 elDirectoryOffset; uint elEntriesInDirectory; } ZIP64ENDLOCATOR; // Defines the end of central directory locator typedef struct { SignatureTYPE elSignature; //0x06054b50 ushort elDiskNumber; ushort elStartDiskNumber; ushort elEntriesOnDisk; ushort elEntriesInDirectory; uint elDirectorySize; uint elDirectoryOffset; ushort elCommentLength; if( elCommentLength > 0 ) char elComment[ elCommentLength ]; } ZIPENDLOCATOR; //-------------------------------------------- // Custom read functions that allows the name of the // of the file to appear in the Template Results. string ReadZIPFILERECORD( ZIPFILERECORD &file ) { if( exists( file.frFileName ) ) return file.frFileName; else return ""; } string ReadZIPDIRENTRY( ZIPDIRENTRY &entry ) { if( exists( entry.deFileName ) ) return entry.deFileName; else return ""; } // Custom write function that allows changing // the name of the file - note that the file // name size cannot be increased void WriteZIPFILERECORD( ZIPFILERECORD &file, string s ) { local int len = Strlen( s ); if( exists( file.frFileName ) ) { Strncpy( file.frFileName, s, file.frFileNameLength ); if( len < file.frFileNameLength ) file.frFileName[len] = 0; //null terminate } } //-------------------------------------------- // Define the file local uint tag; LittleEndian(); while( !FEof() ) { // Read a tag tag = ReadUInt( FTell() ); // Read data depending upon tag - should start with 'PK'. // Note that when duplicate variables are defined, they // are made into an array (see 'Using Templates and Structs' // in the help file). if( tag == S_ZIPFILERECORD ) { SetBackColor( cLtGray ); ZIPFILERECORD record; if (record.frExtraFieldLength > 0 && record.frExtraField.efHeaderID == EH_Zip64) { //Printf("%Lu", record.frExtraField.efCompressedSize); FSkip(record.frExtraField.efCompressedSize); } } else if( tag == S_ZIPDATADESCR ) { SetBackColor( cLtGreen ); ZIPDATADESCR dataDescr; } else if( tag == S_ZIPDIRENTRY ) { SetBackColor( cLtPurple ); ZIPDIRENTRY dirEntry; } else if( tag == S_ZIPDIGITALSIG ) { SetBackColor( cLtBlue ); ZIPDIGITALSIG digitalSig; } else if( tag == S_ZIP64ENDLOCATORRECORD ) { SetBackColor( cYellow ); ZIP64ENDLOCATORRECORD end64Locator; } else if( tag == S_ZIP64ENDLOCATOR ) { SetBackColor( cDkYellow ); ZIP64ENDLOCATOR end64Locator; } else if( tag == S_ZIPENDLOCATOR ) { SetBackColor( cLtYellow ); ZIPENDLOCATOR endLocator; } else { Warning( "Unknown ZIP tag encountered. Template stopped." ); return -1; } }