Skip to content

Commit c523c93

Browse files
Cyan4973Cyan4973
committed
improved and fixed --list command, original patches by @ib (facebook#772)
accepts all skippable frame identifiers. display in MB or KB, depending on frame size. fixed combination of skippable and zstd frames.
1 parent 3f54d78 commit c523c93

File tree

2 files changed

+42
-54
lines changed

2 files changed

+42
-54
lines changed

‎NEWS

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
v1.3.1
22
perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt
33
perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
4+
cli : improved and fixed --list command, by @ib (#772)
45
build: fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (#718)
56
API exp : breaking change : ZSTD_getframeHeader() provides more information
67
API exp : breaking change : pinned down values of error codes

‎programs/fileio.c

+41-54
Original file line numberDiff line numberDiff line change
@@ -947,6 +947,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
947947
return 3;
948948
}
949949
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
950+
950951
/* begin analyzing frame */
951952
for ( ; ; ) {
952953
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
@@ -966,37 +967,31 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
966967
break;
967968
}
968969
}
969-
{
970-
U32 const magicNumber = MEM_readLE32(headerBuffer);
970+
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
971+
/* Zstandard frame */
971972
if (magicNumber == ZSTD_MAGICNUMBER) {
972973
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
973974
if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) {
974975
info->decompUnavailable = 1;
975-
}
976-
else {
976+
} else {
977977
info->decompressedSize += frameContentSize;
978978
}
979-
{
980-
/* move to the end of the frame header */
981-
size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
979+
/* move to the end of the frame header */
980+
{ size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead);
982981
if (ZSTD_isError(headerSize)) {
983982
DISPLAY("Error: could not determine frame header size\n");
984983
detectError = 1;
985984
break;
986985
}
987-
{
988-
int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
986+
{ int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR);
989987
if (ret != 0) {
990988
DISPLAY("Error: could not move to end of frame header\n");
991989
detectError = 1;
992990
break;
993-
}
994-
}
995-
}
991+
} } }
996992

997993
/* skip the rest of the blocks in the frame */
998-
{
999-
int lastBlock = 0;
994+
{ int lastBlock = 0;
1000995
do {
1001996
BYTE blockHeaderBuffer[3];
1002997
U32 blockHeader;
@@ -1009,24 +1004,20 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
10091004
}
10101005
blockHeader = MEM_readLE24(blockHeaderBuffer);
10111006
lastBlock = blockHeader & 1;
1012-
blockSize = blockHeader >> 3;
1013-
{
1014-
int const ret = fseek(srcFile, blockSize, SEEK_CUR);
1007+
blockSize = blockHeader >> 3; /* Warning : does not work when block is RLE type */
1008+
{ int const ret = fseek(srcFile, blockSize, SEEK_CUR);
10151009
if (ret != 0) {
10161010
DISPLAY("Error: could not skip to end of block\n");
10171011
detectError = 1;
10181012
break;
1019-
}
1020-
}
1013+
} }
10211014
} while (lastBlock != 1);
10221015

1023-
if (detectError) {
1024-
break;
1025-
}
1016+
if (detectError) break;
10261017
}
1027-
{
1028-
/* check if checksum is used */
1029-
BYTE const frameHeaderDescriptor = headerBuffer[4];
1018+
1019+
/* check if checksum is used */
1020+
{ BYTE const frameHeaderDescriptor = headerBuffer[4];
10301021
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
10311022
if (contentChecksumFlag) {
10321023
int const ret = fseek(srcFile, 4, SEEK_CUR);
@@ -1035,64 +1026,60 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
10351026
DISPLAY("Error: could not skip past checksum\n");
10361027
detectError = 1;
10371028
break;
1038-
}
1039-
}
1040-
}
1029+
} } }
10411030
info->numActualFrames++;
10421031
}
1043-
else if (magicNumber == ZSTD_MAGIC_SKIPPABLE_START) {
1044-
BYTE frameSizeBuffer[4];
1045-
size_t const readBytes = fread(frameSizeBuffer, 1, 4, srcFile);
1046-
if (readBytes != 4) {
1047-
DISPLAY("There was an error reading skippable frame size");
1032+
/* Skippable frame */
1033+
else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
1034+
U32 const frameSize = MEM_readLE32(headerBuffer + 4);
1035+
long const seek = -numBytesRead + 8 + frameSize;
1036+
int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR);
1037+
if (ret != 0) {
1038+
DISPLAY("Error: could not find end of skippable frame\n");
10481039
detectError = 1;
10491040
break;
10501041
}
1051-
{
1052-
U32 const frameSize = MEM_readLE32(frameSizeBuffer);
1053-
int const ret = LONG_SEEK(srcFile, frameSize, SEEK_CUR);
1054-
if (ret != 0) {
1055-
DISPLAY("Error: could not find end of skippable frame\n");
1056-
detectError = 1;
1057-
break;
1058-
}
1059-
}
10601042
info->numSkippableFrames++;
10611043
}
1044+
/* unknown content */
10621045
else {
10631046
detectError = 2;
10641047
break;
10651048
}
10661049
}
1067-
}
1050+
} /* end analyzing frame */
10681051
fclose(srcFile);
10691052
return detectError;
10701053
}
10711054

10721055
static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLevel){
1073-
double const compressedSizeMB = (double)info->compressedSize/(1 MB);
1074-
double const decompressedSizeMB = (double)info->decompressedSize/(1 MB);
1056+
unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB);
1057+
const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB";
1058+
double const compressedSizeUnit = (double)info->compressedSize / unit;
1059+
double const decompressedSizeUnit = (double)info->decompressedSize / unit;
10751060
double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize;
10761061
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
10771062
if (displayLevel <= 2) {
10781063
if (!info->decompUnavailable) {
10791064
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
1080-
DISPLAYOUT("%9d %13d %7.2f MB %9.2f MB %5.3f %5s %s\n",
1081-
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, decompressedSizeMB,
1065+
DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
1066+
info->numSkippableFrames, info->numActualFrames,
1067+
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
10821068
ratio, checkString, inFileName);
1083-
}
1084-
else {
1069+
} else {
10851070
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
10861071
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
1087-
info->numSkippableFrames, info->numActualFrames, compressedSizeMB, checkString, inFileName);
1072+
info->numSkippableFrames, info->numActualFrames,
1073+
compressedSizeUnit, checkString, inFileName);
10881074
}
1089-
}
1090-
else{
1075+
} else {
10911076
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
10921077
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
1093-
DISPLAYOUT("Compressed Size: %.2f MB (%llu B)\n", compressedSizeMB, info->compressedSize);
1078+
DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
1079+
compressedSizeUnit, unitStr, info->compressedSize);
10941080
if (!info->decompUnavailable) {
1095-
DISPLAYOUT("Decompressed Size: %.2f MB (%llu B)\n", decompressedSizeMB, info->decompressedSize);
1081+
DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
1082+
decompressedSizeUnit, unitStr, info->decompressedSize);
10961083
DISPLAYOUT("Ratio: %.4f\n", ratio);
10971084
}
10981085
DISPLAYOUT("Check: %s\n", checkString);

0 commit comments

Comments
 (0)