Skip to content

Commit 8532971

Browse files
danila-schelkovluben
danila-schelkov
authored and
luben
committed
fix: decompress(byte[]) method now decompresses all frames, new tests added
1 parent 051e8de commit 8532971

7 files changed

Lines changed: 166 additions & 21 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ Android support
126126
---------------
127127

128128
Zstd-jni is usable in Android applications by importing the sources in Android
129-
Studio. I guess using git sub-modules will also work.
129+
Studio. I guess using git submodules will also work.
130130

131131
Android archive (*zstd-jni.aar*) is also published on maven central. You will need
132132
to add the repository in your build.gradle, e.g.:

‎src/main/java/com/github/luben/zstd/Zstd.java‎

Lines changed: 74 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
package com.github.luben.zstd;
22

3-
import java.nio.ByteBuffer;
4-
import java.util.Arrays;
5-
63
import com.github.luben.zstd.util.Native;
7-
import com.github.luben.zstd.ZstdCompressCtx;
8-
import com.github.luben.zstd.ZstdDecompressCtx;
4+
5+
import java.nio.ByteBuffer;
6+
import java.util.ArrayList;
7+
import java.util.List;
98

109
public class Zstd {
1110
static {
@@ -1387,8 +1386,8 @@ public static ByteBuffer compress(ByteBuffer srcBuff, ZstdDictCompress dict) {
13871386
*
13881387
* @param src the source buffer
13891388
* @param originalSize the maximum size of the uncompressed data.
1390-
* If originaSize is greater than the actuall uncompressed size, additional memory copy going to happen.
1391-
* If originalSize is smaller than the uncompressed size, ZstdExeption will be thrown.
1389+
* If originalSize is greater than the actual uncompressed size, additional memory copy going to happen.
1390+
* If originalSize is smaller than the uncompressed size, {@link ZstdException} will be thrown.
13921391
* @return byte array with the decompressed data
13931392
*/
13941393
public static byte[] decompress(byte[] src, int originalSize) {
@@ -1401,13 +1400,67 @@ public static byte[] decompress(byte[] src, int originalSize) {
14011400
}
14021401

14031402
/**
1404-
* Decompress data, assuming that whole buffer is a compressed data
1403+
* Decompress data, assuming that whole buffer is a compressed data.
1404+
* <p>
1405+
* Note, that file must be encoded with pledged content size, not using stream API.
1406+
* For more, see ZSTD_c_contentSizeFlag flag description.
1407+
* </p>
14051408
*
14061409
* @param src the source buffer
14071410
* @return byte array with the decompressed data
14081411
*/
14091412
public static byte[] decompress(byte[] src) {
1410-
return decompress(src, (int) getFrameContentSize(src));
1413+
List<FrameData> frames = new ArrayList<>();
1414+
1415+
int contentSize = calculateContentSizeAndFrames(src, frames);
1416+
1417+
byte[] decompressedData = new byte[contentSize];
1418+
1419+
int srcPosition = 0;
1420+
int decompressedPosition = 0;
1421+
for (int i = 0; i < frames.size(); i++) {
1422+
FrameData frameInfo = frames.get(i);
1423+
long size = decompressByteArray(decompressedData, decompressedPosition, (int) frameInfo.contentSize, src, srcPosition, (int) frameInfo.compressedSize);
1424+
if (Zstd.isError(size)) {
1425+
throw new ZstdException(size, String.format("error %s while decompressing %d frame", Zstd.getErrorName(size), i));
1426+
}
1427+
1428+
if (size != frameInfo.contentSize) {
1429+
throw new IllegalStateException("decompressed size mismatch");
1430+
}
1431+
1432+
srcPosition += (int) frameInfo.compressedSize;
1433+
decompressedPosition += (int) frameInfo.contentSize;
1434+
}
1435+
1436+
return decompressedData;
1437+
}
1438+
1439+
private static int calculateContentSizeAndFrames(byte[] src, List<FrameData> frames) {
1440+
int contentSize = 0;
1441+
1442+
int srcPosition = 0;
1443+
1444+
while (srcPosition < src.length) {
1445+
FrameData frameData = new FrameData(src, srcPosition);
1446+
1447+
if (Zstd.isError(frameData.contentSize)) {
1448+
// known error at the moment, but not for getErrorName
1449+
if (frameData.contentSize == -1) {
1450+
throw new ZstdException(frameData.contentSize, "Content size is unknown");
1451+
}
1452+
1453+
// otherwise let ZstdException get error message itself
1454+
throw new ZstdException(frameData.contentSize);
1455+
}
1456+
1457+
frames.add(frameData);
1458+
1459+
srcPosition += (int) frameData.compressedSize;
1460+
contentSize += (int) frameData.contentSize;
1461+
}
1462+
1463+
return contentSize;
14111464
}
14121465

14131466
/**
@@ -1417,8 +1470,8 @@ public static byte[] decompress(byte[] src) {
14171470
* @param srcOffset the start offset of 'src'
14181471
* @param srcSize the size of 'src'
14191472
* @param originalSize the maximum size of the uncompressed data.
1420-
* If originaSize is greater than the actuall uncompressed size, additional memory copy going to happen.
1421-
* If originalSize is smaller than the uncompressed size, ZstdExeption will be thrown.
1473+
* If originalSize is greater than the actual uncompressed size, additional memory copy going to happen.
1474+
* If originalSize is smaller than the uncompressed size, {@link ZstdException} will be thrown.
14221475
* @return byte array with the decompressed data
14231476
*/
14241477
public static byte[] decompressFrame(byte[] src, int srcOffset, int srcSize, int originalSize) {
@@ -1716,4 +1769,14 @@ static ByteBuffer getArrayBackedBuffer(BufferPool bufferPool, int size) throws Z
17161769
}
17171770
return buffer;
17181771
}
1772+
1773+
private static class FrameData {
1774+
final long contentSize;
1775+
final long compressedSize;
1776+
1777+
FrameData(byte[] src, int srcPosition) {
1778+
compressedSize = findFrameCompressedSize(src, srcPosition);
1779+
contentSize = getFrameContentSize(src, srcPosition, (int) compressedSize);
1780+
}
1781+
}
17191782
}

‎src/main/java/com/github/luben/zstd/ZstdDecompressCtx.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package com.github.luben.zstd;
22

33
import com.github.luben.zstd.util.Native;
4-
import com.github.luben.zstd.ZstdDictDecompress;
54

65
import java.nio.ByteBuffer;
76
import java.util.Arrays;
@@ -13,6 +12,7 @@ public class ZstdDecompressCtx extends AutoCloseBase {
1312
}
1413

1514
private long nativePtr = 0;
15+
// Note: keeps a reference to the dictionary so it's not garbage collected
1616
private ZstdDictDecompress decompression_dict = null;
1717

1818
private static native long init();
@@ -364,8 +364,8 @@ public int decompress(byte[] dst, byte[] src) {
364364
*
365365
* @param src the source buffer
366366
* @param originalSize the maximum size of the uncompressed data.
367-
* If originaSize is greater than the actuall uncompressed size, additional memory copy going to happen.
368-
* If originalSize is smaller than the uncompressed size, ZstdExeption will be thrown.
367+
* If originalSize is greater than the actual uncompressed size, additional memory copy going to happen.
368+
* If originalSize is smaller than the uncompressed size, {@link ZstdException} will be thrown.
369369
* @return byte array with the decompressed data
370370
*/
371371
public byte[] decompress(byte[] src, int originalSize) throws ZstdException {
@@ -379,8 +379,8 @@ public byte[] decompress(byte[] src, int originalSize) throws ZstdException {
379379
* @param srcOffset the start offset of 'src'
380380
* @param srcSize the size of 'src'
381381
* @param originalSize the maximum size of the uncompressed data.
382-
* If originaSize is greater than the actuall uncompressed size, additional memory copy going to happen.
383-
* If originalSize is smaller than the uncompressed size, ZstdExeption will be thrown.
382+
* If originalSize is greater than the actual uncompressed size, additional memory copy going to happen.
383+
* If originalSize is smaller than the uncompressed size, {@link ZstdException} will be thrown.
384384
* @return byte array with the decompressed data
385385
*/
386386
public byte[] decompress(byte[] src, int srcOffset, int srcSize, int originalSize) throws ZstdException {

‎src/test/resources/regenerate.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,7 @@ cat ./xml-1.zst ./xml-1.zst > ./xml-1x2.zst
66

77
# advanced compression check file
88
zstd --single-thread --no-check -o xml-advanced.zst -f --zstd=wlog=23,slog=4,tlen=32,mml=7,strat=7,hlog=16,clog=15 < xml
9+
10+
# generate compressed file with pledged size set (not using streaming api)
11+
zstd xml --single-thread --no-check -1 -o xml-1-sized.zst -f
12+
cat ./xml-1-sized.zst ./xml-1-sized.zst > ./xml-1-sizedx2.zst

‎src/test/resources/xml-1-sized.zst‎

678 KB
Binary file not shown.

‎src/test/resources/xml-1-sizedx2.zst‎

1.32 MB
Binary file not shown.

‎src/test/scala/Zstd.scala‎

Lines changed: 82 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ class ZstdSpec extends AnyFlatSpec with ScalaCheckPropertyChecks {
139139
}
140140

141141
compressedBuffer.flip()
142-
for ((level, compresedSize) <- levels.zip(compressedSizes)) {
142+
for ((level, compressedSize) <- levels.zip(compressedSizes)) {
143143
val oldCompressedPosition = compressedBuffer.position()
144144
val oldDecompressedPosition = decompressedBuffer.position()
145145

@@ -149,12 +149,12 @@ class ZstdSpec extends AnyFlatSpec with ScalaCheckPropertyChecks {
149149
//Note that the duplicate() call doesn't copy memory; it just makes a duplicate ByteBuffer pointing to the same
150150
//location in memory
151151
val thisChunkBuffer = compressedBuffer.duplicate()
152-
thisChunkBuffer.limit(thisChunkBuffer.position() + compresedSize)
152+
thisChunkBuffer.limit(thisChunkBuffer.position() + compressedSize)
153153
val decompressedSize = Zstd.decompress(decompressedBuffer, thisChunkBuffer)
154154

155155
assert(decompressedSize == input.length)
156156
compressedBuffer.position(thisChunkBuffer.position)
157-
assert(compressedBuffer.position() == oldCompressedPosition + compresedSize)
157+
assert(compressedBuffer.position() == oldCompressedPosition + compressedSize)
158158
assert(decompressedBuffer.position() == oldDecompressedPosition + size)
159159
}
160160

@@ -550,11 +550,89 @@ class ZstdSpec extends AnyFlatSpec with ScalaCheckPropertyChecks {
550550
sys.error(s"Failed")
551551
}
552552

553+
it should s"decompress using decompress(byte[]) function" in {
554+
val orig = new File("src/test/resources/xml")
555+
val file = new File(s"src/test/resources/xml-1-sized.zst")
556+
val fis = new FileInputStream(file)
557+
558+
val compressedBuff = fis.readAllBytes()
559+
560+
val original = Source.fromFile(orig)(Codec.ISO8859).map{char => char.toByte}.to(WrappedArray)
561+
562+
val buff = Zstd.decompress(compressedBuff)
563+
564+
if(original != buff.toSeq)
565+
sys.error(s"Failed")
566+
}
567+
568+
it should s"throw content size is unknown using decompress(byte[])function" in {
569+
val file = new File(s"src/test/resources/xml-1.zst")
570+
val fis = new FileInputStream(file)
571+
val compressedBuff = fis.readAllBytes()
572+
fis.close()
573+
574+
assertThrows[ZstdException] {
575+
Zstd.decompress(compressedBuff)
576+
}
577+
}
578+
579+
it should s"throw content size is unknown using decompressFrame(byte[]) function" in {
580+
val file = new File(s"src/test/resources/xml-1.zst")
581+
val fis = new FileInputStream(file)
582+
val compressedBuff = fis.readAllBytes()
583+
fis.close()
584+
585+
assertThrows[ZstdException] {
586+
Zstd.decompressFrame(compressedBuff)
587+
}
588+
}
589+
590+
it should s"decompress frame by frame using decompress(byte[]) function" in {
591+
val file = new File(s"src/test/resources/xml-1.zst")
592+
val fis = new FileInputStream(file)
593+
val compressedBuff = fis.readAllBytes()
594+
fis.close()
595+
596+
assertThrows[ZstdException] {
597+
Zstd.decompress(compressedBuff)
598+
}
599+
}
600+
601+
it should s"be able to consume 2 frames using decompress(byte[]) function" in {
602+
val orig = new File("src/test/resources/xmlx2")
603+
val file = new File(s"src/test/resources/xml-1-sizedx2.zst")
604+
val fis = new FileInputStream(file)
605+
606+
val compressedBuff = fis.readAllBytes()
607+
fis.close()
608+
609+
val original = Source.fromFile(orig)(Codec.ISO8859).map{char => char.toByte}.to(WrappedArray)
610+
611+
val buff = Zstd.decompress(compressedBuff)
612+
613+
if(original != buff.toSeq)
614+
sys.error(s"Failed")
615+
}
616+
617+
it should s"decompress using decompressFrame(byte[]) function" in {
618+
val orig = new File("src/test/resources/xml")
619+
val file = new File(s"src/test/resources/xml-1-sized.zst")
620+
val fis = new FileInputStream(file)
621+
622+
val compressedBuff = fis.readAllBytes()
623+
fis.close()
624+
625+
val original = Source.fromFile(orig)(Codec.ISO8859).map{char => char.toByte}.to(WrappedArray)
626+
627+
val buff = Zstd.decompressFrame(compressedBuff)
628+
629+
if(original != buff.toSeq)
630+
sys.error(s"Failed")
631+
}
553632

554633
it should s"be able to consume 2 frames in a file compressed by the zstd binary" in {
555634
val orig = new File("src/test/resources/xmlx2")
556635
val file = new File(s"src/test/resources/xml-1x2.zst")
557-
val fis = new FileInputStream(file)
558636

559637
val channel = FileChannel.open(file.toPath, StandardOpenOption.READ)
560638
val zis = new ZstdDirectBufferDecompressingStream(channel.map(MapMode.READ_ONLY, 0, channel.size))

0 commit comments

Comments
 (0)