lz4 · Cyan4973 · Jul 5, 2022 · Jun 3, 2022 · Jun 10, 2022 · Jun 10, 2022
diff --git a/contrib/meson/meson/examples/meson.build b/contrib/meson/meson/examples/meson.build
@@ -26,7 +26,7 @@ foreach e, src : examples
   executable(
     e,
     lz4_source_root / 'examples' / src,
-    dependencies: liblz4_dep,
+    dependencies: [liblz4_internal_dep],
     install: false
   )
 endforeach
diff --git a/contrib/meson/meson/lib/meson.build b/contrib/meson/meson/lib/meson.build
@@ -43,7 +43,7 @@ liblz4_dep = declare_dependency(
   include_directories: include_directories(lz4_source_root / 'lib')
 )
 
-if get_option('tests') or get_option('programs')
+if get_option('tests') or get_option('programs') or get_option('examples')
   liblz4_internal = static_library(
     'lz4-internal',
     objects: liblz4.extract_all_objects(recursive: true),

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
@@ -184,6 +184,8 @@ <h1>1.9.4 Manual</h1>
   This value is provided by LZ4F_compressBound().
   If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
   After an error, the state is left in a UB state, and must be re-initialized or freed.
+  If previously an uncompressed block was written, buffered data is flushed
+  before appending compressed data is continued.
  `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
@@ -342,6 +344,24 @@ <h1>1.9.4 Manual</h1>
 <pre><b>typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM)
               _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes;
 </b></pre><BR>
+<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+                                                  void* dstBuffer, size_t dstCapacity,
+                                                  const void* srcBuffer, size_t srcSize,
+                                                  const LZ4F_compressOptions_t* cOptPtr);
+</b><p>  LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+  Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+  no compression is done for this operation
+  If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+  After an error, the state is left in a UB state, and must be re-initialized or freed.
+  If previously a compressed block was written, buffered data is flushed
+  before appending uncompressed data is continued.
+  This is only supported when LZ4F_blockIndependent is used
+ `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+           or an error code if it fails (which can be tested using LZ4F_isError())
+
+</p></pre><BR>
+
 <a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre>
 
 <pre><b>LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
@@ -11,8 +11,9 @@
 #include <errno.h>
 #include <assert.h>
 
+#include <getopt.h>
 #include <lz4frame.h>
-
+#include <lz4frame_static.h>
 
 #define IN_CHUNK_SIZE  (16*1024)
 
@@ -57,10 +58,11 @@ static compressResult_t
 compress_file_internal(FILE* f_in, FILE* f_out,
                        LZ4F_compressionContext_t ctx,
                        void* inBuff,  size_t inChunkSize,
-                       void* outBuff, size_t outCapacity)
+                       void* outBuff, size_t outCapacity,
+                       FILE* f_unc, long uncOffset)
 {
     compressResult_t result = { 1, 0, 0 };  /* result for an error */
-    unsigned long long count_in = 0, count_out;
+    long long count_in = 0, count_out, bytesToOffset = -1;
 
     assert(f_in != NULL); assert(f_out != NULL);
     assert(ctx != NULL);
@@ -81,22 +83,48 @@ compress_file_internal(FILE* f_in, FILE* f_out,
 
     /* stream file */
     for (;;) {
-        size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
+      size_t compressedSize;
+      long long inSize = IN_CHUNK_SIZE;
+      if (uncOffset >= 0) {
+        bytesToOffset = uncOffset - count_in;
+
+        /* read only remaining bytes to offset position */
+        if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) {
+          inSize = bytesToOffset;
+        }
+      }
+
+      /* input data is at uncompressed data offset */
+      if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) {
+        size_t const readSize = fread(inBuff, 1, inSize, f_unc);
+        if (readSize == 0) {
+          uncOffset = -1;
+          continue;
+        }
+        count_in += readSize;
+        compressedSize = LZ4F_uncompressedUpdate(ctx,
+                                             outBuff, outCapacity,
+                                             inBuff, readSize,
+                                             NULL);
+      } else {
+        size_t const readSize = fread(inBuff, 1, inSize, f_in);
         if (readSize == 0) break; /* nothing left to read from input file */
         count_in += readSize;
-
-        size_t const compressedSize = LZ4F_compressUpdate(ctx,
+        compressedSize = LZ4F_compressUpdate(ctx,
                                                 outBuff, outCapacity,
                                                 inBuff, readSize,
                                                 NULL);
-        if (LZ4F_isError(compressedSize)) {
-            printf("Compression failed: error %u \n", (unsigned)compressedSize);
-            return result;
-        }
 
-        printf("Writing %u bytes\n", (unsigned)compressedSize);
-        safe_fwrite(outBuff, 1, compressedSize, f_out);
-        count_out += compressedSize;
+      }
+
+      if (LZ4F_isError(compressedSize)) {
+        printf("Compression failed: error %u \n", (unsigned)compressedSize);
+        return result;
+      }
+
+      printf("Writing %u bytes\n", (unsigned)compressedSize);
+      safe_fwrite(outBuff, 1, compressedSize, f_out);
+      count_out += compressedSize;
     }
 
     /* flush whatever remains within internal buffers */
@@ -120,7 +148,8 @@ compress_file_internal(FILE* f_in, FILE* f_out,
 }
 
 static compressResult_t
-compress_file(FILE* f_in, FILE* f_out)
+compress_file(FILE* f_in, FILE* f_out,
+              FILE* f_unc, int uncOffset)
 {
     assert(f_in != NULL);
     assert(f_out != NULL);
@@ -137,7 +166,8 @@ compress_file(FILE* f_in, FILE* f_out)
         result = compress_file_internal(f_in, f_out,
                                         ctx,
                                         src, IN_CHUNK_SIZE,
-                                        outbuff, outbufCapacity);
+                                        outbuff, outbufCapacity,
+                                        f_unc, uncOffset);
     } else {
         printf("error : resource allocation failed \n");
     }
@@ -305,52 +335,106 @@ static int decompress_file(FILE* f_in, FILE* f_out)
 }
 
 
-int compareFiles(FILE* fp0, FILE* fp1)
+int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset)
 {
     int result = 0;
+    long bytesRead = 0;
+    long bytesToOffset = -1;
+    long b1Size = 1024;
 
     while (result==0) {
+        char b1[b1Size];
+        size_t r1;
+        size_t bytesToRead = sizeof b1;
+        if (uncOffset >= 0) {
+          bytesToOffset = uncOffset - bytesRead;
+
+          /* read remainder to offset */
+          if (bytesToOffset < b1Size) {
+            bytesToRead = bytesToOffset;
+          }
+        }
+
         char b0[1024];
-        char b1[1024];
-        size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
-        size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
+        size_t r0;
+        if (bytesToOffset <= 0 && fpUnc) {
+          bytesToRead = sizeof b1;
+          r0 = fread(b0, 1,bytesToRead, fpUnc);
+        } else {
+          r0 = fread(b0, 1, bytesToRead, fp0);
+        }
+
+        r1 = fread(b1, 1, r0, fp1);
 
         result = (r0 != r1);
         if (!r0 || !r1) break;
         if (!result) result = memcmp(b0, b1, r0);
+
+        bytesRead += r1;
     }
 
     return result;
 }
 
 
-int main(int argc, const char **argv) {
+int main(int argc, char **argv) {
     char inpFilename[256] = { 0 };
     char lz4Filename[256] = { 0 };
     char decFilename[256] = { 0 };
 
+    int uncOffset = -1;
+    char uncFilename[256] = { 0 };
+    int opt;
+
     if (argc < 2) {
         printf("Please specify input filename\n");
-        return 0;
+        return EXIT_FAILURE;
     }
 
     snprintf(inpFilename, 256, "%s", argv[1]);
     snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
     snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
 
+    while ((opt = getopt(argc, argv, "o:d:")) != -1) {
+      switch (opt) {
+      case 'd':
+        snprintf(uncFilename, 256, "%s", optarg);
+        break;
+      case 'o':
+        uncOffset = atoi(optarg);
+        break;
+      default:
+        printf("usage: %s <input file> [-o <offset> -d <file>]\n", argv[0]);
+        printf("-o uncompressed data offset\n");
+        printf("   inject uncompressed data at this offset into the lz4 file\n");
+        printf("-d uncompressed file\n");
+        printf("   file to inject without compression into the lz4 file\n");
+        return EXIT_FAILURE;
+      }
+    }
+
     printf("inp = [%s]\n", inpFilename);
     printf("lz4 = [%s]\n", lz4Filename);
     printf("dec = [%s]\n", decFilename);
+    if (uncOffset > 0) {
+      printf("unc = [%s]\n", uncFilename);
+      printf("ofs = [%i]\n", uncOffset);
+    }
 
     /* compress */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const outFp = fopen(lz4Filename, "wb");
+        FILE* const uncFp = fopen(uncFilename, "rb");
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
-        compressResult_t const ret = compress_file(inpFp, outFp);
+        compressResult_t const ret = compress_file(
+            inpFp, outFp,
+            uncFp, uncOffset);
 
         fclose(outFp);
         fclose(inpFp);
+        if (uncFp)
+          fclose(uncFp);
 
         if (ret.error) {
             printf("compress : failed with code %i\n", ret.error);
@@ -383,12 +467,16 @@ int main(int argc, const char **argv) {
     /* verify */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const decFp = fopen(decFilename, "rb");
+        FILE* const uncFp = fopen(uncFilename, "rb");
 
         printf("verify : %s <-> %s\n", inpFilename, decFilename);
-        int const cmp = compareFiles(inpFp, decFp);
+        int const cmp = compareFiles(inpFp, decFp,
+                                     uncFp, uncOffset);
 
         fclose(decFp);
         fclose(inpFp);
+        if (uncFp)
+          fclose(uncFp);
 
         if (cmp) {
             printf("corruption detected : decompressed file differs from original\n");