feat: Add a pixel comparison method for codecs that don’t generate identical outputs

rsanchez87 · rsanchez87 · commit acee87a08bbb · 2025-04-17T10:47:42.000+02:00
-Old codecs like MPEG-2 Video do not use a standarized IDCT (MPEG-C Part1) causing mismatches
in the generated output.
-This new method allows creating tests suites  that will compare the output pixel by pixel with a tolerance range
diff --git a/check/dummy.json b/check/dummy.json
@@ -11,5 +11,6 @@
             "output_format": "yuv420p",
             "result": "19b8d716307ae8b28c81b21f14f870dc"
         }
-    ]
-}
+    ],
+    "test_method": "md5"
+}
diff --git a/fluster/decoder.py b/fluster/decoder.py
@@ -18,7 +18,7 @@
 from abc import ABC, abstractmethod
 from functools import lru_cache
 from shutil import which
-from typing import List, Type
+from typing import List, Optional, Type
 
 from fluster.codec import Codec, OutputFormat
 from fluster.utils import normalize_binary_cmd
@@ -32,6 +32,7 @@ class Decoder(ABC):
     hw_acceleration = False
     description = ""
     binary = ""
+    is_reference = False
 
     def __init__(self) -> None:
         if self.binary:
@@ -76,3 +77,16 @@ def register_decoder(cls: Type[Decoder]) -> Type[Decoder]:
     DECODERS.append(cls())
     DECODERS.sort(key=lambda dec: dec.name)
     return cls
+
+
+def get_reference_decoder_for_codec(codec: Codec) -> Optional["Decoder"]:
+    """Find the reference decoder for a specific codec"""
+
+    reference_decoders = [d for d in DECODERS if d.codec == codec and d.is_reference]
+
+    if not reference_decoders:
+        return None
+    if len(reference_decoders) > 1:
+        print(f"Multiple reference decoders found for codec {codec.name}")
+
+    return reference_decoders[0]
diff --git a/fluster/decoders/iso_mpeg2_aac.py b/fluster/decoders/iso_mpeg2_aac.py
@@ -30,6 +30,7 @@ class ISOAACDecoder(Decoder):
     description = "ISO MPEG2 AAC reference decoder"
     codec = Codec.AAC
     binary = "aacdec_mc"
+    is_reference = True
 
     def decode(
         self,
diff --git a/fluster/decoders/iso_mpeg2_video.py b/fluster/decoders/iso_mpeg2_video.py
@@ -0,0 +1,85 @@
+# Fluster - testing framework for decoders conformance
+# Copyright (C) 2024, Fluendo, S.A.
+#  Author: Rubén Sánchez <rsanchez@fluendo.com>, Fluendo, S.A.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <https://www.gnu.org/licenses/>.
+import glob
+import os
+import tempfile
+
+from fluster.codec import Codec, OutputFormat
+from fluster.decoder import Decoder, register_decoder
+from fluster.utils import file_checksum, run_command
+
+
+@register_decoder
+class ISOMPEG2VDecoder(Decoder):
+    """ISO MPEG2 Video reference decoder implementation"""
+
+    name = "ISO-MPEG2-VIDEO"
+    description = "ISO MPEG2 Video reference decoder"
+    codec = Codec.MPEG2_VIDEO
+    binary = "mpeg2decode"
+    is_reference = True
+
+    def decode(
+        self,
+        input_filepath: str,
+        output_filepath: str,
+        output_format: OutputFormat,
+        timeout: int,
+        verbose: bool,
+        keep_files: bool,
+    ) -> str:
+        """Decodes input_filepath in output_filepath"""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            run_command(
+                [self.binary, "-b", input_filepath, "-f", "-r", "-o0", os.path.join(temp_dir, "rec%d")],
+                timeout=timeout,
+                verbose=verbose,
+            )
+            self._merge_yuv_files(temp_dir, output_filepath)
+            checksum = file_checksum(output_filepath)
+
+        if not keep_files:
+            os.remove(output_filepath)
+
+        return checksum
+
+    @staticmethod
+    def _merge_yuv_files(input_dir: str, output_filepath: str) -> None:
+        """Merge YUV frames into an only raw .yuv file for mpeg2 video test suite"""
+        num_frames = len(glob.glob(os.path.join(input_dir, "rec*.Y")))
+
+        if num_frames == 0:
+            raise ValueError("No frames were decoded")
+
+        with open(output_filepath, "wb") as output_file:
+            for frame_num in range(num_frames):
+                frame_name = f"rec{frame_num}"
+                y_file = os.path.join(input_dir, f"{frame_name}.Y")
+                u_file = os.path.join(input_dir, f"{frame_name}.U")
+                v_file = os.path.join(input_dir, f"{frame_name}.V")
+
+                if not (os.path.exists(y_file) and os.path.exists(u_file) and os.path.exists(v_file)):
+                    print(f"Warning: Files for frame {frame_name} not found in {input_dir}")
+                    continue
+
+                chunk_size = 1024
+                for plane_file in [y_file, u_file, v_file]:
+                    with open(plane_file, "rb") as file:
+                        chunk = file.read(chunk_size)
+                        while chunk:
+                            output_file.write(chunk)
+                            chunk = file.read(chunk_size)
diff --git a/fluster/decoders/iso_mpeg4_aac.py b/fluster/decoders/iso_mpeg4_aac.py
@@ -30,6 +30,7 @@ class ISOAACDecoder(Decoder):
     description = "ISO MPEG4 AAC reference decoder"
     codec = Codec.AAC
     binary = "mp4audec_mc"
+    is_reference = True
 
     def decode(
         self,
diff --git a/fluster/decoders/iso_mpeg4_aac_er.py b/fluster/decoders/iso_mpeg4_aac_er.py
@@ -30,6 +30,7 @@ class ISOAACDecoder(Decoder):
     description = "ISO MPEG4 AAC ER reference decoder"
     codec = Codec.AAC
     binary = "mp4audec"
+    is_reference = True
 
     def decode(
         self,
diff --git a/fluster/fluster.py b/fluster/fluster.py
@@ -31,7 +31,7 @@
 from fluster.decoders import *  # noqa: F403
 from fluster.decoders.av1_aom import AV1AOMDecoder
 from fluster.test_suite import Context as TestSuiteContext
-from fluster.test_suite import TestSuite
+from fluster.test_suite import TestMethod, TestSuite
 from fluster.test_vector import TestVector, TestVectorResult
 
 
@@ -278,6 +278,8 @@ def run_test_suites(self, ctx: Context) -> None:
                         decoder.multiple_layers = True
                 if decoder.codec != test_suite.codec:
                     continue
+                if test_suite.test_method == TestMethod.PIXEL and decoder.is_reference:
+                    continue
                 test_suite_res = test_suite.run(
                     ctx.to_test_suite_context(
                         decoder,
diff --git a/fluster/test.py b/fluster/test.py
@@ -6,24 +6,17 @@
 # modify it under the terms of the GNU Lesser General Public License
 # as published by the Free Software Foundation, either version 3
 # of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library. If not, see <https://www.gnu.org/licenses/>.
 
 import os
 import unittest
+from abc import abstractmethod
 from subprocess import TimeoutExpired
 from time import perf_counter
 from typing import Any
 
 from fluster.decoder import Decoder
 from fluster.test_vector import TestVector, TestVectorResult
-from fluster.utils import normalize_path
+from fluster.utils import compare_byte_wise_files, normalize_path
 
 
 class Test(unittest.TestCase):
@@ -51,57 +44,160 @@ def __init__(
         self.timeout = timeout
         self.keep_files = keep_files
         self.verbose = verbose
-        setattr(self, test_vector.name, self._test)
+        self._keep_files_during_test = False
+        self.test_vector_result = self.test_suite.test_vectors[self.test_vector.name]
+
+        # Set up the test method
+        setattr(self, test_vector.name, self._test_wrapper)
         super().__init__(test_vector.name)
 
+        # Initialize file paths
+        self._initialize_file_paths()
+
+    def _initialize_file_paths(self) -> None:
+        """Initialize input and output file paths."""
+        self.output_filepath = normalize_path(os.path.join(self.output_dir, self.test_vector.name + ".out"))
+
+        input_dir = os.path.join(self.resources_dir, self.test_suite.name)
+
+        if not self.test_suite.is_single_archive:
+            input_dir = os.path.join(input_dir, self.test_vector.name)
+
+        self.input_filepath = normalize_path(os.path.join(input_dir, self.test_vector.input_file))
+
+    def _execute_decode(self) -> str:
+        """Execute the decoder and return the result."""
+        keep_files_for_decode = self._keep_files_during_test or self.keep_files
+
+        return self.decoder.decode(
+            self.input_filepath,
+            self.output_filepath,
+            self.test_vector.output_format,
+            self.timeout,
+            self.verbose,
+            keep_files_for_decode,
+        )
+
+    def _cleanup_if_needed(self) -> None:
+        """Clean up output files if keep_files is False."""
+        if not self.keep_files and os.path.exists(self.output_filepath):
+            os.remove(self.output_filepath)
+
+    def _test_wrapper(self) -> None:
+        try:
+            self._test()
+        finally:
+            self._cleanup_if_needed()
+
     def _test(self) -> None:
+        """Execute the test and process results."""
         if self.skip:
-            self.test_suite.test_vectors[self.test_vector.name].test_result = TestVectorResult.NOT_RUN
+            self.test_vector_result.test_result = TestVectorResult.NOT_RUN
             return
 
-        output_filepath = os.path.join(self.output_dir, self.test_vector.name + ".out")
-
-        input_filepath = os.path.join(
-            self.resources_dir,
-            self.test_suite.name,
-            (self.test_vector.name if not self.test_suite.is_single_archive else ""),
-            self.test_vector.input_file,
-        )
-        output_filepath = normalize_path(output_filepath)
-        input_filepath = normalize_path(input_filepath)
+        start = perf_counter()
 
         try:
-            start = perf_counter()
-            result = self.decoder.decode(
-                input_filepath,
-                output_filepath,
-                self.test_vector.output_format,
-                self.timeout,
-                self.verbose,
-                self.keep_files,
-            )
-            self.test_suite.test_vectors[self.test_vector.name].test_time = perf_counter() - start
+            result = self._execute_decode()
+            self.test_vector_result.test_time = perf_counter() - start
         except TimeoutExpired:
-            self.test_suite.test_vectors[self.test_vector.name].test_result = TestVectorResult.TIMEOUT
-            self.test_suite.test_vectors[self.test_vector.name].test_time = perf_counter() - start
+            self.test_vector_result.test_result = TestVectorResult.TIMEOUT
+            self.test_vector_result.test_time = perf_counter() - start
             raise
         except Exception:
-            self.test_suite.test_vectors[self.test_vector.name].test_result = TestVectorResult.ERROR
-            self.test_suite.test_vectors[self.test_vector.name].test_time = perf_counter() - start
+            self.test_vector_result.test_result = TestVectorResult.ERROR
+            self.test_vector_result.test_time = perf_counter() - start
             raise
 
-        if not self.keep_files and os.path.exists(output_filepath) and os.path.isfile(output_filepath):
-            os.remove(output_filepath)
-
-        if not self.reference:
-            self.test_suite.test_vectors[self.test_vector.name].test_result = TestVectorResult.FAIL
-            if self.test_vector.result.lower() == result.lower():
-                self.test_suite.test_vectors[self.test_vector.name].test_result = TestVectorResult.SUCCESS
-            self.assertEqual(
-                self.test_vector.result.lower(),
-                result.lower(),
-                self.test_vector.name,
-            )
+        if self.reference:
+            self.test_vector_result.test_result = TestVectorResult.REFERENCE
+            self.test_vector_result.result = result
         else:
-            self.test_suite.test_vectors[self.test_vector.name].test_result = TestVectorResult.REFERENCE
-            self.test_suite.test_vectors[self.test_vector.name].result = result
+            try:
+                self.compare_result(result)
+                self.test_vector_result.test_result = TestVectorResult.SUCCESS
+            except Exception:
+                self.test_vector_result.test_result = TestVectorResult.FAIL
+                raise
+
+    @abstractmethod
+    def compare_result(self, result: str) -> None:
+        """Compare the test result with the expected value.
+
+        Args:
+            result: The result string from the decoder
+        """
+
+
+class MD5ComparisonTest(Test):
+    """Test class for MD5 comparison"""
+
+    def compare_result(self, result: str) -> None:
+        """Compare MD5 hash results."""
+        expected = self.test_vector.result.lower()
+        actual = result.lower()
+
+        self.assertEqual(expected, actual, self.test_vector.name)
+
+
+class PixelComparisonTest(Test):
+    """Test class for pixel comparison"""
+
+    def __init__(
+        self,
+        decoder: Decoder,
+        test_suite: Any,  # can't use TestSuite type because of circular dependency
+        test_vector: TestVector,
+        skip: bool,
+        output_dir: str,
+        reference: bool,
+        timeout: int,
+        keep_files: bool,
+        verbose: bool,
+        reference_decoder: Decoder,
+    ):
+        super().__init__(
+            decoder,
+            test_suite,
+            test_vector,
+            skip,
+            output_dir,
+            reference,
+            timeout,
+            keep_files,
+            verbose,
+        )
+        self._keep_files_during_test = True
+        self.reference_decoder = reference_decoder
+        self.reference_filepath = normalize_path(os.path.join(self.output_dir, self.test_vector.name + "_ref.yuv"))
+
+    def _decode_reference(self) -> str:
+        """Decode the reference file."""
+        keep_files_for_decode = self._keep_files_during_test or self.keep_files
+
+        return self.reference_decoder.decode(
+            self.input_filepath,
+            self.reference_filepath,
+            self.test_vector.output_format,
+            self.timeout,
+            self.verbose,
+            keep_files_for_decode,
+        )
+
+    def _cleanup_if_needed(self) -> None:
+        super()._cleanup_if_needed()
+        if not self.keep_files and os.path.exists(self.reference_filepath):
+            os.remove(self.reference_filepath)
+
+    def compare_result(self, result: str) -> None:
+        """Compare decoded output with reference decoder output pixel-wise."""
+        reference_result = self._decode_reference()
+
+        if not os.path.exists(self.reference_filepath) and os.path.exists(reference_result):
+            self.reference_filepath = reference_result
+
+        comparison_result = compare_byte_wise_files(
+            self.reference_filepath, self.output_filepath, keep_files=self.keep_files
+        )
+
+        self.assertEqual(0, comparison_result, self.test_vector.name)
diff --git a/fluster/test_suite.py b/fluster/test_suite.py
diff --git a/fluster/utils.py b/fluster/utils.py
diff --git a/test_suites/mpeg2v/MPEG2_VIDEO-422.json b/test_suites/mpeg2v/MPEG2_VIDEO-422.json
diff --git a/test_suites/mpeg2v/MPEG2_VIDEO-MAIN.json b/test_suites/mpeg2v/MPEG2_VIDEO-MAIN.json

Original file line number	Diff line number	Diff line change
`@@ -11,5 +11,6 @@`
`11`	`11`	`"output_format": "yuv420p",`
`12`	`12`	`"result": "19b8d716307ae8b28c81b21f14f870dc"`
`13`	`13`	`}`
`14`		`- ]`
`15`		`-}`
	`14`	`+ ],`
	`15`	`+ "test_method": "md5"`
	`16`	`+}`