avcodec/hevc: workaround hevc-alpha videos generated by VideoToolbox

quink-black · quink-black · commit eedf8f0165fe · 2026-04-01T22:54:36.000+08:00
Apple VideoToolbox is the dominant producer of hevc-alpha videos, but
early versions generates non-standard VPS extensions that fail to
parse and return AVERROR_INVALIDDATA. Fix this by returning
AVERROR_PATCHWELCOME instead of AVERROR_INVALIDDATA for unsupported
VPS extension configurations. Setting poc_lsb_not_present for the
alpha layer in the fallback path when it has no direct dependency
on the base layer, so that IDR slices on the alpha layer won't
incorrectly read pic_order_cnt_lsb.

Fix #22384

Signed-off-by: Zhao Zhili &lt;zhilizhao@tencent.com&gt;
diff --git a/libavcodec/hevc/ps.c b/libavcodec/hevc/ps.c
@@ -610,8 +610,11 @@ static int decode_vps_ext(GetBitContext *gb, AVCodecContext *avctx, HEVCVPS *vps
         }
     }
     vps->num_output_layer_sets = vps->vps_num_layer_sets + vps->num_add_layer_sets;
-    if (vps->num_output_layer_sets != 2)
-        return AVERROR_INVALIDDATA;
+    if (vps->num_output_layer_sets != 2) {
+        av_log(avctx, AV_LOG_WARNING,
+               "Unsupported num_output_layer_sets: %d\n", vps->num_output_layer_sets);
+        return AVERROR_PATCHWELCOME;
+    }
 
     sub_layers_max_present = get_bits1(gb); // vps_sub_layers_max_minus1_present_flag
     if (sub_layers_max_present) {
@@ -677,7 +680,7 @@ static int decode_vps_ext(GetBitContext *gb, AVCodecContext *avctx, HEVCVPS *vps
 
     if (get_ue_golomb_31(gb) != 0 /* vps_num_rep_formats_minus1 */) {
         av_log(avctx, AV_LOG_ERROR, "Unexpected extra rep formats\n");
-        return AVERROR_INVALIDDATA;
+        return AVERROR_PATCHWELCOME;
     }
 
     vps->rep_format.pic_width_in_luma_samples  = get_bits(gb, 16);
@@ -895,8 +898,23 @@ int ff_hevc_decode_nal_vps(GetBitContext *gb, AVCodecContext *avctx,
     if (vps->vps_max_layers > 1 && get_bits1(gb)) { /* vps_extension_flag */
         int ret = decode_vps_ext(gb, avctx, vps, layer1_id_included);
         if (ret == AVERROR_PATCHWELCOME) {
-            vps->nb_layers = 1;
-            av_log(avctx, AV_LOG_WARNING, "Ignoring unsupported VPS extension\n");
+            /* If alpha layer info was already parsed, preserve it for alpha decoding */
+            if (!(avctx->err_recognition & (AV_EF_BITSTREAM | AV_EF_COMPLIANT)) &&
+                vps->nb_layers == 2 &&
+                vps->layer_id_in_nuh[1] &&
+                (vps->scalability_mask_flag & HEVC_SCALABILITY_AUXILIARY)) {
+                av_log(avctx, AV_LOG_WARNING,
+                       "Broken VPS extension, treating as alpha video\n");
+                /* If alpha layer has no direct dependency on base layer,
+                 * assume poc_lsb_not_present for the alpha layer, so that
+                 * IDR slices on that layer won't read pic_order_cnt_lsb.
+                 * This matches the behavior of Apple VideoToolbox encoders. */
+                if (!vps->num_direct_ref_layers[1])
+                    vps->poc_lsb_not_present |= 1 << 1;
+            } else {
+                vps->nb_layers = 1;
+                av_log(avctx, AV_LOG_WARNING, "Ignoring unsupported VPS extension\n");
+            }
             ret = 0;
         } else if (ret < 0)
             goto err;
