svac3: pictures of several patches

.github/workflows/ci.yml

@@ -55,6 +55,7 @@ jobs:
           mkdir -p corpus_patch
           head -c 60000 tests/streams/cv608_synth_1080p_cbr2m.svac3 > corpus_patch/idr.svac3
           head -c 60000 tests/streams/cv608_smartp_1080p.svac3 > corpus_patch/library.svac3
+          head -c 60000 tests/streams/cv608_patch4_1080p.svac3 > corpus_patch/patches.svac3
           ./fuzz_patch -max_total_time=60 -max_len=65536 corpus_patch
 
   ffmpeg:

CMakeLists.txt

@@ -50,11 +50,15 @@ add_test(NAME test_dec
          COMMAND test_dec ${CMAKE_SOURCE_DIR}/tests/streams/cv608_synth_1080p_cbr2m.svac3)
 add_test(NAME test_dec_library
          COMMAND test_dec -library ${CMAKE_SOURCE_DIR}/tests/streams/cv608_smartp_1080p.svac3)
+add_test(NAME test_dec_patches
+         COMMAND test_dec -patches ${CMAKE_SOURCE_DIR}/tests/streams/cv608_patch4_1080p.svac3)
 add_executable(test_api tests/test_api.c)
 target_include_directories(test_api PRIVATE source/svac3 tools)
 target_link_libraries(test_api svac3d)
 add_test(NAME test_api
          COMMAND test_api ${CMAKE_SOURCE_DIR}/tests/streams/cv608_synth_1080p_cbr2m.svac3)
+add_test(NAME test_api_lost_patch
+         COMMAND test_api -lost ${CMAKE_SOURCE_DIR}/tests/streams/cv608_patch4_1080p.svac3)
 find_package(Python3 COMPONENTS Interpreter REQUIRED)
 foreach(stream ${SVAC3_TEST_STREAMS})
     get_filename_component(name ${stream} NAME_WE)

README.md

@@ -14,7 +14,8 @@ says what the decoder does instead.
 
 The Main profile (0x30) tool set that HiSilicon encoders emit:
 - 8-bit 4:2:0;
-- I and P pictures, one patch per picture;
+- I and P pictures, in one patch or several (uniform patches, with the
+  loop filters crossing patch boundaries);
 - MAEC entropy coding, SRCC coefficients, EIPM (66 intra modes) and MIPF;
 - TMVP, the median motion vector predictor, and skip/direct;
 - several reference pictures, and library pictures (the library stream
@@ -24,8 +25,9 @@ The Main profile (0x30) tool set that HiSilicon encoders emit:
 Every picture of the test corpus is bit-exact against the reference
 decoder (`tests/golden/*.framemd5`).
 
-Anything else (B pictures, non-displayed library pictures, several
-patches, the security layer, the High profile's tools) is refused with
+Anything else (B pictures, non-displayed library pictures, non-uniform
+patches or loop filters stopping at patch boundaries, the security layer,
+the High profile's tools) is refused with
 `SVAC3D_ERR_UNSUPPORTED` rather than decoded wrongly.
 
 ## Build

source/svac3/svac3_dec.c

@@ -37,8 +37,19 @@ struct svac3_dec {
     int64_t cycle;
     int doi_prev, have_prev;
     int64_t last_poi;
+
+    /* the picture whose patches are being decoded */
+    svac3_picture_t *open;
+    pic_t *open_slot;
+    int open_lib;
+    int open_keep[DPB_SLOTS];
+    svac3_refs_t open_refs;
+    int64_t open_doi, open_poi;
+    uint32_t open_index;
 };
 
+static void abandon(svac3_dec_t *d);
+
 svac3_dec_t *svac3_dec_create(void)
 {
     return calloc(1, sizeof(svac3_dec_t));
@@ -63,13 +74,15 @@ void svac3_dec_destroy(svac3_dec_t *d)
 {
     if (!d)
         return;
+    abandon(d);
     free_pics(d);
     free(d->rbsp);
     free(d);
 }
 
 void svac3_dec_reset(svac3_dec_t *d)
 {
+    abandon(d);
     for (int i = 0; i < DPB_SLOTS; i++)
         d->pic[i].used = 0;
     for (int i = 0; i < LIB_SLOTS; i++)
@@ -172,48 +185,43 @@ static int64_t clamp_dist(int64_t dist)
     return dist < -32768 ? -32768 : dist > 32767 ? 32767 : dist;
 }
 
+static void abandon(svac3_dec_t *d)
+{
+    svac3_picture_free(d->open);
+    d->open = NULL;
+}
+
 /* A library picture (library_stream_flag 1): an I picture kept, with its
  * library_picture_index, for display pictures to reference, and output
  * since only display library patches (type 17) are accepted. A display
- * picture naming another index finds no reference. */
-static int decode_library(svac3_dec_t *d, const uint8_t *rbsp, size_t rl, int64_t doi,
-                          int64_t poi, const svac3_frame_t **out)
+ * picture naming another index finds no reference. It is decoded into
+ * the slot not holding the current library picture, which gives way only
+ * once the new one has decoded. */
+static int start_library(svac3_dec_t *d)
 {
     svac3_hls_t *hls = &d->hls;
-    const uint32_t index = hls->ph.library_picture_index;
     int ret;
 
     if (hls->ph.picture_coding_type != 1)
         return SVAC3_ERR_UNSUPPORTED;           /* inter-coded library pictures */
-
-    /* decode into the slot not holding the current library picture, which
-     * gives way only once the new one has decoded */
     pic_t *cur = d->lib[0].used ? &d->lib[1] : &d->lib[0];
     cur->used = 0;
     if (!cur->buf && (ret = alloc_pic(cur, d->w, d->h)))
         return ret;
-    ret = svac3_patch_decode(hls, rbsp, rl, &cur->f, NULL, cur->tmv, &d->st);
-    if (ret)
+    if ((ret = svac3_picture_begin(hls, &cur->f, NULL, cur->tmv, &d->open)))
         return ret;
-    for (int i = 0; i < LIB_SLOTS; i++)
-        d->lib[i].used = 0;
-    cur->used = 1;
-    cur->lib_index = index;
-    cur->doi = doi;
-    cur->poi = poi;
-    d->last_poi = poi;
-    *out = &cur->f;
+    d->open_slot = cur;
+    d->open_lib = 1;
+    d->open_index = hls->ph.library_picture_index;
     return SVAC3_OK;
 }
 
-static int decode_picture(svac3_dec_t *d, const uint8_t *rbsp, size_t rl,
-                          const svac3_frame_t **out)
+/* Begin a picture at its first patch: DOI and POI, reference lists, a slot */
+static int start_picture(svac3_dec_t *d)
 {
     svac3_hls_t *hls = &d->hls;
     const svac3_sps_t *sps = &hls->sps;
-    int ret = svac3_patch_supported(hls);
-    if (ret)
-        return ret;
+    int ret;
 
     const int w = sps->pic_width_in_luma, h = sps->pic_height_in_luma;
     if (w != d->w || h != d->h) {
@@ -232,8 +240,10 @@ static int decode_picture(svac3_dec_t *d, const uint8_t *rbsp, size_t rl,
     }
     const int64_t doi = doi7 + 256 * d->cycle;
     const int64_t poi = doi + hls->ph.picture_output_delay;
+    d->open_doi = doi;
+    d->open_poi = poi;
     if (sps->library_stream_flag)
-        return decode_library(d, rbsp, rl, doi, poi, out);
+        return start_library(d);
 
     /* reference lists, and the pictures either of them keeps */
     ref_t list[2][SVAC3_MAX_REFS];
@@ -246,7 +256,7 @@ static int decode_picture(svac3_dec_t *d, const uint8_t *rbsp, size_t rl,
     }
     /* which pictures the lists keep; the buffer is pruned only once the
      * picture has decoded, so a rejected one costs no reference */
-    int keep[DPB_SLOTS];
+    int *keep = d->open_keep;
     for (int i = 0; i < DPB_SLOTS; i++) {
         const pic_t *p = &d->pic[i];
         keep[i] = 0;
@@ -255,20 +265,20 @@ static int decode_picture(svac3_dec_t *d, const uint8_t *rbsp, size_t rl,
                 keep[i] |= p->used && !list[l][j].lib && p->doi == list[l][j].key;
     }
 
-    svac3_refs_t refs;
-    memset(&refs, 0, sizeof(refs));
+    svac3_refs_t *refs = &d->open_refs;
+    memset(refs, 0, sizeof(*refs));
     if (hls->ph.picture_coding_type == 2) {
-        refs.num = nlist[0];
+        refs->num = nlist[0];
         for (int j = 0; j < nlist[0]; j++) {
             const ref_t *e = &list[0][j];
             const pic_t *r = e->lib ? find_lib(d, e->key) : find_doi(d, e->key);
             if (!r)
                 return SVAC3_ERR_CORRUPT;       /* reference not decoded */
-            refs.ref[j] = &r->f;
+            refs->ref[j] = &r->f;
             /* a library reference's DistanceIndex is 2 * (POI - 1) (5.4.3.3) */
-            refs.dist[j] = (int)clamp_dist(e->lib ? 2 : 2 * (poi - r->poi));
+            refs->dist[j] = (int)clamp_dist(e->lib ? 2 : 2 * (poi - r->poi));
             if (j == 0)
-                refs.col = r->tmv;
+                refs->col = r->tmv;
         }
     }
 
@@ -286,30 +296,87 @@ static int decode_picture(svac3_dec_t *d, const uint8_t *rbsp, size_t rl,
     cur->used = 0;                              /* its picture is about to be overwritten */
     if (!cur->buf && (ret = alloc_pic(cur, w, h)))
         return ret;
-
-    ret = svac3_patch_decode(hls, rbsp, rl, &cur->f, &refs, cur->tmv, &d->st);
-    if (ret)
+    if ((ret = svac3_picture_begin(hls, &cur->f, refs, cur->tmv, &d->open)))
         return ret;
-    for (int i = 0; i < DPB_SLOTS; i++)
-        d->pic[i].used = keep[i];
+    d->open_slot = cur;
+    d->open_lib = 0;
+    return SVAC3_OK;
+}
+
+/* The picture's last patch has decoded: the buffer takes it */
+static void commit(svac3_dec_t *d, const svac3_frame_t **out)
+{
+    pic_t *cur = d->open_slot;
+    if (d->open_lib) {
+        for (int i = 0; i < LIB_SLOTS; i++)
+            d->lib[i].used = 0;
+        cur->lib_index = d->open_index;
+    } else {
+        for (int i = 0; i < DPB_SLOTS; i++)
+            d->pic[i].used = d->open_keep[i];
+    }
     cur->used = 1;
-    cur->doi = doi;
-    cur->poi = poi;
-    d->last_poi = poi;
+    cur->doi = d->open_doi;
+    cur->poi = d->open_poi;
+    d->last_poi = d->open_poi;
     *out = &cur->f;
-    return SVAC3_OK;
+}
+
+/* One patch NAL unit. A picture that loses patches is dropped, and the
+ * unit that shows it (the next picture's first patch, or a patch out of
+ * order) returns SVAC3_ERR_CORRUPT. */
+static int decode_patch_nal(svac3_dec_t *d, const uint8_t *rbsp, size_t rl,
+                            const svac3_frame_t **out)
+{
+    svac3_hls_t *hls = &d->hls;
+    int ret = svac3_patch_supported(hls), lost = 0;
+    if (ret)
+        return ret;
+
+    if (hls->patch.patch_index == 0) {
+        if (d->open) {
+            abandon(d);
+            lost = 1;
+        }
+        if ((ret = start_picture(d)))
+            return ret;
+    } else if (!d->open) {
+        return SVAC3_ERR_CORRUPT;               /* its picture's first patch was lost */
+    }
+
+    ret = svac3_picture_patch(d->open, hls, rbsp, rl, &d->st);
+    if (ret < 0) {
+        abandon(d);
+        return ret;
+    }
+    if (ret == 1) {
+        svac3_picture_t *pic = d->open;
+        d->open = NULL;
+        if ((ret = svac3_picture_end(pic, hls)))
+            return ret;
+        commit(d, out);
+    }
+    return lost ? SVAC3_ERR_CORRUPT : SVAC3_OK;
 }
 
 int svac3_dec_nal(svac3_dec_t *d, const uint8_t *nal, size_t len, const svac3_frame_t **out)
 {
     *out = NULL;
     if (len < 2)
         return SVAC3_ERR_INVALID;
+    /* a picture still missing patches when a header that replaces its
+     * parameters arrives is lost */
+    const unsigned t = nal[1] & 31u;
+    int lost = 0;
+    if (d->open && (t == SVAC3_NAL_SPS || t == SVAC3_NAL_PPS || t == SVAC3_NAL_PIC_HEAD)) {
+        abandon(d);
+        lost = 1;
+    }
     int ret = svac3_parse_nal(&d->hls, nal, len, 0, NULL);
     if (ret)
         return ret;
-    if (!svac3_nal_is_patch(nal[1] & 31u))
-        return SVAC3_OK;
+    if (!svac3_nal_is_patch(t))
+        return lost ? SVAC3_ERR_CORRUPT : SVAC3_OK;
     if (len - 2 > d->rbsp_cap) {
         uint8_t *b = realloc(d->rbsp, len - 2);
         if (!b)
@@ -318,5 +385,5 @@ int svac3_dec_nal(svac3_dec_t *d, const uint8_t *nal, size_t len, const svac3_fr
         d->rbsp_cap = len - 2;
     }
     const size_t rl = svac3_unescape(nal + 2, len - 2, d->rbsp);
-    return decode_picture(d, d->rbsp, rl, out);
+    return decode_patch_nal(d, d->rbsp, rl, out);
 }

source/svac3/svac3_dec.h

@@ -25,7 +25,9 @@ void svac3_dec_destroy(svac3_dec_t *d);
  * svac3_dec_hls()->sps), and stays valid until the next call; otherwise
  * *out is NULL. Errors are the SVAC3_ERR_* codes: SVAC3_ERR_CORRUPT also
  * covers a P picture whose reference is not in the buffer, as when
- * decoding starts after the I picture. */
+ * decoding starts after the I picture, and a unit that reveals that an
+ * earlier picture lost a patch. Such a unit is itself decoded, and may
+ * complete a picture: *out is set alongside the error then. */
 int svac3_dec_nal(svac3_dec_t *d, const uint8_t *nal, size_t len, const svac3_frame_t **out);
 
 /* Drop every decoded picture (after a seek); parameter sets are kept */