sub/sd_sbr: use instanced rendering to avoid blending in software

The instanced API allows a lot of work to be offloaded to the GPU and
makes software rasterization tolerable performance-wise in most cases.

This is similar to sd_ass but there are some substantial differences:
- It uses a BGRA atlas to allow drawing bitmap emoji.
- De-duplicates images internally to significantly reduce atlas size by
  reusing the same atlas image for different instances.
- Exploits bilinear interpolation to further reduce atlas size of some
  primitives. For example this allows reducing background boxes to one
  very stretched pixel.

Bumps subrandr requirement to `v1.1` since this API is not present in
earlier versions.

ci/build-win32.ps1

@@ -170,7 +170,7 @@ meson.override_dependency('aom', aom_dep)
 if (-not (Test-Path "$subprojects/subrandr")) {
     git clone https://github.com/afishhh/subrandr --depth 1 $subprojects/subrandr
     Set-Content -Path "$subprojects/subrandr/meson.build" -Value @"
-project('subrandr', 'c', version: '1.0.1')
+project('subrandr', 'c', version: '1.1.0')
 cargo = find_program('cargo', required: true)
 cc = meson.get_compiler('c')
 subrandr_build = custom_target(

meson.build

@@ -356,7 +356,7 @@ if features['libdl']
     dependencies += libdl
 endif
 
-subrandr = dependency('subrandr', version: '>= 1.0.0', required: get_option('subrandr'))
+subrandr = dependency('subrandr', version: '>= 1.1.0', required: get_option('subrandr'))
 features += {'subrandr': subrandr.found()}
 if features['subrandr']
     sources += files('demux/demux_sbr.c', 'sub/sd_sbr.c')

sub/osd.h

@@ -29,6 +29,8 @@ enum sub_bitmap_format {
     SUBBITMAP_EMPTY = 0,// no bitmaps; always has num_parts==0
     SUBBITMAP_LIBASS,   // A8, with a per-surface blend color (libass.color)
     SUBBITMAP_BGRA,     // IMGFMT_BGRA (MSB=A, LSB=B), scaled, premultiplied alpha
+    SUBBITMAP_SUBRANDR, // contains `sbr_output_image`s, must be converted to
+                        // SUBBITMAP_BGRA during packing
 
     SUBBITMAP_COUNT
 };
@@ -47,9 +49,14 @@ struct sub_bitmap {
     // with the bitmap pointer.)
     int src_x, src_y;
 
-    struct {
-        uint32_t color;
-    } libass;
+    union {
+        struct {
+            uint32_t color;
+        } libass;
+        struct {
+            const struct sbr_output_image *image;
+        } subrandr;
+    };
 };
 
 struct sub_bitmaps {

sub/packer.c

@@ -26,6 +26,9 @@
 
 #include <ass/ass.h>
 #include <ass/ass_types.h>
+#if HAVE_SUBRANDR
+#include <subrandr/subrandr.h>
+#endif
 
 #include "common/common.h"
 #include "packer.h"
@@ -36,6 +39,7 @@
 
 struct mp_sub_packer {
     struct sub_bitmap *cached_parts; // only for the array memory
+    struct sub_bitmap *cached_subrandr_images;
     struct mp_image *cached_img;
     struct sub_bitmaps cached_subs;
     bool cached_subs_valid;
@@ -321,3 +325,103 @@ void mp_sub_packer_pack_ass(struct mp_sub_packer *p, ASS_Image **image_lists,
     p->cached_subs.change_id = 0;
     p->cached_subs_valid = true;
 }
+
+#if HAVE_SUBRANDR
+// Pack the images in `res` into a BGRA8 atlas and populate `res->parts`
+// with instances of the images as described by `pass`.
+static bool pack_subrandr(struct mp_sub_packer *p, struct sub_bitmaps *res,
+                          struct sbr_instanced_raster_pass *pass)
+{
+    if (!pack(p, res, IMGFMT_BGRA))
+        return false;
+
+    int padding = p->packer->padding;
+    sbr_bgra8 *base = (sbr_bgra8 *)res->packed->planes[0];
+    int byte_stride = res->packed->stride[0];
+    int pixel_stride = byte_stride / 4;
+    struct sbr_output_instance *instances = sbr_instanced_raster_pass_get_instances(pass);
+
+    for (int n = 0; n < res->num_parts; n++) {
+        struct sub_bitmap *b = &res->parts[n];
+        sbr_output_image_rasterize_into(b->subrandr.image, pass, b->src_x, b->src_y,
+                                        base, res->packed_w, res->packed_h, pixel_stride);
+
+        void *pdata = base + b->src_y * pixel_stride + b->src_x;
+        fill_padding_4(pdata, b->w, b->h, byte_stride, padding);
+        b->bitmap = pdata;
+    }
+
+    res->parts = NULL;
+    res->num_parts = 0;
+    res->format = SUBBITMAP_BGRA;
+    for (struct sbr_output_instance *instance = instances; instance; instance = instance->next) {
+        if (!instance->base->user_data)
+            continue;
+
+        MP_TARRAY_GROW(p, p->cached_parts, res->num_parts);
+        res->parts = p->cached_parts;
+        struct sub_bitmap *inst_b = &res->parts[res->num_parts];
+        struct sub_bitmap *image_b = &p->cached_subrandr_images[(size_t)instance->base->user_data - 1];
+
+        *inst_b = (struct sub_bitmap){
+            .x = instance->dst_x,      .y = instance->dst_y,
+            .dw = instance->dst_width, .dh = instance->dst_height,
+            .w = instance->src_width,  .h = instance->src_height,
+            .src_x = image_b->src_x + instance->src_off_x,
+            .src_y = image_b->src_y + instance->src_off_y,
+            .bitmap = image_b->bitmap, .stride = byte_stride,
+        };
+        res->num_parts++;
+    }
+
+    return true;
+}
+
+// Pack the images in `pass` into a single image, make `out` point to it,
+// and populate `out->parts` to correctly describe all instances in `pass`.
+void mp_sub_packer_pack_sbr(struct mp_sub_packer *p, sbr_instanced_raster_pass *pass,
+                            struct sub_bitmaps *out)
+{
+    *out = (struct sub_bitmaps){.change_id = 1};
+    p->cached_subs_valid = false;
+
+    struct sub_bitmaps res = {
+        .change_id = (unsigned)p->cached_subs.change_id + 1,
+        .format = SUBBITMAP_SUBRANDR,
+        .parts = p->cached_parts,
+        .video_color_space = false,
+    };
+
+    struct sbr_output_instance *instances = sbr_instanced_raster_pass_get_instances(pass);
+    for (struct sbr_output_instance *instance = instances; instance; instance = instance->next) {
+        // If `user_data` is non-null then this base image was already appended.
+        if (instance->base->user_data)
+            continue;
+
+        MP_TARRAY_GROW(p, p->cached_subrandr_images, res.num_parts);
+        res.parts = p->cached_subrandr_images;
+        struct sub_bitmap *b = &res.parts[res.num_parts];
+        b->subrandr.image = instance->base;
+        b->w = instance->base->width, b->h = instance->base->height;
+        // Store the index of the `sub_bitmap` for this output image into `user_data`.
+        // This index is incremented by one to differentiate index `0` from an absent index.
+        // Will be read in `pack_subrandr` after packing to determine where each image
+        // was actually packed to in the atlas.
+        instance->base->user_data = (void *)(uintptr_t)(res.num_parts + 1);
+        res.num_parts++;
+    }
+
+    if (!pack_subrandr(p, &res, pass))
+        return;
+
+    *out = res;
+    p->cached_subs = res;
+    p->cached_subs_valid = true;
+}
+
+const struct sub_bitmaps *mp_sub_packer_get_cached(struct mp_sub_packer *p) {
+    if (p->cached_subs_valid)
+        return &p->cached_subs;
+    return NULL;
+}
+#endif

sub/packer.h

@@ -22,6 +22,8 @@
 
 #include <stdbool.h>
 
+#include "config.h"
+
 #include <ass/ass.h>
 #include <ass/ass_types.h>
 
@@ -32,4 +34,11 @@ void mp_sub_packer_pack_ass(struct mp_sub_packer *p, ASS_Image **image_lists,
                             int num_image_lists, bool changed, bool video_color_space,
                             int preferred_osd_format, struct sub_bitmaps *out);
 
+#if HAVE_SUBRANDR
+struct sbr_instanced_raster_pass;
+void mp_sub_packer_pack_sbr(struct mp_sub_packer *p, struct sbr_instanced_raster_pass *pass,
+                            struct sub_bitmaps *out);
+const struct sub_bitmaps *mp_sub_packer_get_cached(struct mp_sub_packer *p);
+#endif
+
 #endif

sub/sd_sbr.c

@@ -28,16 +28,16 @@
 #include "common/common.h"
 #include "demux/packet_pool.h"
 #include "demux/stheader.h"
+#include "sub/packer.h"
 #include "sub/osd.h"
-#include "video/mp_image.h"
 #include "sd.h"
 
 struct sd_sbr_priv {
     struct sbr_library *sbr_library;
     struct sbr_renderer *sbr_renderer;
     struct sbr_subtitles *sbr_subtitles;
-    struct mp_osd_res osd;
-    struct sub_bitmaps *bitmaps;
+    struct mp_osd_res prev_osd;
+    struct mp_sub_packer *packer;
 };
 
 static void enable_output(struct sd *sd, bool enable)
@@ -123,10 +123,7 @@ static int init(struct sd *sd)
     ctx->sbr_library = library;
     sbr_library_set_log_callback(ctx->sbr_library, mp_msg_sbr_log_callback, sd);
 
-    ctx->bitmaps = talloc_zero(ctx, struct sub_bitmaps);
-    ctx->bitmaps->format = SUBBITMAP_BGRA;
-    ctx->bitmaps->num_parts = 1;
-    ctx->bitmaps->parts = talloc_zero(ctx->bitmaps, struct sub_bitmap);
+    ctx->packer = mp_sub_packer_alloc(ctx);
 
     enable_output(sd, true);
 
@@ -173,8 +170,6 @@ static struct sub_bitmaps *get_bitmaps(struct sd *sd, struct mp_osd_res dim,
     struct sd_sbr_priv *ctx = sd->priv;
     struct mp_subtitle_opts *opts = sd->opts;
 
-    ctx->osd = dim;
-
     if (pts == MP_NOPTS_VALUE || !ctx->sbr_renderer || !ctx->sbr_subtitles)
         return NULL;
 
@@ -186,61 +181,59 @@ static struct sub_bitmaps *get_bitmaps(struct sd *sd, struct mp_osd_res dim,
         .video_height = (int32_t)(dim.h - dim.mt - dim.mb) << 6,
         .video_width = (int32_t)(dim.w - dim.ml - dim.mr) << 6,
     };
+    struct sbr_rect2i clip_rect = {
+        .min_x = 0, .min_y = 0,
+        .max_x = dim.w, .max_y = dim.h
+    };
 
     if (opts->sub_use_margins) {
         context.padding_top = (int32_t)dim.mt << 6;
         context.padding_bottom = (int32_t)dim.mb << 6;
         context.padding_left = (int32_t)dim.ml << 6;
         context.padding_right = (int32_t)dim.mr << 6;
+    } else {
+        clip_rect.max_x -= dim.ml + dim.mr;
+        clip_rect.max_y -= dim.mt + dim.mb;
     }
 
     unsigned t = lrint(pts * 1000);
 
-    struct sub_bitmaps *bitmaps = ctx->bitmaps;
-    struct sub_bitmap *bitmap = bitmaps->parts;
-
-    bool size_did_change = bitmap->w != dim.w || bitmap->h != dim.h;
-    if (size_did_change || sbr_renderer_did_change(ctx->sbr_renderer, &context, t)) {
-        talloc_free(bitmaps->packed);
-
-        bitmaps->packed = mp_image_alloc(IMGFMT_BGRA, dim.w, dim.h);
-        mp_require(bitmaps->packed);
-        bitmaps->packed_h = dim.h;
-        bitmaps->packed_w = dim.w;
-        bitmaps->packed->params.repr.alpha = PL_ALPHA_PREMULTIPLIED;
-        ++bitmaps->change_id;
-
-        bitmap->bitmap = bitmaps->packed->planes[0];
-        if (!opts->sub_use_margins) {
-            bitmap->x = dim.ml;
-            bitmap->y = dim.mt;
-        } else {
-            bitmap->x = 0;
-            bitmap->y = 0;
-        }
-        bitmap->w = dim.w;
-        bitmap->h = dim.h;
-        bitmap->dw = dim.w;
-        bitmap->dh = dim.h;
-        bitmap->stride = (*bitmaps->packed).stride[0];
+    struct sub_bitmaps res;
+    const struct sub_bitmaps *cached = mp_sub_packer_get_cached(ctx->packer);
 
+    bool redraw_required = ctx->prev_osd.w != dim.w || ctx->prev_osd.h != dim.h ||
+                           !cached || sbr_renderer_did_change(ctx->sbr_renderer, &context, t);
+    if (redraw_required) {
         sbr_renderer_set_subtitles(ctx->sbr_renderer, ctx->sbr_subtitles);
-        if (sbr_renderer_render(ctx->sbr_renderer, &context, t, bitmap->bitmap,
-                                dim.w, dim.h, bitmap->stride >> 2) < 0) {
+        sbr_instanced_raster_pass *pass =
+            sbr_renderer_render_instanced(ctx->sbr_renderer, &context, t, clip_rect, 0);
+        if (!pass) {
             const char *error = sbr_get_last_error_string();
             mp_err(sd->log, "Failed to render frame: %s\n", error);
             return NULL;
         }
-    }
 
-    return sub_bitmaps_copy(NULL, bitmaps);
+        ctx->prev_osd = dim;
+
+        mp_sub_packer_pack_sbr(ctx->packer, pass, &res);
+        sbr_instanced_raster_pass_finish(pass);
+
+        if (!opts->sub_use_margins)
+            for (int i = 0; i < res.num_parts; ++i) {
+                struct sub_bitmap *part = &res.parts[i];
+                part->x += dim.ml;
+                part->y += dim.mt;
+            }
+    } else
+        res = *cached;
+
+    return sub_bitmaps_copy(NULL, &res);
 }
 
 static void uninit(struct sd *sd)
 {
     struct sd_sbr_priv *ctx = sd->priv;
 
-    talloc_free(ctx->bitmaps->packed);
     enable_output(sd, false);
     if (ctx->sbr_subtitles)
         sbr_subtitles_destroy(ctx->sbr_subtitles);