vo_opengl: implement support for OOTFs and non-display referred content

This introduces (yet another..) mp_colorspace members, an enum `light`
(for lack of a better name) which basically tells us whether we're
dealing with scene-referred or display-referred light, but also a bit
more metadata (in which way is the scene-referred light expected to be
mapped to the display?).

The addition of this parameter accomplishes two goals:

1. Allows us to actually support HLG more-or-less correctly[1]
2. Allows people playing back direct “camera” content (e.g. v-log or
   s-log2) to treat it as scene-referred instead of display-referred

[1] Even better would be to use the display-referred OOTF instead of the
idealized OOTF, but this would require either native HLG support in
LittleCMS (unlikely) or more communication between lcms.c and
video_shaders.c than I'm remotely comfortable with

That being said, in principle we could switch our usage of the BT.1886
EOTF to the BT.709 OETF instead and treat BT.709 content as being
scene-referred under application of the 709+1886 OOTF; which moves that
particular conversion from the 3dlut to the shader code; but also allows
a) users like UliZappe to turn it off and b) supporting the full HLG
OOTF in the same framework. But I think I prefer things as they are
right now.

DOCS/man/input.rst

@@ -1459,6 +1459,9 @@ Property list
     ``video-params/sig-peak``
         The video file's tagged signal peak as float.
 
+    ``video-params/light``
+        The light type in use as a string. (Exact values subject to change.)
+
     ``video-params/chroma-location``
         Chroma location as string. (Exact values subject to change.)
 
@@ -1487,6 +1490,7 @@ Property list
             "primaries"         MPV_FORMAT_STRING
             "gamma"             MPV_FORMAT_STRING
             "sig-peak"          MPV_FORMAT_DOUBLE
+            "light"             MPV_FORMAT_STRING
             "chroma-location"   MPV_FORMAT_STRING
             "rotate"            MPV_FORMAT_INT64
             "stereo-in"         MPV_FORMAT_STRING

DOCS/man/vf.rst

@@ -376,6 +376,20 @@ Available mpv-only filters are:
 
         The default of 0.0 will default to the source's nominal peak luminance.
 
+    ``<light>``
+        Light type of the scene. This is mostly correctly inferred based on the
+        gamma function, but it can be useful to override this when viewing raw
+        camera footage (e.g. V-Log), which is normally scene-referred instead
+        of display-referred.
+
+        Available light types are:
+
+       :auto:         Automatic selection (default)
+       :display:      Display-referred light (most content)
+       :hlg:          Scene-referred using the HLG OOTF (e.g. HLG content)
+       :709-1886:     Scene-referred using the BT709+BT1886 interaction
+       :gamma1.2:     Scene-referred using a pure power OOTF (gamma=1.2)
+
     ``<stereo-in>``
         Set the stereo mode the video is assumed to be encoded in. Takes the
         same values as the ``--video-stereo-mode`` option.

player/command.c

@@ -2696,6 +2696,8 @@ static int property_imgparams(struct mp_image_params p, int action, void *arg)
         {"gamma",
             SUB_PROP_STR(m_opt_choice_str(mp_csp_trc_names, p.color.gamma))},
         {"sig-peak", SUB_PROP_FLOAT(p.color.sig_peak)},
+        {"light",
+            SUB_PROP_STR(m_opt_choice_str(mp_csp_light_names, p.color.light))},
         {"chroma-location",
             SUB_PROP_STR(m_opt_choice_str(mp_chroma_names, p.chroma_location))},
         {"stereo-in",

video/csputils.c

@@ -84,6 +84,15 @@ const struct m_opt_choice_alternatives mp_csp_trc_names[] = {
     {0}
 };
 
+const struct m_opt_choice_alternatives mp_csp_light_names[] = {
+    {"auto",        MP_CSP_LIGHT_AUTO},
+    {"display",     MP_CSP_LIGHT_DISPLAY},
+    {"hlg",         MP_CSP_LIGHT_SCENE_HLG},
+    {"709-1886",    MP_CSP_LIGHT_SCENE_709_1886},
+    {"gamma1.2",    MP_CSP_LIGHT_SCENE_1_2},
+    {0}
+};
+
 const char *const mp_csp_equalizer_names[MP_CSP_EQ_COUNT] = {
     "brightness",
     "contrast",
@@ -112,6 +121,8 @@ void mp_colorspace_merge(struct mp_colorspace *orig, struct mp_colorspace *new)
         orig->gamma = new->gamma;
     if (!orig->sig_peak)
         orig->sig_peak = new->sig_peak;
+    if (!orig->light)
+        orig->light = new->light;
 }
 
 // The short name _must_ match with what vf_stereo3d accepts (if supported).

video/csputils.h

@@ -87,6 +87,17 @@ enum mp_csp_trc {
 
 extern const struct m_opt_choice_alternatives mp_csp_trc_names[];
 
+enum mp_csp_light {
+    MP_CSP_LIGHT_AUTO,
+    MP_CSP_LIGHT_DISPLAY,
+    MP_CSP_LIGHT_SCENE_HLG,
+    MP_CSP_LIGHT_SCENE_709_1886,
+    MP_CSP_LIGHT_SCENE_1_2,
+    MP_CSP_LIGHT_COUNT
+};
+
+extern const struct m_opt_choice_alternatives mp_csp_light_names[];
+
 // These constants are based on the ICC specification (Table 23) and match
 // up with the API of LittleCMS, which treats them as integers.
 enum mp_render_intent {
@@ -121,6 +132,7 @@ struct mp_colorspace {
     enum mp_csp_levels levels;
     enum mp_csp_prim primaries;
     enum mp_csp_trc gamma;
+    enum mp_csp_light light;
     float sig_peak; // highest relative value in signal. 0 = unknown/auto
 };
 

video/filter/vf_format.c

@@ -39,6 +39,7 @@ struct vf_priv_s {
     int primaries;
     int gamma;
     float sig_peak;
+    int light;
     int chroma_location;
     int stereo_in;
     int stereo_out;
@@ -97,6 +98,8 @@ static int reconfig(struct vf_instance *vf, struct mp_image_params *in,
         out->color.gamma = p->gamma;
     if (p->sig_peak)
         out->color.sig_peak = p->sig_peak;
+    if (p->light)
+        out->color.light = p->light;
     if (p->chroma_location)
         out->chroma_location = p->chroma_location;
     if (p->stereo_in)
@@ -146,6 +149,7 @@ static const m_option_t vf_opts_fields[] = {
     OPT_CHOICE_C("primaries", primaries, 0, mp_csp_prim_names),
     OPT_CHOICE_C("gamma", gamma, 0, mp_csp_trc_names),
     OPT_FLOAT("sig-peak", sig_peak, 0),
+    OPT_CHOICE_C("light", light, 0, mp_csp_light_names),
     OPT_CHOICE_C("chroma-location", chroma_location, 0, mp_chroma_names),
     OPT_CHOICE_C("stereo-in", stereo_in, 0, mp_stereo3d_names),
     OPT_CHOICE_C("stereo-out", stereo_out, 0, mp_stereo3d_names),

video/mp_image.c

@@ -409,6 +409,7 @@ void mp_image_copy_attributes(struct mp_image *dst, struct mp_image *src)
     dst->params.color.primaries = src->params.color.primaries;
     dst->params.color.gamma = src->params.color.gamma;
     dst->params.color.sig_peak = src->params.color.sig_peak;
+    dst->params.color.light = src->params.color.light;
     if ((dst->fmt.flags & MP_IMGFLAG_YUV) == (src->fmt.flags & MP_IMGFLAG_YUV)) {
         dst->params.color.space = src->params.color.space;
         dst->params.color.levels = src->params.color.levels;
@@ -688,6 +689,16 @@ void mp_image_params_guess_csp(struct mp_image_params *params)
     // range as the signal peak to prevent clipping
     if (!params->color.sig_peak)
         params->color.sig_peak = mp_trc_nom_peak(params->color.gamma);
+
+    if (params->color.light == MP_CSP_LIGHT_AUTO) {
+        // HLG is always scene-referred (using its own OOTF), everything else
+        // we assume is display-refered by default.
+        if (params->color.gamma == MP_CSP_TRC_HLG) {
+            params->color.light = MP_CSP_LIGHT_SCENE_HLG;
+        } else {
+            params->color.light = MP_CSP_LIGHT_DISPLAY;
+        }
+    }
 }
 
 // Copy properties and data of the AVFrame into the mp_image, without taking

video/out/opengl/video.c

@@ -2056,6 +2056,7 @@ static void pass_colormanage(struct gl_video *p, struct mp_colorspace src, bool
     struct mp_colorspace dst = {
         .gamma = p->opts.target_trc,
         .primaries = p->opts.target_prim,
+        .light = MP_CSP_LIGHT_DISPLAY,
     };
 
     if (p->use_lut_3d) {
@@ -2265,6 +2266,7 @@ static void pass_draw_osd(struct gl_video *p, int draw_flags, double pts,
             static const struct mp_colorspace csp_srgb = {
                 .primaries = MP_CSP_PRIM_BT_709,
                 .gamma = MP_CSP_TRC_SRGB,
+                .light = MP_CSP_LIGHT_DISPLAY,
             };
 
             pass_colormanage(p, csp_srgb, true);

video/out/opengl/video_shaders.c

@@ -368,6 +368,81 @@ void pass_delinearize(struct gl_shader_cache *sc, enum mp_csp_trc trc)
     }
 }
 
+// Apply the OOTF mapping from a given light type to display-referred light.
+// The extra peak parameter is used to scale the values before and after
+// the OOTF, and can be inferred using mp_trc_nom_peak
+void pass_ootf(struct gl_shader_cache *sc, enum mp_csp_light light, float peak)
+{
+    if (light == MP_CSP_LIGHT_DISPLAY)
+        return;
+
+    GLSLF("// apply ootf\n", sc);
+    GLSLF("color.rgb *= vec3(%f);\n", peak);
+
+    switch (light)
+    {
+    case MP_CSP_LIGHT_SCENE_HLG:
+        // HLG OOTF from BT.2100, assuming a reference display with a
+        // peak of 1000 cd/m² -> gamma = 1.2
+        GLSL(float luma = dot(color.rgb, vec3(0.2627, 0.6780, 0.0593));)
+        GLSLF("color.rgb *= vec3(%f * pow(luma, 0.2));\n",
+              (1000 / MP_REF_WHITE) / pow(12, 1.2));
+        break;
+    case MP_CSP_LIGHT_SCENE_709_1886:
+        // This OOTF is defined by encoding the result as 709 and then decoding
+        // it as 1886; although this is called 709_1886 we actually use the
+        // more precise (by one decimal) values from BT.2020 instead
+        GLSL(color.rgb = mix(color.rgb * vec3(4.5),
+                             vec3(1.0993) * pow(color.rgb, vec3(0.45)) - vec3(0.0993),
+                             lessThan(vec3(0.0181), color.rgb));)
+        GLSL(color.rgb = pow(color.rgb, vec3(2.4));)
+        break;
+    case MP_CSP_LIGHT_SCENE_1_2:
+        GLSL(color.rgb = pow(color.rgb, vec3(1.2));)
+        break;
+    default:
+        abort();
+    }
+
+    GLSLF("color.rgb /= vec3(%f);\n", peak);
+}
+
+// Inverse of the function pass_ootf, for completeness' sake. Note that the
+// inverse OOTF for MP_CSP_LIGHT_SCENE_HLG has no analytical solution and is
+// therefore unimplemented. Care must be used to never call this function
+// in that way.(In principle, a iterative algorithm can approach
+// the solution numerically, but this is tricky and we don't really need it
+// since mpv currently only supports outputting display-referred light)
+void pass_inverse_ootf(struct gl_shader_cache *sc, enum mp_csp_light light, float peak)
+{
+    if (light == MP_CSP_LIGHT_DISPLAY)
+        return;
+
+    GLSLF("// apply inverse ootf\n");
+    GLSLF("color.rgb *= vec3(%f);\n", peak);
+
+    switch (light)
+    {
+    case MP_CSP_LIGHT_SCENE_HLG:
+        // Has no analytical solution
+        abort();
+        break;
+    case MP_CSP_LIGHT_SCENE_709_1886:
+        GLSL(color.rgb = pow(color.rgb, vec3(1/2.4));)
+        GLSL(color.rgb = mix(color.rgb / vec3(4.5),
+                             pow((color.rgb + vec3(0.0993)) / vec3(1.0993), vec3(1/0.45)),
+                             lessThan(vec3(0.08145), color.rgb));)
+        break;
+    case MP_CSP_LIGHT_SCENE_1_2:
+        GLSL(color.rgb = pow(color.rgb, vec3(1/1.2));)
+        break;
+    default:
+        abort();
+    }
+
+    GLSLF("color.rgb /= vec3(%f);\n", peak);
+}
+
 // Tone map from a known peak brightness to the range [0,1]
 static void pass_tone_map(struct gl_shader_cache *sc, float ref_peak,
                           enum tone_mapping algo, float param)
@@ -452,20 +527,16 @@ void pass_color_map(struct gl_shader_cache *sc,
     bool need_gamma = src.gamma != dst.gamma ||
                       src.primaries != dst.primaries ||
                       src_range != dst_range ||
-                      src.sig_peak > dst_range;
+                      src.sig_peak > dst_range ||
+                      src.light != dst.light;
 
     if (need_gamma && !is_linear) {
         pass_linearize(sc, src.gamma);
         is_linear= true;
     }
 
-    // NOTE: When src.gamma = MP_CSP_TRC_ARIB_STD_B67, we would technically
-    // need to apply the reference OOTF as part of the EOTF (which is what we
-    // implement with pass_linearize), since HLG considers OOTF to be part of
-    // the display's EOTF (as opposed to the camera's OETF) - although arguably
-    // in our case this would be part of the ICC profile, not mpv. Either way,
-    // in case somebody ends up complaining about HLG looking different from a
-    // reference HLG display, this comment might be why.
+    if (src.light != dst.light)
+        pass_ootf(sc, src.light, mp_trc_nom_peak(src.gamma));
 
     // Rescale the signal to compensate for differences in the encoding range
     // and reference white level. This is necessary because of how mpv encodes
@@ -490,6 +561,9 @@ void pass_color_map(struct gl_shader_cache *sc,
         GLSL(color.rgb = cms_matrix * color.rgb;)
     }
 
+    if (src.light != dst.light)
+        pass_inverse_ootf(sc, dst.light, mp_trc_nom_peak(dst.gamma));
+
     if (is_linear)
         pass_delinearize(sc, dst.gamma);
 }

video/out/opengl/video_shaders.h

@@ -37,6 +37,8 @@ void pass_sample_oversample(struct gl_shader_cache *sc, struct scaler *scaler,
 
 void pass_linearize(struct gl_shader_cache *sc, enum mp_csp_trc trc);
 void pass_delinearize(struct gl_shader_cache *sc, enum mp_csp_trc trc);
+void pass_ootf(struct gl_shader_cache *sc, enum mp_csp_light light, float peak);
+void pass_inverse_ootf(struct gl_shader_cache *sc, enum mp_csp_light light, float peak);
 
 void pass_color_map(struct gl_shader_cache *sc,
                     struct mp_colorspace src, struct mp_colorspace dst,