libobs: Add OBS_ENCODER_CAP_SCALING

Adds a new encoder cap which tells libobs that rather than scaling
video frames in software, the encoder is capable of scaling them via
its own (presumably more efficient) means.

An encoder may implement this cap by comparing the VOI of its assigned
`video_t` and the results of `obs_encoder_get_width/height()`. If the
width/height values differ, then the encoder is being asked by libobs
to self-scale, and the resolution in VOI will be the raw frame size,
with the `...get_width/height()` being the intended output resolution
of the encoder.

It is important to note that GPU rescaling mode will take priority
over self-scaling. If GPU rescaling is enabled, the encoder will never
be asked to self-scale.

This is useful for discrete hardware encoders, where they might have
fixed-function video scaling logic that is highly efficient and fast.
Additionally, this feature allows a hardware device which is encoding
a full ABR ladder of tracks to be smart and only copy a video frame
from GPU -> Host -> Device once for the entire ladder, rather than
once for every track.

docs/sphinx/reference-encoders.rst

@@ -164,8 +164,9 @@ Encoder Definition Structure (obs_encoder_info)
 
    - **OBS_ENCODER_CAP_DEPRECATED** - Encoder is deprecated
    - **OBS_ENCODER_CAP_ROI** - Encoder supports region of interest feature
+   - **OBS_ENCODER_CAP_SCALING** - Encoder implements its own scaling logic,
+                                   desiring to receive unscaled frames
 
-      .. versionadded:: 30.1
 
 Encoder Packet Structure (encoder_packet)
 -----------------------------------------

libobs/obs-encoder.c

@@ -186,6 +186,16 @@ static inline void get_video_info(struct obs_encoder *encoder,
 
 	if (encoder->info.get_video_info)
 		encoder->info.get_video_info(encoder->context.data, info);
+
+	/**
+	 * Prevent video output from performing an actual scale. If GPU scaling is
+	 * enabled, the voi will contain the scaled size. Therefore, GPU scaling
+	 * takes priority over self-scaling functionality.
+	 */
+	if ((encoder->info.caps & OBS_ENCODER_CAP_SCALING) != 0) {
+		info->width = voi->width;
+		info->height = voi->height;
+	}
 }
 
 static inline bool gpu_encode_available(const struct obs_encoder *encoder)

libobs/obs-encoder.h

@@ -37,6 +37,7 @@ typedef struct obs_encoder obs_encoder_t;
 #define OBS_ENCODER_CAP_DYN_BITRATE (1 << 2)
 #define OBS_ENCODER_CAP_INTERNAL (1 << 3)
 #define OBS_ENCODER_CAP_ROI (1 << 4)
+#define OBS_ENCODER_CAP_SCALING (1 << 5)
 
 /** Specifies the encoder type */
 enum obs_encoder_type {

libobs/obs-module.c

@@ -934,6 +934,14 @@ void obs_register_encoder_s(const struct obs_encoder_info *info, size_t size)
 		goto error;
 	}
 
+	if (((info->caps & OBS_ENCODER_CAP_PASS_TEXTURE) != 0 &&
+	     info->caps & OBS_ENCODER_CAP_SCALING) != 0) {
+		encoder_warn(
+			"Texture encoders cannot self-scale. Encoder id '%s' not registered.",
+			info->id);
+		goto error;
+	}
+
 #define CHECK_REQUIRED_VAL_(info, val, func) \
 	CHECK_REQUIRED_VAL(struct obs_encoder_info, info, val, func)
 	CHECK_REQUIRED_VAL_(info, get_name, obs_register_encoder);