From f81ed52ec7c415b8dbbefde16ed404c3fd77cc5f Mon Sep 17 00:00:00 2001
From: Torge Matthies <openglfreak@googlemail.com>
Date: Sun, 11 Jul 2021 21:23:07 +0200
Subject: [PATCH] libobs: Add encode_texture2 function to struct
 obs_encoder_info

And use it if non-NULL instead of encode_texture.
---
 libobs/obs-encoder.h          | 16 +++++++++++++++
 libobs/obs-module.c           | 38 +++++++++++++++++++++++++----------
 libobs/obs-video-gpu-encode.c | 23 ++++++++++++++++-----
 3 files changed, 61 insertions(+), 16 deletions(-)

diff --git a/libobs/obs-encoder.h b/libobs/obs-encoder.h
index 6e359af53..8504bd439 100644
--- a/libobs/obs-encoder.h
+++ b/libobs/obs-encoder.h
@@ -120,6 +120,16 @@ struct obs_encoder_roi {
 	float priority;
 };
 
+struct gs_texture;
+
+/** Encoder input texture */
+struct encoder_texture {
+	/** Shared texture handle, only set on Windows */
+	uint32_t handle;
+	/** Textures, length determined by format */
+	struct gs_texture *tex[4];
+};
+
 /**
  * Encoder interface
  *
@@ -280,6 +290,12 @@ struct obs_encoder_info {
 			       uint64_t lock_key, uint64_t *next_key,
 			       struct encoder_packet *packet,
 			       bool *received_packet);
+
+	bool (*encode_texture2)(void *data, struct encoder_texture *texture,
+				int64_t pts, uint64_t lock_key,
+				uint64_t *next_key,
+				struct encoder_packet *packet,
+				bool *received_packet);
 };
 
 EXPORT void obs_register_encoder_s(const struct obs_encoder_info *info,
diff --git a/libobs/obs-module.c b/libobs/obs-module.c
index 9c6e650fe..e79847955 100644
--- a/libobs/obs-module.c
+++ b/libobs/obs-module.c
@@ -707,16 +707,30 @@ cleanup:
 		da_push_back(dest, &data);                              \
 	} while (false)
 
-#define CHECK_REQUIRED_VAL(type, info, val, func)                       \
-	do {                                                            \
-		if ((offsetof(type, val) + sizeof(info->val) > size) || \
-		    !info->val) {                                       \
-			blog(LOG_ERROR,                                 \
-			     "Required value '" #val "' for "           \
-			     "'%s' not found.  " #func " failed.",      \
-			     info->id);                                 \
-			goto error;                                     \
-		}                                                       \
+#define HAS_VAL(type, info, val) \
+	((offsetof(type, val) + sizeof(info->val) <= size) && info->val)
+
+#define CHECK_REQUIRED_VAL(type, info, val, func)                  \
+	do {                                                       \
+		if (!HAS_VAL(type, info, val)) {                   \
+			blog(LOG_ERROR,                            \
+			     "Required value '" #val "' for "      \
+			     "'%s' not found.  " #func " failed.", \
+			     info->id);                            \
+			goto error;                                \
+		}                                                  \
+	} while (false)
+
+#define CHECK_REQUIRED_VAL_EITHER(type, info, val1, val2, func)     \
+	do {                                                        \
+		if (!HAS_VAL(type, info, val1) &&                   \
+		    !HAS_VAL(type, info, val2)) {                   \
+			blog(LOG_ERROR,                             \
+			     "Neither '" #val1 "' nor '" #val2 "' " \
+			     "for '%s' found.  " #func " failed.",  \
+			     info->id);                             \
+			goto error;                                 \
+		}                                                   \
 	} while (false)
 
 #define HANDLE_ERROR(size_var, structure, info)                            \
@@ -927,7 +941,9 @@ void obs_register_encoder_s(const struct obs_encoder_info *info, size_t size)
 	CHECK_REQUIRED_VAL_(info, destroy, obs_register_encoder);
 
 	if ((info->caps & OBS_ENCODER_CAP_PASS_TEXTURE) != 0)
-		CHECK_REQUIRED_VAL_(info, encode_texture, obs_register_encoder);
+		CHECK_REQUIRED_VAL_EITHER(struct obs_encoder_info, info,
+					  encode_texture, encode_texture2,
+					  obs_register_encoder);
 	else
 		CHECK_REQUIRED_VAL_(info, encode, obs_register_encoder);
 
diff --git a/libobs/obs-video-gpu-encode.c b/libobs/obs-video-gpu-encode.c
index b97e5b15e..020fc34b2 100644
--- a/libobs/obs-video-gpu-encode.c
+++ b/libobs/obs-video-gpu-encode.c
@@ -69,7 +69,7 @@ static void *gpu_encode_thread(struct obs_core_video_mix *video)
 		for (size_t i = 0; i < encoders.num; i++) {
 			struct encoder_packet pkt = {0};
 			bool received = false;
-			bool success;
+			bool success = false;
 			uint32_t skip = 0;
 
 			obs_encoder_t *encoder = encoders.array[i];
@@ -125,10 +125,23 @@ static void *gpu_encode_thread(struct obs_core_video_mix *video)
 			else
 				next_key++;
 
-			success = encoder->info.encode_texture(
-				encoder->context.data, tf.handle,
-				encoder->cur_pts, lock_key, &next_key, &pkt,
-				&received);
+			if (encoder->info.encode_texture2) {
+				struct encoder_texture tex = {0};
+
+				tex.handle = tf.handle;
+				tex.tex[0] = tf.tex;
+				tex.tex[1] = tf.tex_uv;
+				tex.tex[2] = NULL;
+				success = encoder->info.encode_texture2(
+					encoder->context.data, &tex,
+					encoder->cur_pts, lock_key, &next_key,
+					&pkt, &received);
+			} else {
+				success = encoder->info.encode_texture(
+					encoder->context.data, tf.handle,
+					encoder->cur_pts, lock_key, &next_key,
+					&pkt, &received);
+			}
 			send_off_encoder_packet(encoder, success, received,
 						&pkt);