From 6648e55972db7b5fc5ef5e2d9d5961f0f0e3489a Mon Sep 17 00:00:00 2001
From: Eidolon <furyhunter600@gmail.com>
Date: Sun, 28 Jan 2024 12:57:00 -0600
Subject: [PATCH] Prevent overflow when scaling textures

Fixes KartKrew/Kart#844 which is caused by scaling the texheight by a
spryscale that is too large, triggering arithmetic overflow before the
overflow check even occurs. This is a performance hit, but should not be
very intense with release optimizations.

Instead, we use saturating arithmetic using the same integer promotion
technique as before, but checking in both directions, and also checking
for the multiplication before the addition.

There is an optimization opportunity here and anywhere that overflow
checks are used, by using compiler intrinsics which check the overflow
flag bit on the CPU instead of using integer promotion.
---
 src/r_segs.cpp | 41 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/src/r_segs.cpp b/src/r_segs.cpp
index 2d5b4b24c..ac98b78e1 100644
--- a/src/r_segs.cpp
+++ b/src/r_segs.cpp
@@ -11,6 +11,8 @@
 /// \file  r_segs.c
 /// \brief All the clipping: columns, horizontal spans, sky columns
 
+#include <limits>
+
 #include <tracy/tracy/Tracy.hpp>
 
 #include "command.h"
@@ -717,16 +719,45 @@ void R_RenderMaskedSegRange(drawseg_t *drawseg, INT32 x1, INT32 x2)
 	R_SetColumnFunc(BASEDRAWFUNC, false);
 }
 
+template <typename T>
+static constexpr T saturating_add(T x, T y) noexcept
+{
+	INT64 z;
+	z = static_cast<INT64>(x) + static_cast<INT64>(y);
+	if (z > static_cast<INT64>(std::numeric_limits<T>::max()))
+	{
+		z = static_cast<INT64>(std::numeric_limits<T>::max());
+	}
+	else if (z < static_cast<INT64>(std::numeric_limits<T>::min()))
+	{
+		z = static_cast<INT64>(std::numeric_limits<T>::min());
+	}
+	return static_cast<T>(z);
+}
+
+template <typename T>
+static constexpr T saturating_mul(T x, T y) noexcept
+{
+	INT64 z;
+	z = static_cast<INT64>(x) * static_cast<INT64>(y);
+	if (z > static_cast<INT64>(std::numeric_limits<T>::max()))
+	{
+		z = static_cast<INT64>(std::numeric_limits<T>::max());
+	}
+	else if (z < static_cast<INT64>(std::numeric_limits<T>::min()))
+	{
+		z = static_cast<INT64>(std::numeric_limits<T>::min());
+	}
+	return static_cast<T>(z);
+}
+
 // Loop through R_DrawMaskedColumn calls
 static void R_DrawRepeatMaskedColumn(drawcolumndata_t* dc, column_t *col, column_t *bm, INT32 baseclip)
 {
 	while (sprtopscreen < sprbotscreen)
 	{
 		R_DrawMaskedColumn(dc, col, bm, baseclip);
-		if ((INT64)sprtopscreen + dc->texheight*spryscale > (INT64)INT32_MAX) // prevent overflow
-			sprtopscreen = INT32_MAX;
-		else
-			sprtopscreen += dc->texheight*spryscale;
+		sprtopscreen = saturating_add(sprtopscreen, saturating_mul(dc->texheight, spryscale));
 	}
 }
 
@@ -734,7 +765,7 @@ static void R_DrawRepeatFlippedMaskedColumn(drawcolumndata_t* dc, column_t *col,
 {
 	do {
 		R_DrawFlippedMaskedColumn(dc, col, bm, baseclip);
-		sprtopscreen += dc->texheight*spryscale;
+		sprtopscreen = saturating_add(sprtopscreen, saturating_mul(dc->texheight, spryscale));
 	} while (sprtopscreen < sprbotscreen);
 }