Clamp patch column post drawing bounds

Prevents segfaults when drawing patches in certain configurations
This commit is contained in:
Eidolon 2024-01-12 18:32:06 -06:00
parent decf430714
commit 26017c4fdd
4 changed files with 28 additions and 6 deletions

View file

@ -1091,6 +1091,7 @@ typedef struct
//Fix TUTIFRUTI
INT32 texheight;
INT32 sourcelength;
UINT8 r8_flatcolor;
} drawcolumndata_t;

View file

@ -121,16 +121,16 @@ static void R_DrawColumnTemplate(drawcolumndata_t *dc)
realyh = dc_copy.yh;
// This runs through the lightlist from top to bottom and cuts up the column accordingly.
for (i = 0; i < dc->numlights; i++)
for (i = 0; i < dc_copy.numlights; i++)
{
// If the height of the light is above the column, get the colormap
// anyway because the lighting of the top should be affected.
solid = dc->lightlist[i].flags & FOF_CUTSOLIDS;
height = dc->lightlist[i].height >> LIGHTSCALESHIFT;
solid = dc_copy.lightlist[i].flags & FOF_CUTSOLIDS;
height = dc_copy.lightlist[i].height >> LIGHTSCALESHIFT;
if (solid)
{
bheight = dc->lightlist[i].botheight >> LIGHTSCALESHIFT;
bheight = dc_copy.lightlist[i].botheight >> LIGHTSCALESHIFT;
if (bheight < height)
{
@ -146,7 +146,7 @@ static void R_DrawColumnTemplate(drawcolumndata_t *dc)
if (height <= dc_copy.yl)
{
dc_copy.colormap = dc->lightlist[i].rcolormap;
dc_copy.colormap = dc_copy.lightlist[i].rcolormap;
dc_copy.fullbright = colormaps;
if (encoremap)
@ -202,6 +202,8 @@ static void R_DrawColumnTemplate(drawcolumndata_t *dc)
fixed_t fracstep;
fixed_t frac;
INT32 heightmask;
INT32 npow2min;
INT32 npow2max;
// Framebuffer destination address.
// Use ylookup LUT to avoid multiply with ScreenWidth.
@ -220,6 +222,18 @@ static void R_DrawColumnTemplate(drawcolumndata_t *dc)
// Inner loop that does the actual texture mapping, e.g. a DDA-like scaling.
// This is as fast as it gets.
heightmask = dc->texheight-1;
if (dc->sourcelength <= 0)
{
// Note: we need to unconditionally clamp in npow2 draw loop to avoid a CPU branch
// This is to just render it effectively the identity function.
npow2min = INT32_MIN;
npow2max = INT32_MAX;
}
else
{
npow2min = -1;
npow2max = dc->sourcelength;
}
if (dc->texheight & heightmask) // not a power of 2 -- killough
{
@ -246,7 +260,10 @@ static void R_DrawColumnTemplate(drawcolumndata_t *dc)
// Re-map color indices from wall texture column
// using a lighting/special effects LUT.
// heightmask is the Tutti-Frutti fix
*dest = R_DrawColumnPixel<Type>(dc, dest, frac >> FRACBITS);
// -1 is the lower clamp bound because column posts have a "safe" byte before the real data
// and a few bytes after as well
*dest = R_DrawColumnPixel<Type>(dc, dest, std::clamp(frac >> FRACBITS, npow2min, npow2max));
dest += vid.width;

View file

@ -129,6 +129,7 @@ static void R_Render2sidedMultiPatchColumn(drawcolumndata_t* dc, column_t *colum
if (dc->yl <= dc->yh && dc->yh < vid.height && dc->yh > 0)
{
dc->source = (UINT8 *)column + 3;
dc->sourcelength = 0;
if (brightmap != NULL)
{
dc->brightmap = (UINT8 *)brightmap + 3;
@ -1334,6 +1335,7 @@ static void R_DrawWallColumn(drawcolumndata_t* dc, INT32 yl, INT32 yh, fixed_t m
dc->source = R_GetColumn(texture, texturecolumn);
dc->brightmap = (brightmapped ? R_GetBrightmapColumn(texture, texturecolumn) : NULL);
dc->texheight = textureheight[texture] >> FRACBITS;
dc->sourcelength = 0;
R_SetColumnFunc(colfunctype, dc->brightmap != NULL);
coldrawfunc_t* colfunccopy = colfunc;
drawcolumndata_t dc_copy = *dc;

View file

@ -688,6 +688,7 @@ void R_DrawMaskedColumn(drawcolumndata_t* dc, column_t *column, column_t *bright
if (dc->yl <= dc->yh && dc->yh > 0)
{
dc->source = (UINT8 *)column + 3;
dc->sourcelength = column->length;
if (brightmap != NULL)
{
dc->brightmap = (UINT8 *)brightmap + 3;
@ -773,6 +774,7 @@ void R_DrawFlippedMaskedColumn(drawcolumndata_t* dc, column_t *column, column_t
if (dc->yl <= dc->yh && dc->yh > 0)
{
dc->source = static_cast<UINT8*>(ZZ_Alloc(column->length));
dc->sourcelength = column->length;
for (s = (UINT8 *)column+2+column->length, d = dc->source; d < dc->source+column->length; --s)
*d++ = *s;