Lighting Engine: Awesome Edition

Just making a brief commit and thats all to add onto dj's lighting engine improvements because I heard about the PR, in mine I
* allowed all Vtx objects to be manipulated/retrieved
* updated lighting engine demo to be an SM64 night mode with a flashlight
* Fixed longterm bug where lighting bugs out unless you spawn a light
* Added new default LE mode LE_MODE_AFFECT_ALL_SHADED_AND_COLORED which also affects vertex colored surfaces

autogen/convert_constants.py

@@ -52,7 +52,8 @@ in_files = [
     "src/pc/network/lag_compensation.h",
     "src/pc/djui/djui_panel_menu.h",
     "src/engine/lighting_engine.h",
-    "include/PR/gbi.h"
+    "include/PR/gbi.h",
+    "include/PR/gbi_extension.h"
 ]
 
 exclude_constants = {
@@ -106,7 +107,8 @@ include_constants = {
         "^G_SETSCISSOR$",
         "^G_TEXRECTFLIP$",
         "^G_TEXRECT$",
-    ]
+    ],
+    "include/PR/gbi_extension.h": [ "G_VTX_EXT" ]
 }
 
 # Constants that exist in the source code but should not appear

autogen/lua_definitions/constants.lua

@@ -2933,6 +2933,9 @@ G_TEXRECTFLIP = 0xe5
 --- @type integer
 G_TEXRECT = 0xe4
 
+--- @type integer
+G_VTX_EXT = 0x11
+
 BACKGROUND_OCEAN_SKY       =  0 --- @type SkyBackgroundParams
 BACKGROUND_FLAMING_SKY     =  1 --- @type SkyBackgroundParams
 BACKGROUND_UNDERWATER_CITY =  2 --- @type SkyBackgroundParams
@@ -3608,10 +3611,12 @@ HUD_DISPLAY_DEFAULT               = HUD_DISPLAY_FLAG_LIVES | HUD_DISPLAY_FLAG_CO
 --- | `HUD_DISPLAY_NONE`
 --- | `HUD_DISPLAY_DEFAULT`
 
-LE_MODE_AFFECT_ALL_SHADED         = 0 --- @type LEMode
-LE_MODE_AFFECT_ONLY_GEOMETRY_MODE = 1 --- @type LEMode
+LE_MODE_AFFECT_ALL_SHADED_AND_COLORED = 0 --- @type LEMode
+LE_MODE_AFFECT_ALL_SHADED             = 1 --- @type LEMode
+LE_MODE_AFFECT_ONLY_GEOMETRY_MODE     = 2 --- @type LEMode
 
 --- @alias LEMode
+--- | `LE_MODE_AFFECT_ALL_SHADED_AND_COLORED`
 --- | `LE_MODE_AFFECT_ALL_SHADED`
 --- | `LE_MODE_AFFECT_ONLY_GEOMETRY_MODE`
 

data/dynos_mgr_gfx.cpp

@@ -69,7 +69,7 @@ static Gfx *DynOS_Gfx_Duplicate(Gfx *aGfx, bool shouldDuplicate) {
         }
 
         // Duplicate referenced vertices
-        if (op == G_VTX) {
+        if (op == G_VTX || op == G_VTX_EXT) {
             cmd->words.w1 = (uintptr_t) DynOS_Vtx_Duplicate((Vtx *) cmd->words.w1, C0(cmd, 12, 8), shouldDuplicate);
         }
     }

docs/lua/constants.md

@@ -26,6 +26,7 @@
     - [enum DialogSound](#enum-DialogSound)
 - [first_person_cam.h](#first_person_camh)
 - [gbi.h](#gbih)
+- [gbi_extension.h](#gbi_extensionh)
 - [geo_commands.h](#geo_commandsh)
     - [enum SkyBackgroundParams](#enum-SkyBackgroundParams)
 - [graph_node.h](#graph_nodeh)
@@ -1289,6 +1290,13 @@
 
 <br />
 
+## [gbi_extension.h](#gbi_extension.h)
+- G_VTX_EXT
+
+[:arrow_up_small:](#)
+
+<br />
+
 ## [geo_commands.h](#geo_commands.h)
 
 ### [enum SkyBackgroundParams](#SkyBackgroundParams)
@@ -1627,8 +1635,9 @@
 ### [enum LEMode](#LEMode)
 | Identifier | Value |
 | :--------- | :---- |
-| LE_MODE_AFFECT_ALL_SHADED | 0 |
-| LE_MODE_AFFECT_ONLY_GEOMETRY_MODE | 1 |
+| LE_MODE_AFFECT_ALL_SHADED_AND_COLORED | 0 |
+| LE_MODE_AFFECT_ALL_SHADED | 1 |
+| LE_MODE_AFFECT_ONLY_GEOMETRY_MODE | 2 |
 
 ### [enum LEToneMapping](#LEToneMapping)
 | Identifier | Value |

docs/lua/examples/lighting-engine-demo/levels/canals/model.inc.c

@@ -4274,7 +4274,7 @@ Vtx canals_dl_skybox_mesh_layer_5_vtx_0[46] = {
 };
 
 Gfx canals_dl_skybox_mesh_layer_5_tri_0[] = {
-	gsSPVertex(canals_dl_skybox_mesh_layer_5_vtx_0 + 0, 16, 0),
+	gsSPVertexNonGlobal(canals_dl_skybox_mesh_layer_5_vtx_0 + 0, 16, 0),
 	gsSP1Triangle(0, 1, 2, 0),
 	gsSP1Triangle(0, 3, 1, 0),
 	gsSP1Triangle(4, 3, 0, 0),
@@ -4289,7 +4289,7 @@ Gfx canals_dl_skybox_mesh_layer_5_tri_0[] = {
 	gsSP1Triangle(12, 11, 13, 0),
 	gsSP1Triangle(14, 12, 13, 0),
 	gsSP1Triangle(14, 13, 15, 0),
-	gsSPVertex(canals_dl_skybox_mesh_layer_5_vtx_0 + 16, 16, 0),
+	gsSPVertexNonGlobal(canals_dl_skybox_mesh_layer_5_vtx_0 + 16, 16, 0),
 	gsSP1Triangle(0, 1, 2, 0),
 	gsSP1Triangle(0, 2, 3, 0),
 	gsSP1Triangle(1, 0, 4, 0),
@@ -4304,7 +4304,7 @@ Gfx canals_dl_skybox_mesh_layer_5_tri_0[] = {
 	gsSP1Triangle(12, 11, 13, 0),
 	gsSP1Triangle(13, 11, 14, 0),
 	gsSP1Triangle(13, 14, 15, 0),
-	gsSPVertex(canals_dl_skybox_mesh_layer_5_vtx_0 + 32, 14, 0),
+	gsSPVertexNonGlobal(canals_dl_skybox_mesh_layer_5_vtx_0 + 32, 14, 0),
 	gsSP1Triangle(0, 1, 2, 0),
 	gsSP1Triangle(3, 1, 0, 0),
 	gsSP1Triangle(3, 4, 1, 0),

docs/lua/examples/lighting-engine-demo/main.lua

@@ -1,88 +1,27 @@
 -- name: Lighting Engine Demo
--- description: Lighting Engine Demo\nBy \\#ec7731\\Agent X\n\n\\#dcdcdc\\Open the panel in the pause menu to see what you can do.
+-- description: Lighting Engine Demo\nBy \\#ec7731\\Agent X\n\n\\#dcdcdc\\Open the mod menu in the pause menu to see what you can do.
 
-local flashlightColor = { 255, 255, 200 }
+flashlightColor = { 255, 255, 200 }
+
+--- @param cmd Gfx
+--- @param op integer
+local function parse_dl(cmd, op)
+    if op == G_SETENVCOLOR then
+        gfx_set_command(cmd, "gsDPSetEnvColor(255, 255, 255, %i)", if_then_else(_G.dayNightCycleApi ~= nil and _G.dayNightCycleApi.is_dnc_enabled(), 0, 255))
+    end
+end
 
 --- @param node GraphNode
 function geo_hide_if_dnc(node)
     local dl = cast_graph_node(node.next)
-    gfx_parse(dl.displayList, function(cmd, op)
-        if op == G_SETENVCOLOR then
-            gfx_set_command(cmd, "gsDPSetEnvColor(255, 255, 255, %i)", if_then_else(_G.dayNightCycleApi ~= nil and _G.dayNightCycleApi.is_dnc_enabled(), 0, 255))
-        end
-    end)
+    gfx_parse(dl.displayList, parse_dl)
 end
 
---- @param o Object
-local function bhv_spawn_point_light(o)
-    local light = spawn_non_sync_object(
-        id_bhvPointLight,
-        E_MODEL_NONE,
-        o.oPosX, o.oPosY, o.oPosZ,
-        nil
-    )
-    if get_id_from_behavior(o.behavior) == id_bhvKoopaShell then
-        light.oBehParams = 0x00FF0032
-    else
-        light.oBehParams = 0xFFFFFF32
-    end
-    light.parentObj = o
-end
+local function update()
+    shading_update()
 
---- @param o Object
-function bhv_flashlight_loop(o)
-    local pos = gMarioStates[0].pos
-    local yaw = gFirstPersonCamera.yaw + 0x8000
-    o.oPosX = pos.x + sins(yaw) * 300
-    o.oPosY = pos.y + -gFirstPersonCamera.pitch * 0.06
-    o.oPosZ = pos.z + coss(yaw) * 300
-    le_set_light_pos(o.oLightID, o.oPosX, o.oPosY, o.oPosZ)
-    le_set_light_color(o.oLightID, flashlightColor[1], flashlightColor[2], flashlightColor[3])
-end
-
-local function spawn_flashlight()
-    local flashlight = spawn_non_sync_object(
-        bhvFlashlight,
-        E_MODEL_NONE,
-        0, 0, 0,
-        nil
-    )
-    flashlight.oBehParams = (flashlightColor[1] << 24) | (flashlightColor[2] << 16) | (flashlightColor[3] << 8) | (90)
-    return flashlight
-end
-
---- @param flashlight Object
-local function delete_flashlight(flashlight)
-    le_remove_light(flashlight.oLightID)
-    obj_mark_for_deletion(flashlight)
-    return nil
-end
-
---- @param m MarioState
-local function calculate_mario_lighting(m)
-    local color = { r = 0, g = 0, b = 0 }
-    local dir = { x = 0, y = 0, z = 0 }
-
-    le_calculate_lighting_color(m.pos, color, 0.25)
-    le_calculate_lighting_dir(m.pos, dir)
-
-    m.marioBodyState.lightR = color.r
-    m.marioBodyState.lightG = color.g
-    m.marioBodyState.lightB = color.b
-    m.marioBodyState.shadeR = m.marioBodyState.lightR * 0.5
-    m.marioBodyState.shadeG = m.marioBodyState.lightG * 0.5
-    m.marioBodyState.shadeB = m.marioBodyState.lightB * 0.5
-
-    m.marioBodyState.lightingDirX = -DEFAULT_LIGHTING_DIR + dir.x
-    m.marioBodyState.lightingDirY = -DEFAULT_LIGHTING_DIR + dir.y
-    m.marioBodyState.lightingDirZ = -DEFAULT_LIGHTING_DIR + dir.z
-end
-
---- @param m MarioState
-local function mario_update(m)
-    if m.playerIndex ~= 0 then return end
-
-    calculate_mario_lighting(m)
+    --- @type MarioState
+    local m = gMarioStates[0]
 
     if (m.controller.buttonPressed & L_JPAD) ~= 0 then
         audio_sample_play(SOUND_CUSTOM_FLASHLIGHT, m.pos, 1.0)
@@ -95,14 +34,27 @@ local function mario_update(m)
     end
 end
 
+local function on_level_init()
+    local levelNum = gNetworkPlayers[0].currLevelNum
+    if levelNum == LEVEL_HL or levelNum == LEVEL_CANALS then return end
+
+    le_set_ambient_color(30, 30, 75)
+end
+
 
 local function on_set_flashlight_color(index, value)
     flashlightColor[index + 1] = value
 end
 
-id_bhvKoopaShell = hook_behavior(id_bhvKoopaShell, OBJ_LIST_LEVEL, false, bhv_spawn_point_light, nil, "bhvKoopaShell")
+set_override_skybox(BACKGROUND_HAUNTED)
+set_fog_color(0, 30)
+set_fog_color(1, 30)
+set_fog_color(2, 75)
 
-hook_event(HOOK_MARIO_UPDATE, mario_update)
+djui_popup_create("Use Left D-Pad to turn on the flashlight.", 2)
+
+hook_event(HOOK_UPDATE, update)
+hook_event(HOOK_ON_LEVEL_INIT, on_level_init)
 
 hook_mod_menu_slider("Flashlight Red", flashlightColor[1], 0, 255, on_set_flashlight_color)
 hook_mod_menu_slider("Flashlight Green", flashlightColor[2], 0, 255, on_set_flashlight_color)

docs/lua/examples/lighting-engine-demo/objects.lua

@@ -0,0 +1,58 @@
+--- @param o Object
+local function bhv_spawn_point_light(o)
+    local light = spawn_non_sync_object(
+        id_bhvPointLight,
+        E_MODEL_NONE,
+        o.oPosX, o.oPosY, o.oPosZ,
+        nil
+    )
+    if get_id_from_behavior(o.behavior) == id_bhvKoopaShell then
+        light.oBehParams = 0x00FF0064
+    else
+        light.oBehParams = 0xFFFFFF32
+    end
+    light.parentObj = o
+end
+
+--- @param o Object
+function bhv_flashlight_loop(o)
+    local fp = get_first_person_enabled()
+    local pos = gMarioStates[0].pos
+    local yaw = if_then_else(fp, gFirstPersonCamera.yaw + 0x8000, gMarioStates[0].faceAngle.y)
+
+    local dirX = sins(yaw) * 300
+    local dirY = if_then_else(fp, -gFirstPersonCamera.pitch * 0.06, 120)
+    local dirZ = coss(yaw) * 300
+
+    local raycast = collision_find_surface_on_ray(pos.x, pos.y, pos.z, dirX, dirY, dirZ)
+    if raycast.surface == nil then
+        o.oPosX = pos.x + dirX
+        o.oPosY = pos.y + dirY
+        o.oPosZ = pos.z + dirZ
+    else
+        vec3f_to_object_pos(o, raycast.hitPos)
+    end
+
+    le_set_light_pos(o.oLightID, o.oPosX, o.oPosY, o.oPosZ)
+    le_set_light_color(o.oLightID, flashlightColor[1], flashlightColor[2], flashlightColor[3])
+end
+
+function spawn_flashlight()
+    local flashlight = spawn_non_sync_object(
+        bhvFlashlight,
+        E_MODEL_NONE,
+        0, 0, 0,
+        nil
+    )
+    flashlight.oBehParams = (flashlightColor[1] << 24) | (flashlightColor[2] << 16) | (flashlightColor[3] << 8) | (90)
+    return flashlight
+end
+
+--- @param flashlight Object
+function delete_flashlight(flashlight)
+    le_remove_light(flashlight.oLightID)
+    obj_mark_for_deletion(flashlight)
+    return nil
+end
+
+id_bhvKoopaShell = hook_behavior(id_bhvKoopaShell, OBJ_LIST_LEVEL, false, bhv_spawn_point_light, nil, "bhvKoopaShell")

docs/lua/examples/lighting-engine-demo/shading.lua

@@ -0,0 +1,51 @@
+-- special thanks to Coolio for figuring out worldspace shading
+
+--- @param dest Vec3f
+--- @param rotate Vec3f
+--- Rotates `dest` around the Z, Y, and X axes
+local function vec3f_rotate_zyx(dest, rotate)
+    local v = { x = dest.x, y = dest.y, z = dest.z }
+
+    local sx = sins(rotate.x)
+    local cx = coss(rotate.x)
+
+    local sy = sins(rotate.y)
+    local cy = coss(rotate.y)
+
+    local sz = sins(rotate.z)
+    local cz = coss(rotate.z)
+
+    -- Rotation around Z axis
+    local xz = v.x * cz - v.y * sz
+    local yz = v.x * sz + v.y * cz
+    local zz = v.z
+
+    -- Rotation around Y axis
+    local xy = xz * cy + zz * sy
+    local yy = yz
+    local zy = -xz * sy + zz * cy
+
+    -- Rotation around X axis
+    dest.x = xy
+    dest.y = yy * cx - zy * sx
+    dest.z = yy * sx + zy * cx
+
+    return dest
+end
+
+local LIGHTING_OFFSET = { x = 0x28 / 0xFF, y = 0x28 / 0xFF, z = 0x28 / 0xFF }
+
+function shading_update()
+    local l = gLakituState
+    local pitch = calculate_pitch(l.pos, l.focus)
+    local yaw = calculate_yaw(l.pos, l.focus)
+    local roll = l.roll
+
+    local lightingDir = gVec3fOne()
+    vec3f_rotate_zyx(lightingDir, { x = -pitch, y = -yaw, z = roll })
+    vec3f_sub(lightingDir, LIGHTING_OFFSET)
+
+    set_lighting_dir(0, lightingDir.x)
+    set_lighting_dir(1, lightingDir.y)
+    set_lighting_dir(2, lightingDir.z)
+end

src/engine/lighting_engine.c

@@ -6,10 +6,10 @@
 
 #define LE_MAX_LIGHTS 256
 
-static Color sAmbientColor = { 127, 127, 127 };
+Color gLEAmbientColor = { 127, 127, 127 };
 static void* sLights = NULL;
 static s32 sLightID = 0;
-static enum LEMode sMode = LE_MODE_AFFECT_ALL_SHADED;
+static enum LEMode sMode = LE_MODE_AFFECT_ALL_SHADED_AND_COLORED;
 static enum LEToneMapping sToneMapping = LE_TONE_MAPPING_WEIGHTED;
 static bool sEnabled = false;
 
@@ -129,9 +129,9 @@ void le_calculate_vertex_lighting(Vtx_t* v, Vec3f pos, OUT Color out) {
 
     // tone map and output
     Color vtxAmbient = {
-        v->cn[0] * (sAmbientColor[0] / 255.0f),
-        v->cn[1] * (sAmbientColor[1] / 255.0f),
-        v->cn[2] * (sAmbientColor[2] / 255.0f),
+        v->cn[0] * (gLEAmbientColor[0] / 255.0f),
+        v->cn[1] * (gLEAmbientColor[1] / 255.0f),
+        v->cn[2] * (gLEAmbientColor[2] / 255.0f),
     };
     le_tone_map(out, vtxAmbient, color, weight);
 }
@@ -149,7 +149,7 @@ void le_calculate_lighting_color(Vec3f pos, OUT Color out, f32 lightIntensitySca
     }
 
     // tone map and output
-    le_tone_map(out, sAmbientColor, color, weight);
+    le_tone_map(out, gLEAmbientColor, color, weight);
 }
 
 void le_calculate_lighting_color_with_normal(Vec3f pos, Vec3f normal, OUT Color out, f32 lightIntensityScalar) {
@@ -168,7 +168,7 @@ void le_calculate_lighting_color_with_normal(Vec3f pos, Vec3f normal, OUT Color
     }
 
     // tone map and output
-    le_tone_map(out, sAmbientColor, color, weight);
+    le_tone_map(out, gLEAmbientColor, color, weight);
 }
 
 void le_calculate_lighting_dir(Vec3f pos, OUT Vec3f out) {
@@ -241,7 +241,7 @@ s32 le_get_light_count(void) {
 }
 
 void le_set_ambient_color(u8 r, u8 g, u8 b) {
-    color_set(sAmbientColor, r, g, b);
+    color_set(gLEAmbientColor, r, g, b);
     sEnabled = true;
 }
 
@@ -297,16 +297,16 @@ void le_clear(void) {
     }
     hmap_clear(sLights);
     sLightID = 0;
-    sAmbientColor[0] = 127;
-    sAmbientColor[1] = 127;
-    sAmbientColor[2] = 127;
+    gLEAmbientColor[0] = 127;
+    gLEAmbientColor[1] = 127;
+    gLEAmbientColor[2] = 127;
 }
 
 void le_shutdown(void) {
     if (sLights == NULL) { return; }
 
     sEnabled = false;
-    sMode = LE_MODE_AFFECT_ALL_SHADED;
+    sMode = LE_MODE_AFFECT_ALL_SHADED_AND_COLORED;
     sToneMapping = LE_TONE_MAPPING_WEIGHTED;
     le_clear();
     hmap_destroy(sLights);

src/engine/lighting_engine.h

@@ -17,6 +17,7 @@ struct LELight
 };
 
 enum LEMode {
+    LE_MODE_AFFECT_ALL_SHADED_AND_COLORED,
     LE_MODE_AFFECT_ALL_SHADED,
     LE_MODE_AFFECT_ONLY_GEOMETRY_MODE,
 };
@@ -28,6 +29,8 @@ enum LEToneMapping {
     LE_TONE_MAPPING_REINHARD,
 };
 
+extern Color gLEAmbientColor;
+
 /* |description|Sets the lighting engine mode to `mode`|descriptionEnd|*/
 void le_set_mode(enum LEMode mode);
 /* |description|Gets the lighting engine mode|descriptionEnd|*/

src/pc/gfx/gfx_pc.c

@@ -724,7 +724,6 @@ static OPTIMIZE_O3 void gfx_local_to_world_space(OUT Vec3f pos, OUT Vec3f normal
     }
 }
 
-
 static void OPTIMIZE_O3 gfx_sp_vertex(size_t n_vertices, size_t dest_index, const Vtx *vertices, bool luaVertexColor) {
     if (!vertices) { return; }
 
@@ -779,6 +778,9 @@ static void OPTIMIZE_O3 gfx_sp_vertex(size_t n_vertices, size_t dest_index, cons
         short U = v->tc[0] * rsp.texture_scaling_factor.s >> 16;
         short V = v->tc[1] * rsp.texture_scaling_factor.t >> 16;
 
+        // are we on affect all shaded surfaces mode and on a vertex colorable surface
+        bool affectAllVertexColored = (le_get_mode() == LE_MODE_AFFECT_ALL_SHADED_AND_COLORED && luaVertexColor);
+
         if (rsp.geometry_mode & G_LIGHTING) {
             if (rsp.lights_changed) {
                 bool applyLightingDir = !(rsp.geometry_mode & G_TEXTURE_GEN);
@@ -861,8 +863,9 @@ static void OPTIMIZE_O3 gfx_sp_vertex(size_t n_vertices, size_t dest_index, cons
                 V = (int32_t)((doty / 127.0f + 1.0f) / 4.0f * rsp.texture_scaling_factor.t);
             }
 
-            if (le_is_enabled() && ((le_get_mode() == LE_MODE_AFFECT_ALL_SHADED) || (rsp.geometry_mode & G_LIGHTING_ENGINE_EXT))) {
-                Color color;
+            // if lighting engine is enabled and either we want to affect all shaded surfaces or the lighting engine geometry mode is on
+            if (le_is_enabled() && ((le_get_mode() != LE_MODE_AFFECT_ONLY_GEOMETRY_MODE) || (rsp.geometry_mode & G_LIGHTING_ENGINE_EXT))) {
+                Color color = { gLEAmbientColor[0], gLEAmbientColor[1], gLEAmbientColor[2] };
                 CTX_BEGIN(CTX_LIGHTING);
 
                 Vec3f vpos    = { v->ob[0], v->ob[1], v->ob[2] };
@@ -879,8 +882,9 @@ static void OPTIMIZE_O3 gfx_sp_vertex(size_t n_vertices, size_t dest_index, cons
                 d->color.g *= color[1] / 255.0f;
                 d->color.b *= color[2] / 255.0f;
             }
-        } else if (le_is_enabled() && (rsp.geometry_mode & G_LIGHTING_ENGINE_EXT)) {
-            Color color;
+        // if lighting engine is enabled and we should affect all vertex colored surfaces or the lighting engine geometry mode is on
+        } else if (le_is_enabled() && (affectAllVertexColored || (rsp.geometry_mode & G_LIGHTING_ENGINE_EXT))) {
+            Color color = { gLEAmbientColor[0], gLEAmbientColor[1], gLEAmbientColor[2] };
             CTX_BEGIN(CTX_LIGHTING);
 
             Vec3f vpos = { v->ob[0], v->ob[1], v->ob[2] };
@@ -888,18 +892,33 @@ static void OPTIMIZE_O3 gfx_sp_vertex(size_t n_vertices, size_t dest_index, cons
             // transform vpos to world space
             gfx_local_to_world_space(vpos, NULL);
 
-            le_calculate_vertex_lighting((Vtx_t*)v, vpos, color);
+            // do multiplication based lighting instead of additive based lighting if we're not using the lighting engine geometry mode,
+            // this is my compromise for retaining vertex colors vs lighting up darker surfaces.
+            // if retaining color is the most important like on a red coin, don't use the lighting engine geometry mode.
+            // if lighting up darker surfaces like in a map with prebaked lighting is the most important, use the lighting engine geometry mode.
+            if (affectAllVertexColored && !(rsp.geometry_mode & G_LIGHTING_ENGINE_EXT)) {
+                le_calculate_lighting_color(vpos, color, 1.0f);
+            } else {
+                le_calculate_vertex_lighting((Vtx_t*)v, vpos, color);
+            }
 
             CTX_END(CTX_LIGHTING);
 
-            if (luaVertexColor) {
-                d->color.r = color[0] * vertexColorCached[0];
-                d->color.g = color[1] * vertexColorCached[1];
-                d->color.b = color[2] * vertexColorCached[2];
+            // combine the colors
+            if (affectAllVertexColored && !(rsp.geometry_mode & G_LIGHTING_ENGINE_EXT)) {
+                d->color.r = (v->cn[0] * color[0] / 255.0f) * vertexColorCached[0];
+                d->color.g = (v->cn[1] * color[1] / 255.0f) * vertexColorCached[1];
+                d->color.b = (v->cn[2] * color[2] / 255.0f) * vertexColorCached[2];
             } else {
-                d->color.r = color[0];
-                d->color.g = color[1];
-                d->color.b = color[2];
+                if (luaVertexColor) {
+                    d->color.r = color[0] * vertexColorCached[0];
+                    d->color.g = color[1] * vertexColorCached[1];
+                    d->color.b = color[2] * vertexColorCached[2];
+                } else {
+                    d->color.r = color[0];
+                    d->color.g = color[1];
+                    d->color.b = color[2];
+                }
             }
         } else {
             if (!(rsp.geometry_mode & G_LIGHT_MAP_EXT) && luaVertexColor) {

src/pc/lua/smlua_constants_autogen.c

@@ -1515,6 +1515,7 @@ char gSmluaConstants[] = ""
 "G_SETSCISSOR=0xed\n"
 "G_TEXRECTFLIP=0xe5\n"
 "G_TEXRECT=0xe4\n"
+"G_VTX_EXT=0x11\n"
 "BACKGROUND_OCEAN_SKY=0\n"
 "BACKGROUND_FLAMING_SKY=1\n"
 "BACKGROUND_UNDERWATER_CITY=2\n"
@@ -1777,8 +1778,9 @@ char gSmluaConstants[] = ""
 "HUD_DISPLAY_FLAG_EMPHASIZE_POWER=0x8000\n"
 "HUD_DISPLAY_NONE=0x0000\n"
 "HUD_DISPLAY_DEFAULT=HUD_DISPLAY_FLAG_LIVES | HUD_DISPLAY_FLAG_COIN_COUNT | HUD_DISPLAY_FLAG_STAR_COUNT | HUD_DISPLAY_FLAG_CAMERA_AND_POWER | HUD_DISPLAY_FLAG_CAMERA | HUD_DISPLAY_FLAG_POWER | HUD_DISPLAY_FLAG_KEYS | HUD_DISPLAY_FLAG_UNKNOWN_0020\n"
-"LE_MODE_AFFECT_ALL_SHADED=0\n"
-"LE_MODE_AFFECT_ONLY_GEOMETRY_MODE=1\n"
+"LE_MODE_AFFECT_ALL_SHADED_AND_COLORED=0\n"
+"LE_MODE_AFFECT_ALL_SHADED=1\n"
+"LE_MODE_AFFECT_ONLY_GEOMETRY_MODE=2\n"
 "LE_TONE_MAPPING_TOTAL_WEIGHTED=0\n"
 "LE_TONE_MAPPING_WEIGHTED=1\n"
 "LE_TONE_MAPPING_CLAMP=2\n"

src/pc/lua/utils/smlua_gfx_utils.c

@@ -223,7 +223,7 @@ Gfx *gfx_get_display_list(Gfx *cmd) {
 Vtx *gfx_get_vertex_buffer(Gfx *cmd) {
     if (!cmd) { return NULL; }
     u32 op = GFX_OP(cmd);
-    if (op != G_VTX) { return NULL; }
+    if (op != G_VTX && op != G_VTX_EXT) { return NULL; }
     if (cmd->words.w1 == 0) { return NULL; }
 
     return (Vtx *) cmd->words.w1;
@@ -232,7 +232,7 @@ Vtx *gfx_get_vertex_buffer(Gfx *cmd) {
 u16 gfx_get_vertex_count(Gfx *cmd) {
     if (!cmd) { return 0; }
     u32 op = GFX_OP(cmd);
-    if (op != G_VTX) { return 0; }
+    if (op != G_VTX && op != G_VTX_EXT) { return 0; }
     if (cmd->words.w1 == 0) { return 0; }
 
     return C0(cmd, 12, 8);