Allow action hook to return -1 to continue execution (#1053)

* Allow action hook to return -1 to continue execution

* Restrict special value to -1 (so more values could be added in the future); fix format

src/game/mario_actions_airborne.c

@@ -2328,7 +2328,7 @@ Dispatches to the appropriate action function, such as jump, double jump, freefa
 |descriptionEnd| */
 s32 mario_execute_airborne_action(struct MarioState *m) {
     if (!m) { return FALSE; }
-    u32 cancel;
+    s32 cancel;
 
     if (check_common_airborne_cancels(m)) {
         return TRUE;
@@ -2336,7 +2336,7 @@ s32 mario_execute_airborne_action(struct MarioState *m) {
 
     play_far_fall_sound(m);
 
-    if (!smlua_call_action_hook(ACTION_HOOK_EVERY_FRAME, m, (s32*)&cancel)) {
+    if (!smlua_call_action_hook(ACTION_HOOK_EVERY_FRAME, m, &cancel)) {
         /* clang-format off */
         switch (m->action) {
             case ACT_JUMP:                 cancel = act_jump(m);                 break;
@@ -2385,9 +2385,9 @@ s32 mario_execute_airborne_action(struct MarioState *m) {
             case ACT_TOP_OF_POLE_JUMP:     cancel = act_top_of_pole_jump(m);     break;
             case ACT_VERTICAL_WIND:        cancel = act_vertical_wind(m);        break;
             default:
-                LOG_ERROR("Attempted to execute unimplemented action '%04X'", m->action);
+                LOG_ERROR("Attempted to execute unimplemented action '%08X'", m->action);
                 set_mario_action(m, ACT_FREEFALL, 0);
-                return false;
+                return FALSE;
         }
         /* clang-format on */
     }

src/game/mario_actions_automatic.c

@@ -1219,9 +1219,9 @@ s32 mario_execute_automatic_action(struct MarioState *m) {
             case ACT_TORNADO_TWIRLING:       cancel = act_tornado_twirling(m);       break;
             case ACT_BUBBLED:                cancel = act_bubbled(m);                break;
             default:
-                LOG_ERROR("Attempted to execute unimplemented action '%04X'", m->action);
+                LOG_ERROR("Attempted to execute unimplemented action '%08X'", m->action);
                 set_mario_action(m, ACT_IDLE, 0);
-                return false;
+                return FALSE;
         }
         /* clang-format on */
     }

src/game/mario_actions_cutscene.c

@@ -3190,9 +3190,9 @@ s32 mario_execute_cutscene_action(struct MarioState *m) {
             case ACT_FEET_STUCK_IN_GROUND:       cancel = act_feet_stuck_in_ground(m);       break;
             case ACT_PUTTING_ON_CAP:             cancel = act_putting_on_cap(m);             break;
             default:
-                LOG_ERROR("Attempted to execute unimplemented action '%04X'", m->action);
+                LOG_ERROR("Attempted to execute unimplemented action '%08X'", m->action);
                 set_mario_action(m, ACT_IDLE, 0);
-                return false;
+                return FALSE;
         }
         /* clang-format on */
     }

src/game/mario_actions_moving.c

@@ -2262,9 +2262,9 @@ s32 mario_execute_moving_action(struct MarioState *m) {
             case ACT_HOLD_QUICKSAND_JUMP_LAND: cancel = act_hold_quicksand_jump_land(m); break;
             case ACT_LONG_JUMP_LAND:           cancel = act_long_jump_land(m);           break;
             default:
-                LOG_ERROR("Attempted to execute unimplemented action '%04X'", m->action);
+                LOG_ERROR("Attempted to execute unimplemented action '%08X'", m->action);
                 set_mario_action(m, ACT_IDLE, 0);
-                return false;
+                return FALSE;
         }
         /* clang-format on */
     }

src/game/mario_actions_object.c

@@ -534,9 +534,9 @@ s32 mario_execute_object_action(struct MarioState *m) {
             case ACT_HOLDING_BOWSER:     cancel = act_holding_bowser(m);     break;
             case ACT_RELEASING_BOWSER:   cancel = act_releasing_bowser(m);   break;
             default:
-                LOG_ERROR("Attempted to execute unimplemented action '%04X'", m->action);
+                LOG_ERROR("Attempted to execute unimplemented action '%08X'", m->action);
                 set_mario_action(m, ACT_IDLE, 0);
-                return false;
+                return FALSE;
         }
         /* clang-format on */
     }

src/game/mario_actions_stationary.c

@@ -1300,9 +1300,9 @@ s32 mario_execute_stationary_action(struct MarioState *m) {
             case ACT_HOLD_BUTT_SLIDE_STOP:    cancel = act_hold_butt_slide_stop(m);             break;
             case ACT_PALETTE_EDITOR_CAP:      cancel = act_palette_editor_cap(m);               break;
             default:
-                LOG_ERROR("Attempted to execute unimplemented action '%04X'", m->action);
+                LOG_ERROR("Attempted to execute unimplemented action '%08X'", m->action);
                 set_mario_action(m, ACT_IDLE, 0);
-                return false;
+                return FALSE;
         }
         /* clang-format on */
     }

src/game/mario_actions_submerged.c

@@ -1710,9 +1710,9 @@ s32 mario_execute_submerged_action(struct MarioState *m) {
             case ACT_HOLD_METAL_WATER_JUMP:      cancel = act_hold_metal_water_jump(m);      break;
             case ACT_HOLD_METAL_WATER_JUMP_LAND: cancel = act_hold_metal_water_jump_land(m); break;
             default:
-                LOG_ERROR("Attempted to execute unimplemented action '%04X'", m->action);
+                LOG_ERROR("Attempted to execute unimplemented action '%08X'", m->action);
                 set_mario_action(m, ACT_WATER_IDLE, 0);
-                return false;
+                return FALSE;
         }
         /* clang-format on */
     }

src/game/mario_step.c

@@ -675,11 +675,13 @@ u32 should_strengthen_gravity_for_jump_ascent(struct MarioState *m) {
 
 void apply_gravity(struct MarioState *m) {
     if (!m) { return; }
-    s32 result;
 
-    if (smlua_call_action_hook(ACTION_HOOK_GRAVITY, m, &result)) {
+    UNUSED s32 cancel;
-        
+    if (smlua_call_action_hook(ACTION_HOOK_GRAVITY, m, &cancel)) {
-    } else if (m->action == ACT_TWIRLING && m->vel[1] < 0.0f) {
+        return;
+    }
+
+    if (m->action == ACT_TWIRLING && m->vel[1] < 0.0f) {
         apply_twirl_gravity(m);
     } else if (m->action == ACT_SHOT_FROM_CANNON) {
         m->vel[1] -= 1.0f;

src/pc/lua/smlua_hooks.c

@@ -324,7 +324,7 @@ int smlua_hook_mario_action(lua_State* L) {
     return 1;
 }
 
-bool smlua_call_action_hook(enum LuaActionHookType hookType, struct MarioState* m, s32* returnValue) {
+bool smlua_call_action_hook(enum LuaActionHookType hookType, struct MarioState* m, s32* cancel) {
     lua_State* L = gLuaState;
     if (L == NULL) { return false; }
 
@@ -343,18 +343,39 @@ bool smlua_call_action_hook(enum LuaActionHookType hookType, struct MarioState*
 
             // call the callback
             if (0 != smlua_call_hook(L, 1, 1, 0, hook->mod, hook->modFile)) {
-                LOG_LUA("Failed to call the action callback: %u", m->action);
+                LOG_LUA("Failed to call the action callback: '%08X'", m->action);
                 continue;
             }
 
             // output the return value
-            *returnValue = false;
+            // special return values:
-            if (lua_type(L, -1) == LUA_TBOOLEAN || lua_type(L, -1) == LUA_TNUMBER) {
+            // - returning -1 allows to continue the execution, useful when overriding vanilla actions
-                *returnValue = smlua_to_integer(L, -1);
+            bool stopActionHook = true;
+            *cancel = FALSE;
+
+            switch (lua_type(L, -1)) {
+                case LUA_TBOOLEAN: {
+                    *cancel = smlua_to_boolean(L, -1) ? TRUE : FALSE;
+                } break;
+
+                case LUA_TNUMBER: {
+                    s32 returnValue = (s32) smlua_to_integer(L, -1);
+                    if (returnValue > 0) {
+                        *cancel = TRUE;
+                    } else if (returnValue == 0) {
+                        *cancel = FALSE;
+                    } else if (returnValue == ACTION_HOOK_CONTINUE_EXECUTION) {
+                        stopActionHook = false;
+                    } else {
+                        LOG_LUA("Invalid return value when calling the action callback: '%08X' returned %d", m->action, returnValue);
+                    }
+                } break;
             }
             lua_pop(L, 1);
 
-            return true;
+            if (stopActionHook) {
+                return true;
+            }
         }
     }
 

src/pc/lua/smlua_hooks.h

@@ -95,6 +95,8 @@ static const char* LuaActionHookTypeArgName[] = {
     "max (dummy)",
 };
 
+#define ACTION_HOOK_CONTINUE_EXECUTION -1
+
 #define MAX_HOOKED_MOD_MENU_ELEMENTS 256
 
 enum LuaModMenuElementType {
@@ -143,7 +145,7 @@ const char* smlua_get_name_from_hooked_behavior_id(enum BehaviorId id);
 bool smlua_call_behavior_hook(const BehaviorScript** behavior, struct Object* object, bool before);
 
 int smlua_call_hook(lua_State* L, int nargs, int nresults, int errfunc, struct Mod* activeMod, struct ModFile* activeModFile);
-bool smlua_call_action_hook(enum LuaActionHookType hookType, struct MarioState* m, s32* returnValue);
+bool smlua_call_action_hook(enum LuaActionHookType hookType, struct MarioState* m, s32* cancel);
 u32 smlua_get_action_interaction_type(struct MarioState* m);
 
 bool smlua_call_chat_command_hook(char* command);