Add bytestring packets for mods to use (#866)

Adds network_send_bytestring(), network_send_bytestring_to(), and
HOOK_ON_PACKET_BYTESTRING_RECEIVE

This is as close to raw bytes as you can get in lua. It allows you
to efficiently pack as much data as possible into each packet.

The existing network_send() was built for ease of use, but is quite
inefficient when you want to send a lot of data. Each number or int
field in the table amounts to 9 bytes for the key and 9 bytes for the
value. (amounting to 18 bytes per value). Whereas with the new method
you can pack that information into 1, 2, 4, or 8 bytes depending on
your type's size.

---------

Co-authored-by: MysterD <myster@d>

autogen/lua_definitions/constants.lua

@@ -8046,7 +8046,8 @@ HOOK_ON_INSTANT_WARP                        = 55 --- @type LuaHookedEventType
 HOOK_MARIO_OVERRIDE_FLOOR_CLASS             = 56 --- @type LuaHookedEventType
 HOOK_ON_ADD_SURFACE                         = 57 --- @type LuaHookedEventType
 HOOK_ON_CLEAR_AREAS                         = 58 --- @type LuaHookedEventType
-HOOK_MAX                                    = 59 --- @type LuaHookedEventType
+HOOK_ON_PACKET_BYTESTRING_RECEIVE           = 59 --- @type LuaHookedEventType
+HOOK_MAX                                    = 60 --- @type LuaHookedEventType
 
 --- @alias LuaHookedEventType
 --- | `HOOK_UPDATE`
@@ -8108,6 +8109,7 @@ HOOK_MAX                                    = 59 --- @type LuaHookedEventType
 --- | `HOOK_MARIO_OVERRIDE_FLOOR_CLASS`
 --- | `HOOK_ON_ADD_SURFACE`
 --- | `HOOK_ON_CLEAR_AREAS`
+--- | `HOOK_ON_PACKET_BYTESTRING_RECEIVE`
 --- | `HOOK_MAX`
 
 HUD_DISPLAY_LIVES         = 0 --- @type HudDisplayValue

autogen/lua_definitions/manual.lua

@@ -264,7 +264,7 @@ end
 
 --- @param reliable boolean Whether or not the game should try to resend the packet in case its lost, good for important packets
 --- @param dataTable table<string, number|boolean|string|nil> Table of values to be included in the packet
---- Sends a global Lua packet with the values of `dataTable`
+--- Sends a global Lua packet with the values of `dataTable`. Received with the `HOOK_ON_PACKET_RECEIVE` hook.
 function network_send(reliable, dataTable)
     -- ...
 end
@@ -272,11 +272,26 @@ end
 --- @param toLocalIndex integer The local index to send the packet to
 --- @param reliable boolean Whether or not the game should try to resend the packet in case its lost, good for important packets
 --- @param dataTable table Table of values to be included in the packet
---- Sends a Lua packet with the values of `dataTable` to a specific client through local indices
+--- Sends a Lua packet with the values of `dataTable` to a specific client through local indices. Received with the `HOOK_ON_PACKET_RECEIVE` hook.
 function network_send_to(toLocalIndex, reliable, dataTable)
     -- ...
 end
 
+--- @param reliable boolean Whether or not the game should try to resend the packet in case its lost, good for important packets
+--- @param bytestring string The bytestring to be included in the packet
+--- Sends a global Lua packet with the bytestring of `bytestring`. Received with the `HOOK_ON_PACKET_BYTESTRING_RECEIVE` hook.
+function network_send_bytestring(reliable, bytestring)
+    -- ...
+end
+
+--- @param toLocalIndex integer The local index to send the packet to
+--- @param reliable boolean Whether or not the game should try to resend the packet in case its lost, good for important packets
+--- @param bytestring string The bytestring to be included in the packet
+--- Sends a Lua packet with the bytestring of `bytestring` to a specific client through local indices. Received with the `HOOK_ON_PACKET_BYTESTRING_RECEIVE` hook.
+function network_send_bytestring_to(toLocalIndex, reliable, bytestring)
+    -- ...
+end
+
 --- @param textureName string The texture name
 --- @return TextureInfo
 --- Gets the `TextureInfo` of a texture by name

docs/lua/constants.md

@@ -3474,7 +3474,8 @@
 | HOOK_MARIO_OVERRIDE_FLOOR_CLASS | 56 |
 | HOOK_ON_ADD_SURFACE | 57 |
 | HOOK_ON_CLEAR_AREAS | 58 |
-| HOOK_MAX | 59 |
+| HOOK_ON_PACKET_BYTESTRING_RECEIVE | 59 |
+| HOOK_MAX | 60 |
 
 [:arrow_up_small:](#)
 

docs/lua/examples/bytestring-packet-example.lua

@@ -0,0 +1,132 @@
+-- name: Bytestring Packet Example
+-- description: Shows a way to send and receive bytestring packets. Send a packet with /example [1|2].
+
+-- Here is the lua documentation for string.pack() and string.unpack():
+--    https://www.lua.org/manual/5.3/manual.html#6.4.2
+-- It explains the different format strings used in this example mod.
+
+---------------------------------------------------------------------------------------------------
+
+local PACKET_EXAMPLE_1 = 1
+local PACKET_EXAMPLE_2 = 2
+
+---------------------------------------------------------------------------------------------------
+
+function send_example_1(byte_param, short_param, long_param, float_param, double_param)
+
+    local bytestring = ''
+        -------------- PACKET ID --------------
+        .. string.pack("<B", PACKET_EXAMPLE_1)
+        ---------------------------------------
+        .. string.pack("<b", byte_param)
+        .. string.pack("<h", short_param)
+        .. string.pack("<l", long_param)
+        .. string.pack("<f", float_param)
+        .. string.pack("<d", double_param)
+        ---------------------------------------
+
+    network_send_bytestring(true, bytestring)
+
+    djui_chat_message_create('Sent bytestring packet example 1:')
+    djui_chat_message_create('    byte_param:   ' .. byte_param)
+    djui_chat_message_create('    short_param:  ' .. short_param)
+    djui_chat_message_create('    long_param:   ' .. long_param)
+    djui_chat_message_create('    float_param:  ' .. float_param)
+    djui_chat_message_create('    double_param: ' .. double_param)
+end
+
+function on_packet_bytestring_receive_example_1(bytestring)
+    local offset = 1
+
+    local function unpack(fmt)
+        local value
+        value, offset = string.unpack(fmt, bytestring, offset)
+        return value
+    end
+
+    -------------- PACKET ID --------------
+    local packet_id    = unpack("<B")
+    ---------------------------------------
+    local byte_param   = unpack("<b")
+    local short_param  = unpack("<h")
+    local long_param   = unpack("<l")
+    local float_param  = unpack("<f")
+    local double_param = unpack("<d")
+    ---------------------------------------
+
+    djui_chat_message_create('Received bytestring packet example 1:')
+    djui_chat_message_create('    byte_param:   ' .. byte_param)
+    djui_chat_message_create('    short_param:  ' .. short_param)
+    djui_chat_message_create('    long_param:   ' .. long_param)
+    djui_chat_message_create('    float_param:  ' .. float_param)
+    djui_chat_message_create('    double_param: ' .. double_param)
+end
+
+---------------------------------------------------------------------------------------------------
+
+function send_example_2(long_param, string_param)
+
+    local bytestring = ''
+        -------------- PACKET ID --------------
+        .. string.pack("<B", PACKET_EXAMPLE_2)
+        ---------------------------------------
+        .. string.pack("<l",  long_param)
+        .. string.pack("<s2", string_param)
+        ---------------------------------------
+
+    network_send_bytestring(true, bytestring)
+
+    djui_chat_message_create('Sent bytestring packet example 2:')
+    djui_chat_message_create('    byte_param:   ' .. long_param)
+    djui_chat_message_create('    string_param: ' .. string_param)
+end
+
+function on_packet_bytestring_receive_example_2(bytestring)
+    local offset = 1
+
+    local function unpack(fmt)
+        local value
+        value, offset = string.unpack(fmt, bytestring, offset)
+        return value
+    end
+
+    -------------- PACKET ID --------------
+    local packet_id    = unpack("<B")
+    ---------------------------------------
+    local long_param   = unpack("<l")
+    local string_param = unpack("<s2")
+    ---------------------------------------
+
+    djui_chat_message_create('Received bytestring packet example 2:')
+    djui_chat_message_create('    long_param:   ' .. long_param)
+    djui_chat_message_create('    string_param: ' .. string_param)
+end
+
+---------------------------------------------------------------------------------------------------
+
+local sPacketTable = {
+    [PACKET_EXAMPLE_1] = on_packet_bytestring_receive_example_1,
+    [PACKET_EXAMPLE_2] = on_packet_bytestring_receive_example_2,
+}
+
+local function on_packet_bytestring_receive(bytestring)
+    local packet_id = string.unpack("<B", bytestring, 1)
+    if sPacketTable[packet_id] ~= nil then
+        sPacketTable[packet_id](bytestring)
+    end
+end
+
+hook_event(HOOK_ON_PACKET_BYTESTRING_RECEIVE, on_packet_bytestring_receive)
+
+---------------------------------------------------------------------------------------------------
+
+hook_chat_command("example", "[1|2]", function(msg)
+    if msg == '1' then
+        send_example_1(math.random(-127, 127), math.random(-32767, 32767), math.random(-2147483647, 2147483647), math.random(), math.random())
+        return true
+    elseif string.sub(msg, 1, 1) == "2" then
+        send_example_2(math.random(-2147483647, 2147483647), msg)
+        return true
+    end
+    return false
+end)

docs/lua/lua.md

@@ -69,6 +69,7 @@ All of this is a holdover from when there were only two players. It was a reason
 - [Custom HUD Texture](examples/custom-hud-texture)
 - [Custom Audio Test](examples/audio-test)
 - [Custom Texture Overriding](examples/texture-override)
+- [Bytestring Packet Example](examples/bytestring-packet-example.lua)
 
 ## Example Lua mods (large)
 - [Hide and Seek Gamemode](../../mods/hide-and-seek.lua)

src/pc/lua/smlua_constants_autogen.c

@@ -3455,7 +3455,8 @@ char gSmluaConstants[] = ""
 "HOOK_MARIO_OVERRIDE_FLOOR_CLASS=56\n"
 "HOOK_ON_ADD_SURFACE=57\n"
 "HOOK_ON_CLEAR_AREAS=58\n"
-"HOOK_MAX=59\n"
+"HOOK_ON_PACKET_BYTESTRING_RECEIVE=59\n"
+"HOOK_MAX=60\n"
 "HUD_DISPLAY_LIVES=0\n"
 "HUD_DISPLAY_COINS=1\n"
 "HUD_DISPLAY_STARS=2\n"

src/pc/lua/smlua_functions.c

@@ -294,6 +294,18 @@ int smlua_func_network_send_to(lua_State* L) {
     return 1;
 }
 
+int smlua_func_network_send_bytestring(lua_State* L) {
+    if (!smlua_functions_valid_param_count(L, 2)) { return 0; }
+    network_send_lua_custom_bytestring(true);
+    return 1;
+}
+
+int smlua_func_network_send_bytestring_to(lua_State* L) {
+    if (!smlua_functions_valid_param_count(L, 3)) { return 0; }
+    network_send_lua_custom_bytestring(false);
+    return 1;
+}
+
 int smlua_func_set_exclamation_box_contents(lua_State* L) {
     if (!smlua_functions_valid_param_count(L, 1)) { return 0; }
 
@@ -1020,6 +1032,8 @@ void smlua_bind_functions(void) {
     smlua_bind_function(L, "reset_level", smlua_func_reset_level);
     smlua_bind_function(L, "network_send", smlua_func_network_send);
     smlua_bind_function(L, "network_send_to", smlua_func_network_send_to);
+    smlua_bind_function(L, "network_send_bytestring", smlua_func_network_send_bytestring);
+    smlua_bind_function(L, "network_send_bytestring_to", smlua_func_network_send_bytestring_to);
     smlua_bind_function(L, "set_exclamation_box_contents", smlua_func_set_exclamation_box_contents);
     smlua_bind_function(L, "get_exclamation_box_contents", smlua_func_get_exclamation_box_contents);
     smlua_bind_function(L, "get_texture_info", smlua_func_get_texture_info);

src/pc/lua/smlua_hook_events.inl

@@ -57,3 +57,4 @@ SMLUA_EVENT_HOOK(HOOK_ON_INSTANT_WARP, HOOK_RETURN_NEVER, u8 areaIdx, u8 nodeId,
 SMLUA_EVENT_HOOK(HOOK_MARIO_OVERRIDE_FLOOR_CLASS, HOOK_RETURN_ON_OUTPUT_SET, struct MarioState *m, s32 floorClass, OUTPUT s32 *floorClassOverride)
 SMLUA_EVENT_HOOK(HOOK_ON_ADD_SURFACE, HOOK_RETURN_NEVER, struct Surface *surface, bool dynamic)
 SMLUA_EVENT_HOOK(HOOK_ON_CLEAR_AREAS, HOOK_RETURN_NEVER)
+SMLUA_EVENT_HOOK(HOOK_ON_PACKET_BYTESTRING_RECEIVE, HOOK_RETURN_NEVER, s32 modIndex, s32 valueIndex)

src/pc/lua/smlua_hook_events_autogen.inl

@@ -1835,3 +1835,31 @@ bool smlua_call_event_hooks_HOOK_ON_CLEAR_AREAS() {
     }
     return hookResult;
 }
+
+bool smlua_call_event_hooks_HOOK_ON_PACKET_BYTESTRING_RECEIVE(s32 modIndex, s32 valueIndex) {
+    lua_State *L = gLuaState;
+    if (L == NULL) { return false; }
+    bool hookResult = false;
+
+    struct LuaHookedEvent *hook = &sHookedEvents[HOOK_ON_PACKET_BYTESTRING_RECEIVE];
+    for (int i = 0; i < hook->count; i++) {
+        if (hook->mod[i]->index != modIndex) { continue; }
+        s32 prevTop = lua_gettop(L);
+
+        // push the callback onto the stack
+        lua_rawgeti(L, LUA_REGISTRYINDEX, hook->reference[i]);
+
+        // push valueIndex
+        lua_pushvalue(L, valueIndex);
+
+        // call the callback
+        if (0 != smlua_call_hook(L, 1, 0, 0, hook->mod[i], hook->modFile[i])) {
+            LOG_LUA("Failed to call the callback for hook %s", sLuaHookedEventTypeName[HOOK_ON_PACKET_BYTESTRING_RECEIVE]);
+            continue;
+        }
+        hookResult = true;
+
+        lua_settop(L, prevTop);
+    }
+    return hookResult;
+}

src/pc/lua/smlua_hooks.h

@@ -73,6 +73,7 @@ enum LuaHookedEventType {
     HOOK_MARIO_OVERRIDE_FLOOR_CLASS,
     HOOK_ON_ADD_SURFACE,
     HOOK_ON_CLEAR_AREAS,
+    HOOK_ON_PACKET_BYTESTRING_RECEIVE,
     HOOK_MAX,
 };
 

src/pc/network/packets/packet.c

@@ -137,6 +137,7 @@ void packet_process(struct Packet* p) {
         case PACKET_REQUEST_FAILED:          network_receive_request_failed(p);          break;
 
         case PACKET_LUA_CUSTOM:              network_receive_lua_custom(p);              break;
+        case PACKET_LUA_CUSTOM_BYTESTRING:   network_receive_lua_custom_bytestring(p);   break;
 
         // custom
         case PACKET_CUSTOM:                  network_receive_custom(p);                  break;

src/pc/network/packets/packet.h

@@ -73,6 +73,7 @@ enum PacketType {
     PACKET_REQUEST_FAILED,
 
     PACKET_LUA_CUSTOM,
+    PACKET_LUA_CUSTOM_BYTESTRING,
     
     PACKET_COMMAND,
     PACKET_MODERATOR,
@@ -380,5 +381,7 @@ void network_receive_request_failed(struct Packet* p);
 // packet_lua_custom.c
 void network_send_lua_custom(bool broadcast);
 void network_receive_lua_custom(struct Packet* p);
+void network_send_lua_custom_bytestring(bool broadcast);
+void network_receive_lua_custom_bytestring(struct Packet* p);
 
 #endif

src/pc/network/packets/packet_lua_custom.c

@@ -6,7 +6,6 @@
 #include "pc/debuglog.h"
 
 void network_send_lua_custom(bool broadcast) {
-    LOG_INFO("Sending lua custom packet");
     lua_State* L = gLuaState;
     u16 zero = 0;
     s32 paramIndex = 1;
@@ -97,7 +96,6 @@ void network_send_lua_custom(bool broadcast) {
 }
 
 void network_receive_lua_custom(struct Packet* p) {
-    LOG_INFO("Receiving lua custom packet");
     lua_State* L = gLuaState;
     u16 modIndex = 0;
     u8  keyCount = 0;
@@ -132,3 +130,117 @@ void network_receive_lua_custom(struct Packet* p) {
     smlua_call_event_hooks(HOOK_ON_PACKET_RECEIVE, modIndex, tableIndex);
     lua_pop(L, 1); // pop table
 }
+
+  ////////////////
+ // bytestring //
+////////////////
+
+#define MAX_BYTESTRING_LENGTH (PACKET_LENGTH - 15)
+
+void network_send_lua_custom_bytestring(bool broadcast) {
+    lua_State* L = gLuaState;
+    s32 paramIndex = 1;
+
+    if (!L) {
+        LOG_ERROR("Sent lua custom bytestring packet when lua is dead");
+        return;
+    }
+
+    // figure out mod index
+    if (gLuaActiveMod == NULL) {
+        LOG_LUA_LINE("Could not figure out the current active mod!");
+        return;
+    }
+    u16 modIndex = gLuaActiveMod->index;
+
+    // get local index
+    s32 toLocalIndex = 0;
+    if (!broadcast) {
+        toLocalIndex = smlua_to_integer(L, paramIndex++);
+        if (toLocalIndex <= 0 || toLocalIndex >= MAX_PLAYERS) {
+            LOG_LUA_LINE("Tried to send bytestring packet to invalid local index: %d", toLocalIndex)
+            return;
+        }
+        if (!gSmLuaConvertSuccess) {
+            LOG_LUA("Invalid 'localIndex' type");
+            return;
+        }
+    }
+
+    // get reliability
+    bool reliability = smlua_to_boolean(L, paramIndex++);
+    if (!gSmLuaConvertSuccess) {
+        LOG_LUA("Invalid 'reliable' type");
+        return;
+    }
+
+    // write packet header
+    struct Packet p = { 0 };
+    packet_init(&p, PACKET_LUA_CUSTOM_BYTESTRING, reliability, PLMT_NONE);
+    packet_write(&p, &modIndex, sizeof(u16));
+
+    // make sure value passed in is a string
+    s32 bytestringIndex = paramIndex;
+    if (lua_type(L, bytestringIndex) != LUA_TSTRING) {
+        LOG_LUA_LINE("Tried to send a bytestring packet with a non-string");
+        return;
+    }
+
+    // get string information
+    size_t totalLength = 0;
+    const char* bytestring = lua_tolstring(L, bytestringIndex, &totalLength);
+
+    // check length
+    if (totalLength <= 0 || totalLength > MAX_BYTESTRING_LENGTH) {
+        LOG_LUA_LINE("Tried to send a bytestring packet with an invalid length '%llu'. Must be above 0 and below '%u'", (u64)totalLength, MAX_BYTESTRING_LENGTH);
+    }
+
+    // write length
+    u16 bytestringLength = totalLength;
+    packet_write(&p, &bytestringLength, sizeof(u16));
+
+    // write bytestring
+    packet_write(&p, (char*)bytestring, totalLength);
+
+    // send packet
+    if (broadcast) {
+        network_send(&p);
+    } else {
+        network_send_to(toLocalIndex, &p);
+    }
+}
+
+void network_receive_lua_custom_bytestring(struct Packet* p) {
+    lua_State* L = gLuaState;
+    u16 modIndex = 0;
+    u16 bytestringLength = 0;
+    packet_read(p, &modIndex, sizeof(u16));
+    packet_read(p, &bytestringLength, sizeof(u16));
+
+    if (!L) {
+        LOG_ERROR("Received lua custom bytestring packet when lua is dead");
+        return;
+    }
+
+    if (bytestringLength == 0 || bytestringLength > MAX_BYTESTRING_LENGTH) {
+        LOG_ERROR("Received lua custom bytestring packet with an invalid length");
+        return;
+    }
+
+    // read byte string
+    static char sBytestring[MAX_BYTESTRING_LENGTH + 1] = "";
+    packet_read(p, &sBytestring, bytestringLength);
+
+    if (p->error) {
+        LOG_ERROR("Received malformed lua custom bytestring packet");
+        return;
+    }
+
+    // push bytestring
+    lua_pushlstring(L, sBytestring, bytestringLength);
+    s32 bytestringIndex = lua_gettop(L);
+
+    // call hook
+    smlua_call_event_hooks(HOOK_ON_PACKET_BYTESTRING_RECEIVE, modIndex, bytestringIndex);
+    lua_pop(L, 1); // pop bytestring
+}