give Lua mods some more control over dialog boxes (#19)

exposed set_menu_mode, create_dialog_box, create_dialog_box_with_response, create_dialog_box_with_var, create_dialog_inverted_box, reset_dialog_render_state
added and exposed get_dialog_response
HOOK_ON_DIALOG can now take an extra return value as a string, which will be used instead of what ever was going to be in the textbox.

autogen/convert_functions.py

@@ -78,7 +78,7 @@ override_allowed_functions = {
     "src/game/level_update.h":              [ "level_trigger_warp", "get_painting_warp_node", "initiate_painting_warp", "warp_special", "lvl_set_current_level", "level_control_timer_running" ],
     "src/game/area.h":                      [ "area_get_warp_node" ],
     "src/engine/level_script.h":            [ "area_create_warp_node" ],
-    "src/game/ingame_menu.h":               [ "set_min_dialog_width", "set_dialog_override_pos", "reset_dialog_override_pos", "set_dialog_override_color", "reset_dialog_override_color" ]
+    "src/game/ingame_menu.h":               [ "set_min_dialog_width", "set_dialog_override_pos", "reset_dialog_override_pos", "set_dialog_override_color", "reset_dialog_override_color", "set_menu_mode", "create_dialog_box", "create_dialog_box_with_var", "create_dialog_inverted_box", "create_dialog_box_with_response", "reset_dialog_render_state", "close_dialog_box", ]
 }
 
 override_disallowed_functions = {

autogen/lua_definitions/functions.lua

@@ -4022,6 +4022,31 @@ function set_first_person_enabled(enable)
     -- ...
 end
 
+--- @param dialog integer
+--- @return nil
+function create_dialog_box(dialog)
+    -- ...
+end
+
+--- @param dialog integer
+--- @return nil
+function create_dialog_box_with_response(dialog)
+    -- ...
+end
+
+--- @param dialog integer
+--- @param dialogVar integer
+--- @return nil
+function create_dialog_box_with_var(dialog, dialogVar)
+    -- ...
+end
+
+--- @param dialog integer
+--- @return nil
+function create_dialog_inverted_box(dialog)
+    -- ...
+end
+
 --- @return nil
 function reset_dialog_override_color()
     -- ...
@@ -4032,6 +4057,11 @@ function reset_dialog_override_pos()
     -- ...
 end
 
+--- @return nil
+function reset_dialog_render_state()
+    -- ...
+end
+
 --- @param bgR integer
 --- @param bgG integer
 --- @param bgB integer
@@ -4052,6 +4082,12 @@ function set_dialog_override_pos(x, y)
     -- ...
 end
 
+--- @param mode integer
+--- @return nil
+function set_menu_mode(mode)
+    -- ...
+end
+
 --- @param width integer
 --- @return nil
 function set_min_dialog_width(width)
@@ -8943,6 +8979,11 @@ function get_dialog_id()
     -- ...
 end
 
+--- @return integer
+function get_dialog_response()
+    -- ...
+end
+
 --- @return integer
 function get_envfx()
     -- ...

docs/lua/functions-3.md

@@ -3348,6 +3348,87 @@
 <br />
 
 
+## [create_dialog_box](#create_dialog_box)
+
+### Lua Example
+`create_dialog_box(dialog)`
+
+### Parameters
+| Field | Type |
+| ----- | ---- |
+| dialog | `integer` |
+
+### Returns
+- None
+
+### C Prototype
+`void create_dialog_box(s16 dialog);`
+
+[:arrow_up_small:](#)
+
+<br />
+
+## [create_dialog_box_with_response](#create_dialog_box_with_response)
+
+### Lua Example
+`create_dialog_box_with_response(dialog)`
+
+### Parameters
+| Field | Type |
+| ----- | ---- |
+| dialog | `integer` |
+
+### Returns
+- None
+
+### C Prototype
+`void create_dialog_box_with_response(s16 dialog);`
+
+[:arrow_up_small:](#)
+
+<br />
+
+## [create_dialog_box_with_var](#create_dialog_box_with_var)
+
+### Lua Example
+`create_dialog_box_with_var(dialog, dialogVar)`
+
+### Parameters
+| Field | Type |
+| ----- | ---- |
+| dialog | `integer` |
+| dialogVar | `integer` |
+
+### Returns
+- None
+
+### C Prototype
+`void create_dialog_box_with_var(s16 dialog, s32 dialogVar);`
+
+[:arrow_up_small:](#)
+
+<br />
+
+## [create_dialog_inverted_box](#create_dialog_inverted_box)
+
+### Lua Example
+`create_dialog_inverted_box(dialog)`
+
+### Parameters
+| Field | Type |
+| ----- | ---- |
+| dialog | `integer` |
+
+### Returns
+- None
+
+### C Prototype
+`void create_dialog_inverted_box(s16 dialog);`
+
+[:arrow_up_small:](#)
+
+<br />
+
 ## [reset_dialog_override_color](#reset_dialog_override_color)
 
 ### Lua Example
@@ -3384,6 +3465,24 @@
 
 <br />
 
+## [reset_dialog_render_state](#reset_dialog_render_state)
+
+### Lua Example
+`reset_dialog_render_state()`
+
+### Parameters
+- None
+
+### Returns
+- None
+
+### C Prototype
+`void reset_dialog_render_state(void);`
+
+[:arrow_up_small:](#)
+
+<br />
+
 ## [set_dialog_override_color](#set_dialog_override_color)
 
 ### Lua Example
@@ -3432,6 +3531,26 @@
 
 <br />
 
+## [set_menu_mode](#set_menu_mode)
+
+### Lua Example
+`set_menu_mode(mode)`
+
+### Parameters
+| Field | Type |
+| ----- | ---- |
+| mode | `integer` |
+
+### Returns
+- None
+
+### C Prototype
+`void set_menu_mode(s16 mode);`
+
+[:arrow_up_small:](#)
+
+<br />
+
 ## [set_min_dialog_width](#set_min_dialog_width)
 
 ### Lua Example

docs/lua/functions-5.md

@@ -1340,6 +1340,24 @@
 
 <br />
 
+## [get_dialog_response](#get_dialog_response)
+
+### Lua Example
+`local integerValue = get_dialog_response()`
+
+### Parameters
+- None
+
+### Returns
+- `integer`
+
+### C Prototype
+`s32 get_dialog_response();`
+
+[:arrow_up_small:](#)
+
+<br />
+
 ## [get_envfx](#get_envfx)
 
 ### Lua Example

docs/lua/functions.md

@@ -808,10 +808,16 @@
 <br />
 
 - ingame_menu.h
+   - [create_dialog_box](functions-3.md#create_dialog_box)
+   - [create_dialog_box_with_response](functions-3.md#create_dialog_box_with_response)
+   - [create_dialog_box_with_var](functions-3.md#create_dialog_box_with_var)
+   - [create_dialog_inverted_box](functions-3.md#create_dialog_inverted_box)
    - [reset_dialog_override_color](functions-3.md#reset_dialog_override_color)
    - [reset_dialog_override_pos](functions-3.md#reset_dialog_override_pos)
+   - [reset_dialog_render_state](functions-3.md#reset_dialog_render_state)
    - [set_dialog_override_color](functions-3.md#set_dialog_override_color)
    - [set_dialog_override_pos](functions-3.md#set_dialog_override_pos)
+   - [set_menu_mode](functions-3.md#set_menu_mode)
    - [set_min_dialog_width](functions-3.md#set_min_dialog_width)
 
 <br />
@@ -1661,6 +1667,7 @@
    - [get_date_and_time](functions-5.md#get_date_and_time)
    - [get_dialog_box_state](functions-5.md#get_dialog_box_state)
    - [get_dialog_id](functions-5.md#get_dialog_id)
+   - [get_dialog_response](functions-5.md#get_dialog_response)
    - [get_envfx](functions-5.md#get_envfx)
    - [get_environment_region](functions-5.md#get_environment_region)
    - [get_fog_color](functions-5.md#get_fog_color)

docs/lua/guides/hooks.md

@@ -121,7 +121,7 @@ The lua functions sent to `hook_event()` will be automatically called by SM64 wh
 | HOOK_BEFORE_SET_MARIO_ACTION | Called before Mario's action changes Return an action to change the incoming action or `1` to cancel the action change | [MarioState](../structs.md#MarioState) mario, `integer` incomingAction |
 | HOOK_JOINED_GAME | Called when the local player finishes the join process (if the player isn't the host) | None |
 | HOOK_ON_OBJECT_ANIM_UPDATE | Called when an object's animation is updated | [Object](../structs.md#Object) objNode |
-| HOOK_ON_DIALOG | Called when a dialog appears. Return `false` to prevent it from appearing | `integer` dialogId |
+| HOOK_ON_DIALOG | Called when a dialog appears. Return `false` to prevent it from appearing. Return a second parameter as any string to override the text in the textbox | `integer` dialogId |
 | HOOK_ON_EXIT | Called before the game shuts down | None |
 | HOOK_DIALOG_SOUND | Called when a dialog box sound is going to play, return a `DS_*` constant to override the sound | `integer` dialogSound |
 | HOOK_ON_COLLIDE_LEVEL_BOUNDS | Called when a mario collides with the level boundaries | [MarioState](../structs.md#MarioState) mario |

src/game/ingame_menu.c

@@ -1156,9 +1156,14 @@ void handle_special_dialog_text(s16 dialogID) { // dialog ID tables, in order
     }
 }
 
+static u8 sHookString[255];
+static bool sOverrideDialogString = false;
+void convert_string_ascii_to_sm64(u8 *str64, const char *strAscii, bool menu);
 void handle_dialog_hook(s16 dialogId) {
     bool open = false;
-    smlua_call_event_hooks_int_params_ret_bool(HOOK_ON_DIALOG, dialogId, &open);
+    const char *str = smlua_call_event_hooks_int_ret_bool_and_string(HOOK_ON_DIALOG, dialogId, &open);
+    sOverrideDialogString = str != NULL;
+    if (sOverrideDialogString) { convert_string_ascii_to_sm64(sHookString, str, false); }
     if (!open) {
         gDialogLineNum = 1;
         gDialogBoxState = DIALOG_STATE_CLOSING;
@@ -1521,7 +1526,7 @@ void handle_dialog_text_and_pages(s8 colorMode, struct DialogEntry *dialog, s8 l
 
     u8 strChar;
 
-    u8 *str = segmented_to_virtual(dialog->str);
+    u8 *str = sOverrideDialogString ? sHookString : segmented_to_virtual(dialog->str);
     s8 lineNum = 1;
 
     s8 totalLines;
@@ -2297,7 +2302,7 @@ void print_peach_letter_message(void) {
 #endif
     dialog = segmented_to_virtual(dialogTable[gDialogID]);
 
-    str = segmented_to_virtual(dialog->str);
+    str = sOverrideDialogString ? sHookString : segmented_to_virtual(dialog->str);
 
     create_dl_translation_matrix(MENU_MTX_PUSH, 97.0f, 118.0f, 0);
 
@@ -3581,3 +3586,8 @@ void set_dialog_override_color(u8 bgR, u8 bgG, u8 bgB, u8 bgA, u8 textR, u8 text
 void reset_dialog_override_color(void) {
     gOverrideDialogColor = 0;
 }
+
+void close_dialog_box(u8 state) {
+    if (state > DIALOG_STATE_CLOSING) { return; }
+    gDialogBoxState = state;
+}

src/game/level_info.c

@@ -102,7 +102,7 @@ static char *sm64_to_ascii_char(char *strAscii, const u8 *str64) {
     return strAscii + 1;
 }
 
-static void convert_string_ascii_to_sm64(u8 *str64, const char *strAscii, bool menu) {
+void convert_string_ascii_to_sm64(u8 *str64, const char *strAscii, bool menu) {
     for (; *strAscii != 0; str64++) {
         strAscii = ascii_to_sm64_char(str64, strAscii, menu);
     }

src/pc/lua/smlua_functions_autogen.c

@@ -13252,6 +13252,76 @@ int smlua_func_set_first_person_enabled(lua_State* L) {
  // ingame_menu.h //
 ///////////////////
 
+int smlua_func_create_dialog_box(lua_State* L) {
+    if (L == NULL) { return 0; }
+
+    int top = lua_gettop(L);
+    if (top != 1) {
+        LOG_LUA_LINE("Improper param count for '%s': Expected %u, Received %u", "create_dialog_box", 1, top);
+        return 0;
+    }
+
+    s16 dialog = smlua_to_integer(L, 1);
+    if (!gSmLuaConvertSuccess) { LOG_LUA("Failed to convert parameter %u for function '%s'", 1, "create_dialog_box"); return 0; }
+
+    create_dialog_box(dialog);
+
+    return 1;
+}
+
+int smlua_func_create_dialog_box_with_response(lua_State* L) {
+    if (L == NULL) { return 0; }
+
+    int top = lua_gettop(L);
+    if (top != 1) {
+        LOG_LUA_LINE("Improper param count for '%s': Expected %u, Received %u", "create_dialog_box_with_response", 1, top);
+        return 0;
+    }
+
+    s16 dialog = smlua_to_integer(L, 1);
+    if (!gSmLuaConvertSuccess) { LOG_LUA("Failed to convert parameter %u for function '%s'", 1, "create_dialog_box_with_response"); return 0; }
+
+    create_dialog_box_with_response(dialog);
+
+    return 1;
+}
+
+int smlua_func_create_dialog_box_with_var(lua_State* L) {
+    if (L == NULL) { return 0; }
+
+    int top = lua_gettop(L);
+    if (top != 2) {
+        LOG_LUA_LINE("Improper param count for '%s': Expected %u, Received %u", "create_dialog_box_with_var", 2, top);
+        return 0;
+    }
+
+    s16 dialog = smlua_to_integer(L, 1);
+    if (!gSmLuaConvertSuccess) { LOG_LUA("Failed to convert parameter %u for function '%s'", 1, "create_dialog_box_with_var"); return 0; }
+    s32 dialogVar = smlua_to_integer(L, 2);
+    if (!gSmLuaConvertSuccess) { LOG_LUA("Failed to convert parameter %u for function '%s'", 2, "create_dialog_box_with_var"); return 0; }
+
+    create_dialog_box_with_var(dialog, dialogVar);
+
+    return 1;
+}
+
+int smlua_func_create_dialog_inverted_box(lua_State* L) {
+    if (L == NULL) { return 0; }
+
+    int top = lua_gettop(L);
+    if (top != 1) {
+        LOG_LUA_LINE("Improper param count for '%s': Expected %u, Received %u", "create_dialog_inverted_box", 1, top);
+        return 0;
+    }
+
+    s16 dialog = smlua_to_integer(L, 1);
+    if (!gSmLuaConvertSuccess) { LOG_LUA("Failed to convert parameter %u for function '%s'", 1, "create_dialog_inverted_box"); return 0; }
+
+    create_dialog_inverted_box(dialog);
+
+    return 1;
+}
+
 int smlua_func_reset_dialog_override_color(UNUSED lua_State* L) {
     if (L == NULL) { return 0; }
 
@@ -13282,6 +13352,21 @@ int smlua_func_reset_dialog_override_pos(UNUSED lua_State* L) {
     return 1;
 }
 
+int smlua_func_reset_dialog_render_state(UNUSED lua_State* L) {
+    if (L == NULL) { return 0; }
+
+    int top = lua_gettop(L);
+    if (top != 0) {
+        LOG_LUA_LINE("Improper param count for '%s': Expected %u, Received %u", "reset_dialog_render_state", 0, top);
+        return 0;
+    }
+
+
+    reset_dialog_render_state();
+
+    return 1;
+}
+
 int smlua_func_set_dialog_override_color(lua_State* L) {
     if (L == NULL) { return 0; }
 
@@ -13332,6 +13417,23 @@ int smlua_func_set_dialog_override_pos(lua_State* L) {
     return 1;
 }
 
+int smlua_func_set_menu_mode(lua_State* L) {
+    if (L == NULL) { return 0; }
+
+    int top = lua_gettop(L);
+    if (top != 1) {
+        LOG_LUA_LINE("Improper param count for '%s': Expected %u, Received %u", "set_menu_mode", 1, top);
+        return 0;
+    }
+
+    s16 mode = smlua_to_integer(L, 1);
+    if (!gSmLuaConvertSuccess) { LOG_LUA("Failed to convert parameter %u for function '%s'", 1, "set_menu_mode"); return 0; }
+
+    set_menu_mode(mode);
+
+    return 1;
+}
+
 int smlua_func_set_min_dialog_width(lua_State* L) {
     if (L == NULL) { return 0; }
 
@@ -29402,6 +29504,21 @@ int smlua_func_get_dialog_id(UNUSED lua_State* L) {
     return 1;
 }
 
+int smlua_func_get_dialog_response(UNUSED lua_State* L) {
+    if (L == NULL) { return 0; }
+
+    int top = lua_gettop(L);
+    if (top != 0) {
+        LOG_LUA_LINE("Improper param count for '%s': Expected %u, Received %u", "get_dialog_response", 0, top);
+        return 0;
+    }
+
+
+    lua_pushinteger(L, get_dialog_response());
+
+    return 1;
+}
+
 int smlua_func_get_envfx(UNUSED lua_State* L) {
     if (L == NULL) { return 0; }
 
@@ -32781,10 +32898,16 @@ void smlua_bind_functions_autogen(void) {
     smlua_bind_function(L, "set_first_person_enabled", smlua_func_set_first_person_enabled);
 
     // ingame_menu.h
+    smlua_bind_function(L, "create_dialog_box", smlua_func_create_dialog_box);
+    smlua_bind_function(L, "create_dialog_box_with_response", smlua_func_create_dialog_box_with_response);
+    smlua_bind_function(L, "create_dialog_box_with_var", smlua_func_create_dialog_box_with_var);
+    smlua_bind_function(L, "create_dialog_inverted_box", smlua_func_create_dialog_inverted_box);
     smlua_bind_function(L, "reset_dialog_override_color", smlua_func_reset_dialog_override_color);
     smlua_bind_function(L, "reset_dialog_override_pos", smlua_func_reset_dialog_override_pos);
+    smlua_bind_function(L, "reset_dialog_render_state", smlua_func_reset_dialog_render_state);
     smlua_bind_function(L, "set_dialog_override_color", smlua_func_set_dialog_override_color);
     smlua_bind_function(L, "set_dialog_override_pos", smlua_func_set_dialog_override_pos);
+    smlua_bind_function(L, "set_menu_mode", smlua_func_set_menu_mode);
     smlua_bind_function(L, "set_min_dialog_width", smlua_func_set_min_dialog_width);
 
     // interaction.h
@@ -33589,6 +33712,7 @@ void smlua_bind_functions_autogen(void) {
     smlua_bind_function(L, "get_date_and_time", smlua_func_get_date_and_time);
     smlua_bind_function(L, "get_dialog_box_state", smlua_func_get_dialog_box_state);
     smlua_bind_function(L, "get_dialog_id", smlua_func_get_dialog_id);
+    smlua_bind_function(L, "get_dialog_response", smlua_func_get_dialog_response);
     smlua_bind_function(L, "get_envfx", smlua_func_get_envfx);
     smlua_bind_function(L, "get_environment_region", smlua_func_get_environment_region);
     smlua_bind_function(L, "get_fog_color", smlua_func_get_fog_color);

src/pc/lua/smlua_hooks.c

@@ -1052,6 +1052,42 @@ void smlua_call_event_hooks_graph_node_object_and_int_param(enum LuaHookedEventT
     }
 }
 
+const char *smlua_call_event_hooks_int_ret_bool_and_string(enum LuaHookedEventType hookType, s32 param, bool* returnValue) {
+    lua_State* L = gLuaState;
+    if (L == NULL) { return NULL; }
+    *returnValue = true;
+    const char *retString = NULL;
+
+    struct LuaHookedEvent* hook = &sHookedEvents[hookType];
+    for (int i = 0; i < hook->count; i++) {
+        s32 prevTop = lua_gettop(L);
+
+        // push the callback onto the stack
+        lua_rawgeti(L, LUA_REGISTRYINDEX, hook->reference[i]);
+
+        // push param
+        lua_pushinteger(L, param);
+
+        // call the callback
+        if (0 != smlua_call_hook(L, 1, 2, 0, hook->mod[i])) {
+            LOG_LUA("Failed to call the callback: %u", hookType);
+            continue;
+        }
+
+        // output the return values
+        if (lua_type(L, -2) == LUA_TBOOLEAN) {
+            *returnValue = smlua_to_boolean(L, -2);
+        }
+        if (lua_type(L, -1) == LUA_TSTRING) {
+            retString = smlua_to_string(L, -1);
+            lua_settop(L, prevTop);
+            return retString;
+        }
+        lua_settop(L, prevTop);
+    }
+    return NULL;
+}
+
   ////////////////////
  // hooked actions //
 ////////////////////

src/pc/lua/smlua_hooks.h

@@ -150,6 +150,7 @@ bool smlua_call_event_hooks_mario_param_ret_float(enum LuaHookedEventType hookTy
 bool smlua_call_event_hooks_mario_param_and_int_and_int_ret_int(enum LuaHookedEventType hookType, struct MarioState* m, s32 param, u32 args, s32* returnValue);
 void smlua_call_event_hooks_graph_node_object_and_int_param(enum LuaHookedEventType hookType, struct GraphNodeObject* node, s32 param);
 void smlua_call_event_hooks_on_seq_load(enum LuaHookedEventType hookType, u32 player, u32 seqId, s32 loadAsync, u8* returnValue);
+const char *smlua_call_event_hooks_int_ret_bool_and_string(enum LuaHookedEventType hookType, s32 param, bool* returnValue);
 
 enum BehaviorId smlua_get_original_behavior_id(const BehaviorScript* behavior);
 const BehaviorScript* smlua_override_behavior(const BehaviorScript* behavior);

src/pc/lua/utils/smlua_misc_utils.c

@@ -654,3 +654,9 @@ const char* get_os_name(void) {
     return "Unknown";
 #endif
 }
+
+///
+
+s32 get_dialog_response() {
+    return gDialogResponse;
+}

src/pc/lua/utils/smlua_misc_utils.h

@@ -163,4 +163,6 @@ void reset_window_title(void);
 
 const char* get_os_name(void);
 
+s32 get_dialog_response();
+
 #endif