Use UTF-8 for djui text

src/pc/crash_handler.c

@@ -24,6 +24,7 @@
 #include "game/mario.h"
 #include "gfx_dimensions.h"
 #include "src/pc/djui/djui.h"
+#include "src/pc/djui/djui_unicode.h"
 #include "pc/network/network.h"
 #include "pc/gfx/gfx_rendering_api.h"
 #include "pc/mods/mods.h"
@@ -168,13 +169,13 @@ static void crash_handler_produce_one_frame() {
 
         // render the line
         f32 addX = 0;
-        size_t length = strlen(text->s);
+        char* c = text->s;
-        for (size_t i = 0; i < length; i++) {
+        while (*c != '\0') {
-            char c = text->s[i];
             f32 charWidth = 0.4f;
 
             if (c <= 0x20 || c >= 0x7F) {
                 addX += charWidth;
+                c = djui_unicode_next_char(c);
                 continue;
             }
 
@@ -185,6 +186,7 @@ static void crash_handler_produce_one_frame() {
             // render
             font->render_char(c);
             create_dl_translation_matrix(DJUI_MTX_NOPUSH, charWidth, 0, 0);
+            c = djui_unicode_next_char(c);
         }
 
         // pop

src/pc/djui/djui_font.c

@@ -1,156 +1,7 @@
 #include "djui.h"
+#include "djui_unicode.h"
 #include "game/segment2.h"
 
-struct SmCodeGlyph {
-    char unicode[3];
-    char base;
-    f32 width;
-};
-
-struct SmCodeGlyph sSmCodeGlyphs[] = {
-    { "Á", 'A', 0 },
-    { "Å", 'A', 0 },
-    { "Â", 'A', 0 },
-    { "À", 'A', 0 },
-    { "Ã", 'A', 0 },
-    { "Ä", 'A', 0 },
-    { "Ç", 'C', 0 },
-    { "É", 'E', 0 },
-    { "Ê", 'E', 0 },
-    { "È", 'E', 0 },
-    { "Ë", 'E', 0 },
-    { "Í", 'I', 0 },
-    { "Î", 'I', 0 },
-    { "Ì", 'I', 0 },
-    { "Ï", 'I', 0 },
-    { "Ñ", 'N', 0 },
-    { "Ó", 'O', 0 },
-    { "Ô", 'O', 0 },
-    { "Ò", 'O', 0 },
-    { "Õ", 'O', 0 },
-    { "Ö", 'O', 0 },
-    { "Ú", 'U', 0 },
-    { "Û", 'U', 0 },
-    { "Ù", 'U', 0 },
-    { "Ü", 'U', 0 },
-    { "Ý", 'Y', 0 },
-    { "Ÿ", 'Y', 0 },
-
-    { "á", 'a', 0 },
-    { "å", 'a', 0 },
-    { "â", 'a', 0 },
-    { "à", 'a', 0 },
-    { "ã", 'a', 0 },
-    { "ä", 'a', 0 },
-    { "ç", 'c', 0 },
-    { "é", 'e', 0 },
-    { "ê", 'e', 0 },
-    { "è", 'e', 0 },
-    { "ë", 'e', 0 },
-    { "í", 'i', 0 },
-    { "î", 'i', 0 },
-    { "ì", 'i', 0 },
-    { "ï", 'i', 0 },
-    { "ñ", 'n', 0 },
-    { "ó", 'o', 0 },
-    { "ô", 'o', 0 },
-    { "ò", 'o', 0 },
-    { "õ", 'o', 0 },
-    { "ö", 'o', 0 },
-    { "ú", 'u', 0 },
-    { "û", 'u', 0 },
-    { "ù", 'u', 0 },
-    { "ü", 'u', 0 },
-    { "ý", 'y', 0 },
-    { "ÿ", 'y', 0 },
-
-    { "æ", 'a', 0.5000f },
-    { "Æ", 'a', 0.6000f },
-    { "œ", 'o', 0.5000f },
-    { "Œ", 'o', 0.5000f },
-    { "ð", 'd', 0 },
-    { "Ð", 'D', 0.4375f },
-    { "ø", 'o', 0 },
-    { "Ø", 'O', 0 },
-    { "ß", 'S', 0 },
-
-    { "¡", '!', 0 },
-    { "¿", '?', 0 },
-};
-
-
-u8 djui_font_convert_smcode_to_base(char c) {
-    if ((u8)c < 128) {
-        return c;
-    }
-    size_t glyphCount = sizeof(sSmCodeGlyphs) / sizeof(sSmCodeGlyphs[0]);
-    u8 max = 128 + glyphCount;
-    if ((u8)c > max) {
-        return '?';
-    }
-    return sSmCodeGlyphs[((u8)c - 128)].base;
-}
-
-void djui_font_convert_to_unicode(char* from, char* to, int length, int maxlength) {
-    int clen = 0;
-    int count = 0;
-    to[0] = '\0';
-    while (*from != '\0' && count < length) {
-        count++;
-        if ((u8)*from < 128 || !djui_font_valid_smcode(*from)) {
-            clen = strlen(to);
-            snprintf(to + clen, maxlength - clen, "%c", *from);
-            from++;
-            continue;
-        }
-
-        int i = (u8)*from - 128;
-        struct SmCodeGlyph* glyph = &sSmCodeGlyphs[i];
-        clen = strlen(to);
-        snprintf(to + clen, maxlength - clen, "%s", glyph->unicode);
-
-        from++;
-    }
-}
-
-void djui_font_convert_to_smcode(char* text) {
-    size_t glyphCount = sizeof(sSmCodeGlyphs) / sizeof(sSmCodeGlyphs[0]);
-
-    //printf("....................\n");
-    //printf("%s\n", text);
-    char* t = text;
-    while (*t != '\0') {
-        //printf("%d ", *t);
-        for (size_t i = 0; i < glyphCount; i++) {
-            struct SmCodeGlyph* glyph = &sSmCodeGlyphs[i];
-            if (t[0] == glyph->unicode[0] && t[1] == glyph->unicode[1]) {
-                // consume down to one character
-                char* t2 = t;
-                while (*t2 != '\0') { t2[0] = t2[1]; t2++; }
-                // replace
-                t[0] = (s8)(128 + i);
-            }
-        }
-        t++;
-    }
-    //printf("\n....................\n");
-}
-
-bool djui_font_valid_smcode(char c) {
-    if (c >= '!' && (u8)c <= ((u8)'~' + 1)) {
-        return true;
-    } else if (c == ' ') {
-        return true;
-    }
-
-    size_t glyphCount = sizeof(sSmCodeGlyphs) / sizeof(sSmCodeGlyphs[0]);
-    for (size_t i = 0; i < glyphCount; i++) {
-        if ((u8)c == ((u8)(128 + i))) { return true; }
-    }
-
-    return false;
-}
-
   ///////////////////////////////////
  // font 0 (built-in normal font) //
 ///////////////////////////////////
@@ -182,38 +33,23 @@ const Gfx dl_font_normal_display_list[] = {
     gsSPEndDisplayList(),
 };
 
-static void djui_font_normal_render_char(char c) {
+static void djui_font_normal_render_char(char* c) {
-    extern const u8* const font_normal_chars[];
     // replace undisplayable characters
-    if (!djui_font_valid_smcode(c)) { c = '?'; }
+    if (*c == ' ') { return; }
-    if (c == ' ') { return; }
+
-    void* fontChar = (void*)font_normal_chars[(u8)c - '!'];
+    u32 index = djui_unicode_get_sprite_index(c);
-    if (fontChar == NULL) { fontChar = (void*)font_normal_chars[94]; }
+
+    extern const u8* const font_normal_chars[];
+    void* fontChar = (void*)font_normal_chars[index];
 
     gDPSetTextureImage(gDisplayListHead++, G_IM_FMT_IA, G_IM_SIZ_16b, 1, (void*)fontChar);
     gSPDisplayList(gDisplayListHead++, dl_font_normal_display_list);
 }
 
-static f32 djui_font_normal_char_width(char c) {
+static f32 djui_font_normal_char_width(char* c) {
-    if (c == ' ') { return 0.30f; }
+    if (*c == ' ') { return 0.30f; }
     extern const f32 font_normal_widths[];
-
+    return djui_unicode_get_sprite_width(c, font_normal_widths);
-    if ((u8)c < 128) {
-        return font_normal_widths[(u8)c - '!'];
-    }
-
-    size_t glyphCount = sizeof(sSmCodeGlyphs) / sizeof(sSmCodeGlyphs[0]);
-    u8 max = 128 + glyphCount;
-    if ((u8)c > max) {
-        return font_normal_widths[(u8)'?' - '!'];
-    }
-
-    if (sSmCodeGlyphs[(u8)c - 128].width > 0) {
-        return sSmCodeGlyphs[(u8)c - 128].width;
-    }
-
-    c = djui_font_convert_smcode_to_base(c);
-    return font_normal_widths[(u8)c - '!'];
 }
 
 static const struct DjuiFont sDjuiFontNormal = {
@@ -231,16 +67,19 @@ static const struct DjuiFont sDjuiFontNormal = {
  // font 1 (custom title font) //
 ////////////////////////////////
 
-static void djui_font_title_render_char(char c) {
+static void djui_font_title_render_char(char* text) {
+    char c = *text;
     extern const u8* const font_title_chars[];
     // replace undisplayable characters
-    if (c < ' ' || (u8)c > ('~' + 1)) { c = '?'; }
     if (c == ' ') { return; }
+    c = djui_unicode_get_base_char(text);
     djui_gfx_render_texture(font_title_chars[c - '!'], 64, 64, 32);
 }
 
-static f32 djui_font_title_char_width(char c) {
+static f32 djui_font_title_char_width(char* text) {
+    char c = *text;
     if (c == ' ') { return 0.30f; }
+    c = djui_unicode_get_base_char(text);
     extern const f32 font_title_widths[];
     return font_title_widths[(u8)c - '!'];
 }
@@ -265,6 +104,7 @@ static u8 djui_font_hud_index(char c) {
     if (c == 'v' || c == 'V') { return 50; }
     if (c == 'x' || c == 'X') { return 50; }
     if (c == 'z' || c == 'Z') { return 50; }
+    if ((u8)c < ' ' || (u8)c > 127) { return 50; }
 
     switch (c) {
         case '$':  return 51;
@@ -283,13 +123,16 @@ static u8 djui_font_hud_index(char c) {
     return c;
 }
 
-static void djui_font_hud_render_char(char c) {
+static void djui_font_hud_render_char(char* text) {
+    char c = *text;
     if (c == ' ') { return; }
+    c = djui_unicode_get_base_char(text);
     u8 index = djui_font_hud_index(c);
     djui_gfx_render_texture(main_hud_lut[index], 16, 16, 16);
 }
 
-static f32 djui_font_hud_char_width(char c) {
+static f32 djui_font_hud_char_width(char* text) {
+    char c = *text;
     if (c == ' ') { return 0.5; }
     return 0.75f;
 }

src/pc/djui/djui_font.h

@@ -9,13 +9,8 @@ struct DjuiFont {
     u8 textureBitSize;
     bool rotatedUV;
     const Gfx* textBeginDisplayList;
-    void (*render_char)(char);
+    void (*render_char)(char*);
-    f32 (*char_width)(char);
+    f32 (*char_width)(char*);
 };
 
 extern const struct DjuiFont* gDjuiFonts[];
-
-u8 djui_font_convert_smcode_to_base(char c);
-void djui_font_convert_to_unicode(char* from, char* to, int length, int maxlength);
-void djui_font_convert_to_smcode(char* text);
-bool djui_font_valid_smcode(char c);

src/pc/djui/djui_hud_utils.c

@@ -14,6 +14,7 @@
 #include "gfx_dimensions.h"
 #include "config.h"
 #include "djui.h"
+#include "djui_unicode.h"
 #include "djui_hud_utils.h"
 #include "game/camera.h"
 
@@ -195,8 +196,8 @@ f32 djui_hud_measure_text(const char* message) {
     f32 width = 0;
     const char* c = message;
     while(*c != '\0') {
-        width += font->char_width(*c);
+        width += font->char_width((char*)c);
-        c++;
+        c = djui_unicode_next_char((char*)c);
     }
     return width * font->defaultFontScale;
 }
@@ -226,13 +227,13 @@ void djui_hud_print_text(const char* message, float x, float y, float scale) {
 
     // render the line
     f32 addX = 0;
-    size_t length = strlen(message);
+    char* c = (char*)message;
-    for (size_t i = 0; i < length; i++) {
+    while (*c != '\0') {
-        char c = message[i];
         f32 charWidth = font->char_width(c);
 
-        if (c == '\n' && c == ' ') {
+        if (*c == '\n' && *c == ' ') {
             addX += charWidth;
+            c++;
             continue;
         }
 
@@ -240,6 +241,8 @@ void djui_hud_print_text(const char* message, float x, float y, float scale) {
         font->render_char(c);
         create_dl_translation_matrix(DJUI_MTX_NOPUSH, charWidth + addX, 0, 0);
         addX = 0;
+
+        c = djui_unicode_next_char(c);
     }
 
     // pop

src/pc/djui/djui_inputbox.c

@@ -1,6 +1,7 @@
 #include <string.h>
 #include <stdio.h>
 #include "djui.h"
+#include "djui_unicode.h"
 #include "pc/gfx/gfx_window_manager_api.h"
 #include "pc/pc_main.h"
 #include "game/segment2.h"
@@ -50,7 +51,7 @@ void djui_inputbox_set_text(struct DjuiInputbox* inputbox, char* text) {
 
 void djui_inputbox_select_all(struct DjuiInputbox* inputbox) {
     inputbox->selection[1] = 0;
-    inputbox->selection[0] = strlen(inputbox->buffer);
+    inputbox->selection[0] = djui_unicode_len(inputbox->buffer);
 }
 
 void djui_inputbox_hook_enter_press(struct DjuiInputbox* inputbox, void (*on_enter_press)(struct DjuiInputbox*)) {
@@ -68,13 +69,16 @@ static u16 djui_inputbox_get_cursor_index(struct DjuiInputbox* inputbox) {
     f32 cX = (gCursorX - (comp->x + inputbox->viewX)) / font->defaultFontScale;
     f32 x = 0;
     u16 index = 0;
-    for (u16 i = 0; i < inputbox->bufferSize; i++) {
+    u16 i = 0;
-        char c = inputbox->buffer[i];
+    char* c = inputbox->buffer;
+    while (*c != '\0') {
         if (x < cX) {
             index = i;
         }
-        if (c == '\0') { break; }
+        if (*c == '\0') { break; }
         x += font->char_width(c);
+        c = djui_unicode_next_char(c);
+        i++;
     }
 
     return index;
@@ -90,7 +94,7 @@ static void djui_inputbox_on_cursor_down_begin(struct DjuiBase* base, UNUSED boo
     struct DjuiInputbox* inputbox = (struct DjuiInputbox*)base;
     u16 index = djui_inputbox_get_cursor_index(inputbox);
     u16 selLength = abs(inputbox->selection[0] - inputbox->selection[1]);
-    if (selLength != strlen(inputbox->buffer) || djui_interactable_is_input_focus(base)) {
+    if (selLength != djui_unicode_len(inputbox->buffer) || djui_interactable_is_input_focus(base)) {
         inputbox->selection[0] = index;
         inputbox->selection[1] = index;
         djui_interactable_hook_cursor_down(base, djui_inputbox_on_cursor_down_begin, djui_inputbox_on_cursor_down, NULL);
@@ -106,12 +110,14 @@ static u16 djui_inputbox_jump_word_left(char* msg, UNUSED u16 len, u16 i) {
 
     s32 lastI = i;
     bool seenNonSpace = false;
+    char* c = djui_unicode_at_index(msg, i);
     while (true) {
-        if (msg[i] == ' ' && seenNonSpace) { i = lastI; break; }
+        if (*c == ' ' && seenNonSpace) { i = lastI; break; }
         lastI = i;
         i--;
+        c = djui_unicode_at_index(msg, i);
         if (i <= 0) { i = 0; break; }
-        if (msg[i] != ' ') { seenNonSpace = true; }
+        if (*c != ' ') { seenNonSpace = true; }
     }
 
     return i;
@@ -121,11 +127,13 @@ static u16 djui_inputbox_jump_word_right(char *msg, u16 len, u16 i) {
     if (i >= len) { return len; }
 
     bool seenSpace = false;
+    char* c = djui_unicode_at_index(msg, i);
     while (true) {
         i++;
+        c = djui_unicode_at_index(msg, i);
         if (i >= len) { i = len; break; }
-        if (msg[i] != ' ' && seenSpace) { break; }
+        if (*c != ' ' && seenSpace) { break; }
-        if (msg[i] == ' ') { seenSpace = true; }
+        if (*c == ' ') { seenSpace = true; }
     };
 
     return i;
@@ -139,7 +147,8 @@ static void djui_inputbox_delete_selection(struct DjuiInputbox *inputbox) {
     if (sel[0] != sel[1]) {
         u16 s1 = fmin(sel[0], sel[1]);
         u16 s2 = fmax(sel[0], sel[1]);
-        memmove(&msg[s1], &msg[s2], (len + 1) - s2);
+        size_t s2len = djui_unicode_at_index(msg, s2) - msg;
+        memmove(djui_unicode_at_index(msg, s1), djui_unicode_at_index(msg, s2), (len + 1) - s2len);
         sel[0] = s1;
         sel[1] = s1;
     }
@@ -150,7 +159,7 @@ bool djui_inputbox_on_key_down(struct DjuiBase *base, int scancode) {
     struct DjuiInputbox *inputbox = (struct DjuiInputbox *) base;
     u16 *sel = inputbox->selection;
     char *msg = inputbox->buffer;
-    u16 len = strlen(msg);
+    u16 len = djui_unicode_len(msg);
     u16 s1 = fmin(sel[0], sel[1]);
     u16 s2 = fmax(sel[0], sel[1]);
 
@@ -236,7 +245,9 @@ bool djui_inputbox_on_key_down(struct DjuiBase *base, int scancode) {
     if (sHeldControl && (scancode == SCANCODE_C || scancode == SCANCODE_X)) {
         if (sel[0] != sel[1]) {
             char clipboardText[256] = { 0 };
-            djui_font_convert_to_unicode(&msg[s1], clipboardText, fmin(256, 1 + s2 - s1), 255);
+            char* cs1 = djui_unicode_at_index(msg, s1);
+            char* cs2 = djui_unicode_at_index(msg, s2);
+            snprintf(clipboardText, fmin(256, 1 + cs2 - cs1), "%s", cs1);
             wm_api->set_clipboard_text(clipboardText);
             if (scancode == SCANCODE_X) {
                 djui_inputbox_delete_selection(inputbox);
@@ -247,7 +258,7 @@ bool djui_inputbox_on_key_down(struct DjuiBase *base, int scancode) {
     }
 
     if (sHeldControl && scancode == SCANCODE_A) {
-        inputbox->selection[0] = len;
+        inputbox->selection[0] = djui_unicode_len(msg);
         inputbox->selection[1] = 0;
         sCursorBlink = 0;
         return true;
@@ -297,17 +308,15 @@ static void djui_inputbox_on_text_input(struct DjuiBase *base, char* text) {
     int msgLen = strlen(msg);
     int textLen = strlen(text);
 
-    djui_font_convert_to_smcode(text);
-
     // make sure we're not just printing garbage characters
     bool containsValidAscii = false;
     char* tinput = text;
     while (*tinput != '\0') {
-        if (djui_font_valid_smcode(*tinput)) {
+        if (djui_unicode_valid_char(tinput)) {
             containsValidAscii = true;
             break;
         }
-        tinput++;
+        tinput = djui_unicode_next_char(tinput);
     }
     if (!containsValidAscii) {
         return;
@@ -331,9 +340,9 @@ static void djui_inputbox_on_text_input(struct DjuiBase *base, char* text) {
     while (*t != '\0') {
         if (*t == '\n') { *t = ' '; }
         else if (*t == '\r') { *t = ' '; }
-        else if (djui_font_valid_smcode(*t)) { ; }
+        else if (djui_unicode_valid_char(t)) { ; }
-        else if (*t < '!' || *t > '~') { *t = '?'; }
+
-        t++;
+        t = djui_unicode_next_char(t);
     }
 
     // back up current message
@@ -341,18 +350,22 @@ static void djui_inputbox_on_text_input(struct DjuiBase *base, char* text) {
     memcpy(sMsg, msg, inputbox->bufferSize);
 
     // insert text
-    u16 sel = inputbox->selection[0];
+    size_t sel = djui_unicode_at_index(inputbox->buffer, inputbox->selection[0]) - inputbox->buffer;
+
     snprintf(&msg[sel], (inputbox->bufferSize - sel), "%s%s", text, &sMsg[sel]);
     free(sMsg);
+    djui_unicode_cleanup_end(msg);
 
     // adjust cursor
-    inputbox->selection[0] += strlen(text);
+    inputbox->selection[0] += djui_unicode_len(text);
+    s32 ulen = djui_unicode_len(msg);
+    if (inputbox->selection[0] > ulen) { inputbox->selection[0] = ulen; }
     inputbox->selection[1] = inputbox->selection[0];
     sCursorBlink = 0;
     djui_inputbox_on_change(inputbox);
 }
 
-static void djui_inputbox_render_char(struct DjuiInputbox* inputbox, char c, f32* drawX, f32* additionalShift) {
+static void djui_inputbox_render_char(struct DjuiInputbox* inputbox, char* c, f32* drawX, f32* additionalShift) {
     struct DjuiBaseRect*   comp = &inputbox->base.comp;
     const struct DjuiFont* font = gDjuiFonts[0];
     f32 dX = comp->x + *drawX;
@@ -363,7 +376,7 @@ static void djui_inputbox_render_char(struct DjuiInputbox* inputbox, char c, f32
     f32 charWidth = font->char_width(c);
     *drawX += charWidth * font->defaultFontScale;
 
-    if (c != ' ' && !djui_gfx_add_clipping_specific(&inputbox->base, font->rotatedUV, dX, dY, dW, dH)) {
+    if (*c != ' ' && !djui_gfx_add_clipping_specific(&inputbox->base, font->rotatedUV, dX, dY, dW, dH)) {
         if (*additionalShift > 0) {
             create_dl_translation_matrix(DJUI_MTX_NOPUSH, *additionalShift, 0, 0);
             *additionalShift = 0;
@@ -381,15 +394,16 @@ static void djui_inputbox_render_selection(struct DjuiInputbox* inputbox) {
     selection[0] = fmin(inputbox->selection[0], inputbox->selection[1]);
     selection[1] = fmax(inputbox->selection[0], inputbox->selection[1]);
 
-    char* msg = inputbox->buffer;
+    char* c = inputbox->buffer;
     f32 x = 0;
     f32 width = 0;
     for (u16 i = 0; i < selection[1]; i++) {
         if (i < selection[0]) {
-            x += font->char_width(msg[i]);
+            x += font->char_width(c);
         } else {
-            width += font->char_width(msg[i]);
+            width += font->char_width(c);
         }
+        c = djui_unicode_next_char(c);
     }
     
     sCursorBlink = (sCursorBlink + 1) % DJUI_INPUTBOX_MAX_BLINK;
@@ -441,9 +455,11 @@ static void djui_inputbox_keep_selection_in_view(struct DjuiInputbox* inputbox)
 
     // calculate where our cursor is
     f32 cursorX = inputbox->viewX;
-    char* msg = inputbox->buffer;
+    char* c = inputbox->buffer;
     for (u16 i = 0; i < inputbox->selection[0]; i++) {
-        cursorX += font->char_width(msg[i]) * font->defaultFontScale;
+        if (*c == '\0') { break; }
+        cursorX += font->char_width(c) * font->defaultFontScale;
+        c = djui_unicode_next_char(c);
     }
 
     // shift viewing window
@@ -491,12 +507,12 @@ static bool djui_inputbox_render(struct DjuiBase* base) {
     selection[1] = fmax(inputbox->selection[0], inputbox->selection[1]);
 
     // render text
-    char* msg = inputbox->buffer;
+    char* c = inputbox->buffer;
     f32 drawX = inputbox->viewX;
     f32 additionalShift = 0;
     bool wasInsideSelection = false;
     for (u16 i = 0; i < inputbox->bufferSize; i++) {
-        if (msg[i] == '\0') { break; }
+        if (*c == '\0') { break; }
 
         // deal with seleciton color
         if (selection[0] != selection[1]) {
@@ -510,7 +526,8 @@ static bool djui_inputbox_render(struct DjuiBase* base) {
         }
 
         // render character
-        djui_inputbox_render_char(inputbox, msg[i], &drawX, &additionalShift);
+        djui_inputbox_render_char(inputbox, c, &drawX, &additionalShift);
+        c = djui_unicode_next_char(c);
     }
 
     gSPPopMatrix(gDisplayListHead++, G_MTX_MODELVIEW);

src/pc/djui/djui_text.c

@@ -1,5 +1,6 @@
 #include <string.h>
 #include "djui.h"
+#include "djui_unicode.h"
 #include "game/segment2.h"
 
 static u8 sSavedR = 0;
@@ -62,7 +63,7 @@ static void djui_text_translate(f32 x, f32 y) {
     sTextRenderY += y;
 }
 
-static void djui_text_render_single_char(struct DjuiText* text, char c) {
+static void djui_text_render_single_char(struct DjuiText* text, char* c) {
     struct DjuiBaseRect* comp = &text->base.comp;
 
     f32 dX = comp->x + sTextRenderX * text->fontScale;
@@ -81,7 +82,7 @@ static void djui_text_render_single_char(struct DjuiText* text, char c) {
     sTextRenderLastY = sTextRenderY;
 }
 
-static void djui_text_render_char(struct DjuiText* text, char c) {
+static void djui_text_render_char(struct DjuiText* text, char* c) {
     if (text->dropShadow.a > 0) {
         // render drop shadow
         sTextRenderX += 1.0f / text->fontScale;
@@ -98,45 +99,44 @@ static void djui_text_render_char(struct DjuiText* text, char c) {
 static f32 djui_text_measure_word_width(struct DjuiText* text, char* message) {
     f32 width = 0;
     bool skipping = false;
-    while (*message != '\0') {
+    char* c = message;
-        char c = *message;
+    while (*c != '\0') {
-        if (c == ' ')  { return width; }
+        if (*c == ' ')  { return width; }
-        if (c == '\n') { return width; }
+        if (*c == '\n') { return width; }
-        if (c == '\0') { return width; }
+        if (*c == '\0') { return width; }
-        if (c == '\\') { skipping = !skipping; }
+        if (*c == '\\') { skipping = !skipping; }
         if (!skipping) {
             width += text->font->char_width(c);
         }
-        message++;
+        c = djui_unicode_next_char(c);
     }
     return width;
 }
 
-static void djui_text_read_line(struct DjuiText* text, u16* index, f32* lineWidth, f32 maxLineWidth, bool onLastLine, UNUSED bool* ellipses) {
+static void djui_text_read_line(struct DjuiText* text, char** message, f32* lineWidth, f32 maxLineWidth, bool onLastLine, UNUSED bool* ellipses) {
-    char* message = text->message;
     *lineWidth = 0;
-    char lastC = '\0';
+    char* lastC = "\0";
 
     /*f32 ellipsesWidth = gDialogCharWidths[0x3F] * 3.0f;
     u16 lastSafeEllipsesIndex = *index;
     u16 lastSafeEllipsesLineWidth = *lineWidth + ellipsesWidth;*/
 
     bool skipping = false;
-    while (message[*index] != '\0') {
+    char* c = *message;
-        char c = message[*index];
+    while (*c != '\0') {
         f32 charWidth = text->font->char_width(c);
 
         // check for special escape sequences
-        if (c == '\\') { skipping = !skipping; }
+        if (*c == '\\') { skipping = !skipping; }
-        if (skipping || c == '\\') {
+        if (skipping || *c == '\\') {
-            *index = *index + 1;
             lastC = c;
+            c = djui_unicode_next_char(c);
             continue;
         }
 
         // check for newline
-        if (c == '\n') {
+        if (*c == '\n') {
-            *index = *index + 1;
+            c = djui_unicode_next_char(c);
             break;
         }
 
@@ -146,9 +146,10 @@ static void djui_text_read_line(struct DjuiText* text, u16* index, f32* lineWidt
         }
 
         // check to see if this word exceeds size
-        if (!onLastLine && lastC == ' ' && c != ' ') {
+        if (!onLastLine && *lastC == ' ' && *c != ' ') {
-            f32 wordWidth = djui_text_measure_word_width(text, &message[*index]);
+            f32 wordWidth = djui_text_measure_word_width(text, c);
             if (*lineWidth + wordWidth >= maxLineWidth) {
+                *message = c;
                 return;
             }
         }
@@ -161,8 +162,8 @@ static void djui_text_read_line(struct DjuiText* text, u16* index, f32* lineWidt
             lastSafeEllipsesLineWidth = *lineWidth + ellipsesWidth;
         }*/
 
-        *index = *index + 1;
         lastC = c;
+        c = djui_unicode_next_char(c);
     }
 
     // check to see if we should replace the end of the last line with ellipses
@@ -171,20 +172,19 @@ static void djui_text_read_line(struct DjuiText* text, u16* index, f32* lineWidt
         *lineWidth = lastSafeEllipsesLineWidth;
         *ellipses = true;
     }*/
+    *message = c;
 }
 
 int djui_text_count_lines(struct DjuiText* text, u16 maxLines) {
     struct DjuiBaseRect* comp = &text->base.comp;
-    u16 startIndex = 0;
+    char* c = text->message;
-    u16 endIndex = 0;
     f32 maxLineWidth = comp->width / ((f32)text->fontScale);
     u16 lineCount = 0;
-    while (text->message[startIndex] != '\0') {
+    while (*c != '\0') {
         bool onLastLine = lineCount + 1 >= maxLines;
         f32 lineWidth;
         bool ellipses;
-        djui_text_read_line(text, &endIndex, &lineWidth, maxLineWidth, onLastLine, &ellipses);
+        djui_text_read_line(text, &c, &lineWidth, maxLineWidth, onLastLine, &ellipses);
-        startIndex = endIndex;
         lineCount++;
         if (onLastLine) { break; }
     }
@@ -193,43 +193,41 @@ int djui_text_count_lines(struct DjuiText* text, u16 maxLines) {
 
 f32 djui_text_find_width(struct DjuiText* text, u16 maxLines) {
     struct DjuiBaseRect* comp = &text->base.comp;
-    u16 startIndex = 0;
+    char* c = text->message;
-    u16 endIndex = 0;
     f32 maxLineWidth = comp->width / ((f32)text->fontScale);
     u16 lineCount = 0;
 
     f32 largestWidth = 0;
 
-    while (text->message[startIndex] != '\0') {
+    while (*c != '\0') {
         bool onLastLine = lineCount + 1 >= maxLines;
         f32 lineWidth;
         bool ellipses;
-        djui_text_read_line(text, &endIndex, &lineWidth, maxLineWidth, onLastLine, &ellipses);
+        djui_text_read_line(text, &c, &lineWidth, maxLineWidth, onLastLine, &ellipses);
         largestWidth = fmax(largestWidth, lineWidth);
-        startIndex = endIndex;
         lineCount++;
         if (onLastLine) { break; }
     }
     return largestWidth * text->fontScale;
 }
 
-static int djui_text_render_line_parse_escape(struct DjuiText* text, u16 startIndex, u16 endIndex) {
+static char* djui_text_render_line_parse_escape(char* c1, char* c2) {
-    bool parsingColor = text->message[startIndex + 1] == '#';
+    bool parsingColor = (c1[1] == '#');
-    u16 i = parsingColor ? (startIndex + 1) : startIndex;
+    char* c = parsingColor ? (c1 + 2) : (c1 + 1);
 
     u32 color = 0;
     u8 colorPieces = 0;
-    while (++i < endIndex) {
+    while (c < c2) {
-        char c = text->message[i];
+        if (*c == '\\') { break; }
-        if (c == '\\') { break; }
         if (parsingColor) {
             u8 colorPiece = 0;
-            if (c >= '0' && c <= '9') { colorPiece = c - '0'; }
+            if (*c >= '0' && *c <= '9') { colorPiece = *c - '0'; }
-            else if (c >= 'a' && c <= 'f') { colorPiece = 10 + c - 'a'; }
+            else if (*c >= 'a' && *c <= 'f') { colorPiece = 10 + *c - 'a'; }
-            else if (c >= 'A' && c <= 'F') { colorPiece = 10 + c - 'A'; }
+            else if (*c >= 'A' && *c <= 'F') { colorPiece = 10 + *c - 'A'; }
             color = (color << 4) | colorPiece;
             colorPieces++;
         }
+        c = djui_unicode_next_char(c);
     }
 
     if (parsingColor) {
@@ -246,10 +244,11 @@ static int djui_text_render_line_parse_escape(struct DjuiText* text, u16 startIn
         gDPSetEnvColor(gDisplayListHead++, sSavedR, sSavedG, sSavedB, sSavedA);
     }
 
-    return i;
+    c = djui_unicode_next_char(c);
+    return c;
 }
 
-static void djui_text_render_line(struct DjuiText* text, u16 startIndex, u16 endIndex, f32 lineWidth, bool ellipses) {
+static void djui_text_render_line(struct DjuiText* text, char* c1, char* c2, f32 lineWidth, bool ellipses) {
     struct DjuiBase* base     = &text->base;
     struct DjuiBaseRect* comp = &base->comp;
     f32 curWidth = 0;
@@ -268,30 +267,31 @@ static void djui_text_render_line(struct DjuiText* text, u16 startIndex, u16 end
     }
 
     // render the line
-    for (int i = startIndex; i < endIndex; i++) {
+    for (char* c = c1; c < c2;) {
-        char c = text->message[i];
+        if (*c == '\\') {
-        if (c == '\\') {
+            c = djui_text_render_line_parse_escape(c, c2);
-            i = djui_text_render_line_parse_escape(text, i, endIndex);
             continue;
         }
 
         f32 charWidth = text->font->char_width(c);
-        if (c != '\n' && c != ' ') {
+        if (*c != '\n' && *c != ' ') {
             djui_text_render_char(text, c);
         }
 
         djui_text_translate(charWidth, 0);
         curWidth += charWidth;
+        c = djui_unicode_next_char(c);
     }
 
     // render ellipses
     if (ellipses) {
+        char* c = ".";
         for (int i = 0; i < 3; i++) {
-            char c = '.';
             f32 charWidth = text->font->char_width(c);
             djui_text_render_char(text, c);
             djui_text_translate(charWidth, 0);
             curWidth += charWidth;
+            c = djui_unicode_next_char(c);
         }
     }
 
@@ -357,16 +357,16 @@ static bool djui_text_render(struct DjuiBase* base) {
     djui_text_translate(0, vOffset);
 
     // render lines
-    u16 startIndex = 0;
+    char* c1 = text->message;
-    u16 endIndex = 0;
+    char* c2 = c1;
     f32 lineWidth;
     u16 lineIndex = 0;
     bool ellipses = false;
-    while (text->message[startIndex] != '\0') {
+    while (*c1 != '\0') {
         bool onLastLine = lineIndex + 1 >= maxLines;
-        djui_text_read_line(text, &endIndex, &lineWidth, maxLineWidth, onLastLine, &ellipses);
+        djui_text_read_line(text, &c2, &lineWidth, maxLineWidth, onLastLine, &ellipses);
-        djui_text_render_line(text, startIndex, endIndex, lineWidth, ellipses);
+        djui_text_render_line(text, c1, c2, lineWidth, ellipses);
-        startIndex = endIndex;
+        c1 = c2;
         lineIndex++;
         if (onLastLine) { break; }
     }

src/pc/djui/djui_unicode.c

@@ -0,0 +1,259 @@
+#include <PR/ultratypes.h>
+#include <assert.h>
+#include <stdbool.h>
+#include <string.h>
+#include "data/dynos_cmap.cpp.h"
+
+#define SPRITE_INDEX_START_CHAR '!'
+
+struct SmCodeGlyph {
+    char unicode[3];
+    char base;
+    f32 width;
+    u32 spriteIndex;
+};
+
+struct SmCodeGlyph sSmCodeGlyphs[] = {
+    { "Á", 'A', 0, 0 },
+    { "Å", 'A', 0, 0 },
+    { "Â", 'A', 0, 0 },
+    { "À", 'A', 0, 0 },
+    { "Ã", 'A', 0, 0 },
+    { "Ä", 'A', 0, 0 },
+    { "Ç", 'C', 0, 0 },
+    { "É", 'E', 0, 0 },
+    { "Ê", 'E', 0, 0 },
+    { "È", 'E', 0, 0 },
+    { "Ë", 'E', 0, 0 },
+    { "Í", 'I', 0, 0 },
+    { "Î", 'I', 0, 0 },
+    { "Ì", 'I', 0, 0 },
+    { "Ï", 'I', 0, 0 },
+    { "Ñ", 'N', 0, 0 },
+    { "Ó", 'O', 0, 0 },
+    { "Ô", 'O', 0, 0 },
+    { "Ò", 'O', 0, 0 },
+    { "Õ", 'O', 0, 0 },
+    { "Ö", 'O', 0, 0 },
+    { "Ú", 'U', 0, 0 },
+    { "Û", 'U', 0, 0 },
+    { "Ù", 'U', 0, 0 },
+    { "Ü", 'U', 0, 0 },
+    { "Ý", 'Y', 0, 0 },
+    { "Ÿ", 'Y', 0, 0 },
+
+    { "á", 'a', 0, 0 },
+    { "å", 'a', 0, 0 },
+    { "â", 'a', 0, 0 },
+    { "à", 'a', 0, 0 },
+    { "ã", 'a', 0, 0 },
+    { "ä", 'a', 0, 0 },
+    { "ç", 'c', 0, 0 },
+    { "é", 'e', 0, 0 },
+    { "ê", 'e', 0, 0 },
+    { "è", 'e', 0, 0 },
+    { "ë", 'e', 0, 0 },
+    { "í", 'i', 0, 0 },
+    { "î", 'i', 0, 0 },
+    { "ì", 'i', 0, 0 },
+    { "ï", 'i', 0, 0 },
+    { "ñ", 'n', 0, 0 },
+    { "ó", 'o', 0, 0 },
+    { "ô", 'o', 0, 0 },
+    { "ò", 'o', 0, 0 },
+    { "õ", 'o', 0, 0 },
+    { "ö", 'o', 0, 0 },
+    { "ú", 'u', 0, 0 },
+    { "û", 'u', 0, 0 },
+    { "ù", 'u', 0, 0 },
+    { "ü", 'u', 0, 0 },
+    { "ý", 'y', 0, 0 },
+    { "ÿ", 'y', 0, 0 },
+
+    { "æ", 'a', 0.5000f, 0 },
+    { "Æ", 'a', 0.6000f, 0 },
+    { "œ", 'o', 0.5000f, 0 },
+    { "Œ", 'o', 0.5000f, 0 },
+    { "ð", 'd', 0, 0 },
+    { "Ð", 'D', 0.4375f, 0 },
+    { "ø", 'o', 0, 0 },
+    { "Ø", 'O', 0, 0 },
+    { "ß", 'S', 0, 0 },
+
+    { "¡", '!', 0, 0 },
+    { "¿", '?', 0, 0 },
+};
+
+static void* sCharMap = NULL;
+static void* sCharIter = NULL;
+
+static s32 count_bytes_for_char(char* text) {
+    s32 bytes = 0;
+    u8 mask = (1 << 7);
+    while (*text & mask) {
+        bytes++;
+        mask >>= 1;
+    }
+    return bytes ? bytes : 1;
+}
+
+static u64 convert_unicode_char_to_u64(char* text) {
+    s32 bytes = count_bytes_for_char(text);
+    u64 value = (u8)*text;
+
+    // HACK: we only support up to 4 bytes per character
+    if (bytes > 4) { return 0; }
+
+    bytes--;
+    while (bytes > 0) {
+        value <<= 8;
+        value |= (u8)*(++text);
+        bytes--;
+        text++;
+    }
+    return value;
+}
+
+void djui_unicode_init(void) {
+    sCharMap = hmap_create();
+    sCharIter = hmap_iter(sCharMap);
+
+    size_t glyphCount = sizeof(sSmCodeGlyphs) / sizeof(sSmCodeGlyphs[0]);
+    for (size_t i = 0; i < glyphCount; i++) {
+        struct SmCodeGlyph* glyph = &sSmCodeGlyphs[i];
+        glyph->spriteIndex = (128 + i) - SPRITE_INDEX_START_CHAR;
+
+        u64 key = convert_unicode_char_to_u64(glyph->unicode);
+        s32 bytes = count_bytes_for_char(glyph->unicode);
+        assert(bytes >= 2 && bytes <= 4);
+        assert(key > 127);
+        hmap_put(sCharMap, key, glyph);
+    }
+}
+
+u32 djui_unicode_get_sprite_index(char* text) {
+    // check for ASCI
+    if ((u8)*text < 128) {
+        // make sure it's in the valid range
+        if ((u8)*text < SPRITE_INDEX_START_CHAR) {
+            return (u8)'?' - SPRITE_INDEX_START_CHAR;
+        }
+
+        // output the ASCII index
+        return (u8)*text - SPRITE_INDEX_START_CHAR;
+    }
+
+    // retrieve the character
+    u64 key = convert_unicode_char_to_u64(text);
+
+    // retrieve the sprite glyph
+    struct SmCodeGlyph* glyph = hmap_get(sCharMap, key);
+    if (glyph) {
+        return glyph->spriteIndex;
+    }
+
+    // return default value
+    return (u8)'?' - SPRITE_INDEX_START_CHAR;
+}
+
+f32 djui_unicode_get_sprite_width(char* text, const f32 font_widths[]) {
+    // check for ASCI
+    if ((u8)*text < 128) {
+        // make sure it's in the valid range
+        if ((u8)*text < SPRITE_INDEX_START_CHAR) {
+            return font_widths[(u8)'?' - SPRITE_INDEX_START_CHAR];
+        }
+
+        // output the ASCII width
+        return font_widths[(u8)*text - SPRITE_INDEX_START_CHAR];
+    }
+
+    // retrieve the character
+    u64 key = convert_unicode_char_to_u64(text);
+
+    // retrieve the glyph
+    struct SmCodeGlyph* glyph = hmap_get(sCharMap, key);
+    if (glyph) {
+        if (glyph->width) {
+            // use the custom width
+            return glyph->width;
+        }
+        // use the base width
+        return font_widths[(u8)glyph->base - SPRITE_INDEX_START_CHAR];
+    }
+
+    // return default value
+    return font_widths[(u8)'?' - SPRITE_INDEX_START_CHAR];
+}
+
+char* djui_unicode_next_char(char* text) {
+    s32 bytes = count_bytes_for_char(text);
+    while (bytes-- > 0) {
+        if (*text == '\0') { return text; }
+        text++;
+    }
+    return text;
+}
+
+char* djui_unicode_at_index(char* text, s32 index) {
+    while (index-- > 0) {
+        text = djui_unicode_next_char(text);
+    }
+    return text;
+}
+
+size_t djui_unicode_len(char* text) {
+    size_t len = 0;
+    while (*text) {
+        text = djui_unicode_next_char(text);
+        len++;
+    }
+    return len;
+}
+
+bool djui_unicode_valid_char(char* text) {
+    if ((u8)*text < 128) {
+        return ((u8)*text >= ' ');
+    }
+    u64 key = convert_unicode_char_to_u64(text);
+    struct SmCodeGlyph* glyph = hmap_get(sCharMap, key);
+    return glyph != NULL;
+}
+
+void djui_unicode_cleanup_end(char* text) {
+    s32 slen = strlen(text);
+    s32 idx = strlen(text);
+    bool foundMulti = false;
+    if (idx < 2) { return; }
+    idx--;
+
+    // look for the start of a byte sequence
+    while (idx >= 0 && text[idx] & (1 << 7)) {
+        foundMulti = true;
+        if ((text[idx] & 192) == 192) {
+            break;
+        }
+        idx--;
+    }
+
+    if (!foundMulti) { return; }
+    if (idx < 0) { return; }
+
+    s32 bytes = count_bytes_for_char(&text[idx]);
+    if (bytes <= 1) {
+        text[idx] = '\0';
+        return;
+    }
+
+    if ((slen - idx) != bytes) {
+        text[idx] = '\0';
+    }
+}
+
+char djui_unicode_get_base_char(char* text) {
+    if ((u8)*text < ' ') { return '?'; }
+    if ((u8)*text < 128) { return *text; }
+    u64 key = convert_unicode_char_to_u64(text);
+    struct SmCodeGlyph* glyph = hmap_get(sCharMap, key);
+    return (glyph != NULL) ? glyph->base : '?';
+}

src/pc/djui/djui_unicode.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <PR/ultratypes.h>
+#include <stdbool.h>
+
+void djui_unicode_init(void);
+u32 djui_unicode_get_sprite_index(char* text);
+f32 djui_unicode_get_sprite_width(char* text, const f32 font_widths[]);
+char* djui_unicode_next_char(char* text);
+char* djui_unicode_at_index(char* text, s32 index);
+size_t djui_unicode_len(char* text);
+bool djui_unicode_valid_char(char* text);
+void djui_unicode_cleanup_end(char* text);
+char djui_unicode_get_base_char(char* text);

src/pc/network/sync_object.c

@@ -12,8 +12,8 @@
 #include "pc/utils/misc.h"
 #include "data/dynos_cmap.cpp.h"
 
-void* sSoMap = NULL;
+static void* sSoMap = NULL;
-void* sSoIter = NULL;
+static void* sSoIter = NULL;
 
 #define FORGET_TIMEOUT 10
 

src/pc/pc_main.c

@@ -50,6 +50,7 @@
 #include "pc/network/socket/domain_res.h"
 #include "pc/network/network_player.h"
 #include "pc/djui/djui.h"
+#include "pc/djui/djui_unicode.h"
 #include "pc/debuglog.h"
 #include "pc/utils/misc.h"
 
@@ -273,6 +274,7 @@ void main_func(void) {
     fs_init(sys_ropaths, gamedir, userpath);
 
     sync_objects_init_system();
+    djui_unicode_init();
     mods_init();
     configfile_load(configfile_name());
     dynos_pack_init();