1 files changed, 313 insertions, 138 deletions

diff --git a/firmware/common/unicode.c b/firmware/common/unicode.c
index 3ff1814c4b..954ad47e1d 100644
--- a/firmware/common/unicode.c
+++ b/firmware/common/unicode.c
@@ -28,161 +28,227 @@
 
 #include <stdio.h>
 #include "config.h"
+#include "system.h"
+#include "thread.h"
 #include "file.h"
 #include "debug.h"
 #include "rbunicode.h"
 #include "rbpaths.h"
+#include "pathfuncs.h"
+#include "core_alloc.h"
 
 #ifndef O_BINARY
 #define O_BINARY 0
 #endif
+#ifndef O_NOISODECODE
+#define O_NOISODECODE 0
+#endif
 
-static int default_codepage = 0;
-static int loaded_cp_table = 0;
-
-#ifdef HAVE_LCD_BITMAP
+#define getle16(p) (p[0] | (p[1] >> 8))
+#define getbe16(p) ((p[1] << 8) | p[0])
 
-#define MAX_CP_TABLE_SIZE  32768
-#define NUM_TABLES             5
+#if !defined (__PCTOOL__) && (CONFIG_PLATFORM & PLATFORM_NATIVE)
+/* Because file scanning uses the default CP table when matching entries,
+   on-demand loading is not feasible; we also must use the filesystem lock */
+#include "file_internal.h"
+#else /* APPLICATION */
+#ifdef __PCTOOL__
+#define yield()
+#endif
+#define open_noiso_internal open
+#endif /* !APPLICATION */
+
+#if 0 /* not needed just now (will probably end up a spinlock) */
+#include "mutex.h"
+static struct mutex cp_mutex SHAREDBSS_ATTR;
+#define cp_lock_init()   mutex_init(&cp_mutex)
+#define cp_lock_enter()  mutex_lock(&cp_mutex)
+#define cp_lock_leave()  mutex_unlock(&cp_mutex)
+#else
+#define cp_lock_init()   do {} while (0)
+#define cp_lock_enter()  asm volatile ("")
+#define cp_lock_leave()  asm volatile ("")
+#endif
 
-static const char * const filename[NUM_TABLES] =
+enum cp_tid
 {
-    CODEPAGE_DIR"/iso.cp",
-    CODEPAGE_DIR"/932.cp",  /* SJIS    */
-    CODEPAGE_DIR"/936.cp",  /* GB2312  */
-    CODEPAGE_DIR"/949.cp",  /* KSX1001 */
-    CODEPAGE_DIR"/950.cp"   /* BIG5    */
+    CP_TID_NONE = -1,
+    CP_TID_ISO,
+    CP_TID_932,
+    CP_TID_936,
+    CP_TID_949,
+    CP_TID_950,
 };
 
-static const char cp_2_table[NUM_CODEPAGES] =
+struct cp_info
 {
-    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 0
+    int8_t      tid;
+    const char  *filename;
+    const char  *name;
 };
 
-static const char * const name_codepages[NUM_CODEPAGES+1] =
-{
-    "ISO-8859-1",
-    "ISO-8859-7",
-    "ISO-8859-8",
-    "CP1251",
-    "ISO-8859-11",
-    "CP1256",
-    "ISO-8859-9",
-    "ISO-8859-2",
-    "CP1250",
-    "CP1252",
-    "SJIS",
-    "GB-2312",
-    "KSX-1001",
-    "BIG5",
-    "UTF-8",
-    "unknown"
-};
+#ifdef HAVE_LCD_BITMAP
 
-#if defined(APPLICATION) && defined(__linux__)
-static const char * const name_codepages_linux[NUM_CODEPAGES+1] =
-{
-    /* "ISO-8859-1" */ "iso8859-1",
-    /* "ISO-8859-7" */ "iso8859-7",
-    /* "ISO-8859-8" */ "iso8859-8",
-    /* "CP1251"     */ "cp1251",
-    /* "ISO-8859-11"*/ "iso8859-11",
-    /* "CP1256"     */ "cp1256",
-    /* "ISO-8859-9" */ "iso8859-9",
-    /* "ISO-8859-2" */ "iso8859-2",
-    /* "CP1250"     */ "cp1250",
-    /* "CP1252"     */ "iso8859-15", /* closest, linux doesnt have a codepage named cp1252 */
-    /* "SJIS"       */ "cp932",
-    /* "GB-2312"    */ "cp936",
-    /* "KSX-1001"   */ "cp949",
-    /* "BIG5"       */ "cp950",
-    /* "UTF-8"      */ "utf8",
-    /* "unknown"    */ "cp437"
-};
+#define MAX_CP_TABLE_SIZE  32768
 
-const char *get_current_codepage_name_linux(void)
+#define CPF_ISO "iso.cp"
+#define CPF_932 "932.cp"  /* SJIS    */
+#define CPF_936 "936.cp"  /* GB2312  */
+#define CPF_949 "949.cp"  /* KSX1001 */
+#define CPF_950 "950.cp"  /* BIG5    */
+
+static const struct cp_info cp_info[NUM_CODEPAGES+1] =
 {
-    if (default_codepage < 0 || default_codepage >= NUM_CODEPAGES)
-        return name_codepages_linux[NUM_CODEPAGES];
-    return name_codepages_linux[default_codepage];
-}
-#endif
+    [0 ... NUM_CODEPAGES] = { CP_TID_NONE, NULL   , "unknown"     },
+    [ISO_8859_1]          = { CP_TID_NONE, NULL   , "ISO-8859-1"  },
+    [ISO_8859_7]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-7"  },
+    [ISO_8859_8]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-8"  },
+    [WIN_1251]            = { CP_TID_ISO , CPF_ISO, "CP1251"      },
+    [ISO_8859_11]         = { CP_TID_ISO , CPF_ISO, "ISO-8859-11" },
+    [WIN_1256]            = { CP_TID_ISO , CPF_ISO, "CP1256"      },
+    [ISO_8859_9]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-9"  },
+    [ISO_8859_2]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-2"  },
+    [WIN_1250]            = { CP_TID_ISO , CPF_ISO, "CP1250"      },
+    [WIN_1252]            = { CP_TID_ISO , CPF_ISO, "CP1252"      },
+    [SJIS]                = { CP_TID_932 , CPF_932, "SJIS"        },
+    [GB_2312]             = { CP_TID_936 , CPF_936, "GB-2312"     },
+    [KSX_1001]            = { CP_TID_949 , CPF_949, "KSX-1001"    },
+    [BIG_5]               = { CP_TID_950 , CPF_950, "BIG5"        },
+    [UTF_8]               = { CP_TID_NONE, NULL   , "UTF-8"       },
+};
 
 #else /* !HAVE_LCD_BITMAP, reduced support */
 
 #define MAX_CP_TABLE_SIZE  768
-#define NUM_TABLES           1
 
-static const char * const filename[NUM_TABLES] = {
-    CODEPAGE_DIR"/isomini.cp"
-};
+#define CPF_ISOMINI "isomini.cp"
 
-static const char cp_2_table[NUM_CODEPAGES] =
+static const struct cp_info cp_info[NUM_CODEPAGES+1] =
 {
-    0, 1, 1, 1, 1, 1, 1, 0
+    [0 ... NUM_CODEPAGES] = { CP_TID_NONE, NULL       , "unknown"    },
+    [ISO_8859_1]          = { CP_TID_NONE, NULL       , "ISO-8859-1" },
+    [ISO_8859_7]          = { CP_TID_ISO , CPF_ISOMINI, "ISO-8859-7" },
+    [WIN_1251]            = { CP_TID_ISO , CPF_ISOMINI, "CP1251"     },
+    [ISO_8859_9]          = { CP_TID_ISO , CPF_ISOMINI, "ISO-8859-9" },
+    [ISO_8859_2]          = { CP_TID_ISO , CPF_ISOMINI, "ISO-8859-2" },
+    [WIN_1250]            = { CP_TID_ISO , CPF_ISOMINI, "CP1250"     },
+    [WIN_1252]            = { CP_TID_ISO , CPF_ISOMINI, "CP1252"     },
+    [UTF_8]               = { CP_TID_ISO , NULL       , "UTF-8"      },
 };
 
-static const char * const name_codepages[NUM_CODEPAGES+1] =
+#endif /* HAVE_LCD_BITMAP */
+
+static int default_cp = INIT_CODEPAGE;
+static int default_cp_tid = CP_TID_NONE;
+static int default_cp_handle = 0;
+static int volatile default_cp_table_ref = 0;
+
+static int loaded_cp_tid = CP_TID_NONE;
+static int volatile cp_table_ref = 0;
+#define CP_LOADING BIT_N(sizeof(int)*8-1) /* guard against multi loaders */
+
+/* non-default codepage table buffer (cannot be bufalloced! playback itself
+   may be making the load request) */
+static unsigned short codepage_table[MAX_CP_TABLE_SIZE+1];
+
+#if defined(APPLICATION) && defined(__linux__)
+static const char * const name_codepages_linux[NUM_CODEPAGES+1] =
 {
-    "ISO-8859-1",
-    "ISO-8859-7",
-    "CP1251",
-    "ISO-8859-9",
-    "ISO-8859-2",
-    "CP1250",
-    "CP1252",
-    "UTF-8",
-    "unknown"
+    [0 ... NUM_CODEPAGES] = "unknown",
+    [ISO_8859_1]          = "iso8859-1",
+    [ISO_8859_7]          = "iso8859-7",
+    [ISO_8859_8]          = "iso8859-8",
+    [WIN_1251]            = "cp1251",
+    [ISO_8859_11]         = "iso8859-11",
+    [WIN_1256]            = "cp1256",
+    [ISO_8859_9]          = "iso8859-9",
+    [ISO_8859_2]          = "iso8859-2",
+    [WIN_1250]            = "cp1250",
+    /* iso8859-15 is closest, linux doesnt have a codepage named cp1252 */
+    [WIN_1252]            = "iso8859-15",
+    [SJIS]                = "cp932",
+    [GB_2312]             = "cp936",
+    [KSX_1001]            = "cp949",
+    [BIG_5]               = "cp950",
+    [UTF_8]               = "utf8",
 };
 
-#endif
-
-static unsigned short codepage_table[MAX_CP_TABLE_SIZE];
+const char *get_current_codepage_name_linux(void)
+{
+    int cp = default_cp;
+    if (cp < 0 || cp>= NUM_CODEPAGES)
+        cp = NUM_CODEPAGES;
+    return name_codepages_linux[cp];
+}
+#endif /* defined(APPLICATION) && defined(__linux__) */
 
 static const unsigned char utf8comp[6] =
 {
     0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
 };
 
-/* Load codepage file into memory */
-static int load_cp_table(int cp)
+static inline void cptable_tohw16(uint16_t *buf, unsigned int count)
 {
-    int i = 0;
-    int table = cp_2_table[cp];
-    int file, tablesize;
-    unsigned char tmp[2];
+#ifdef ROCKBOX_BIG_ENDIAN
+    for (unsigned int i = 0; i < count; i++)
+        buf[i] = letoh16(buf[i]);
+#endif
+    (void)buf; (void)count;
+}
 
-    if (table == 0 || table == loaded_cp_table)
-        return 1;
+static int move_callback(int handle, void *current, void *new)
+{
+    /* we don't keep a pointer but we have to stop it if this applies to a
+       buffer not yet swapped-in since it will likely be in use in an I/O
+       call */
+    return (handle != default_cp_handle || default_cp_table_ref != 0) ?
+                BUFLIB_CB_CANNOT_MOVE : BUFLIB_CB_OK;
+    (void)current; (void)new;
+}
 
-    file = open(filename[table-1], O_RDONLY|O_BINARY);
+static int alloc_and_load_cp_table(int cp, void *buf)
+{
+    static struct buflib_callbacks ops =
+        { .move_callback = move_callback };
 
-    if (file < 0) {
-        DEBUGF("Can't open codepage file: %s.cp\n", filename[table-1]);
+    /* alloc and read only if there is an associated file */
+    const char *filename = cp_info[cp].filename;
+    if (!filename)
         return 0;
+
+    char path[MAX_PATH];
+    if (path_append(path, CODEPAGE_DIR, filename, sizeof (path))
+        >= sizeof (path)) {
+        return -1;
     }
 
-    tablesize = filesize(file) / 2;
+    /* must be opened without a chance of reentering from FS code */
+    int fd = open_noiso_internal(path, O_RDONLY);
+    if (fd < 0)
+        return -1;
 
-    if (tablesize > MAX_CP_TABLE_SIZE) {
-        DEBUGF("Invalid codepage file: %s.cp\n", filename[table-1]);
-        close(file);
-        return 0;
-    }
+    off_t size = filesize(fd);
 
-    while (i < tablesize) {
-        if (!read(file, tmp, 2)) {
-            DEBUGF("Can't read from codepage file: %s.cp\n", 
-                    filename[table-1]);
-            loaded_cp_table = 0;
-            return 0;
+    if (size > 0 && size <= MAX_CP_TABLE_SIZE*2 &&
+        !(size % (off_t)sizeof (uint16_t))) {
+
+        /* if the buffer is provided, use that but don't alloc */
+        int handle = buf ? 0 : core_alloc_ex(filename, size, &ops);
+        if (handle > 0)
+            buf = core_get_data(handle);
+
+        if (buf && read(fd, buf, size) == size) {
+            close(fd);
+            cptable_tohw16(buf, size / sizeof (uint16_t));
+            return handle;
         }
-        codepage_table[i++] = (tmp[1] << 8) | tmp[0];
+
+        if (handle > 0)
+            core_free(handle);
     }
 
-    loaded_cp_table = table;
-    close(file);
-    return 1;
+    close(fd);
+    return -1;
 }
 
 /* Encode a UCS value as UTF-8 and return a pointer after this UTF-8 char. */
@@ -205,47 +271,96 @@ unsigned char* utf8encode(unsigned long ucs, unsigned char *utf8)
 unsigned char* iso_decode(const unsigned char *iso, unsigned char *utf8,
                           int cp, int count)
 {
-    unsigned short ucs, tmp;
+    uint16_t *table = NULL;
+
+    cp_lock_enter();
+
+    if (cp < 0 || cp >= NUM_CODEPAGES)
+        cp = default_cp;
 
-    if (cp == -1) /* use default codepage */
-        cp = default_codepage;
+    int tid = cp_info[cp].tid;
 
-    if (!load_cp_table(cp)) cp = 0;
+    while (1) {
+        if (tid == default_cp_tid) {
+            /* use default table */
+            if (default_cp_handle > 0) {
+                table = core_get_data(default_cp_handle);
+                default_cp_table_ref++;
+            }
+
+            break;
+        }
+
+        bool load = false;
+
+        if (tid == loaded_cp_tid) {
+            /* use loaded table */
+            if (!(cp_table_ref & CP_LOADING)) {
+                if (tid != CP_TID_NONE) {
+                    table = codepage_table;
+                    cp_table_ref++;
+                }
+
+                break;
+            }
+        } else if (cp_table_ref == 0) {
+            load = true;
+            cp_table_ref |= CP_LOADING;
+        }
+
+        /* alloc and load must be done outside the lock */
+        cp_lock_leave();
+
+        if (!load) {
+            yield();
+        } else if (alloc_and_load_cp_table(cp, codepage_table) < 0) {
+            cp = INIT_CODEPAGE; /* table may be clobbered now */
+            tid = cp_info[cp].tid;
+        }
+
+        cp_lock_enter();
+
+        if (load) {
+            loaded_cp_tid = tid;
+            cp_table_ref &= ~CP_LOADING;
+        }
+    }
+
+    cp_lock_leave();
 
     while (count--) {
+        unsigned short ucs, tmp;
+
         if (*iso < 128 || cp == UTF_8) /* Already UTF-8 */
             *utf8++ = *iso++;
 
         else {
-
-            /* cp tells us which codepage to convert from */
-            switch (cp) {
-                case ISO_8859_7:  /* Greek */
-                case WIN_1252:    /* Western European */
-                case WIN_1251:    /* Cyrillic */
-                case ISO_8859_9:  /* Turkish */
-                case ISO_8859_2:  /* Latin Extended */
-                case WIN_1250:    /* Central European */
-#ifdef HAVE_LCD_BITMAP
-                case ISO_8859_8:  /* Hebrew */
-                case ISO_8859_11: /* Thai */
-                case WIN_1256:    /* Arabic */
-#endif
+            /* tid tells us which table to use and how */
+            switch (tid) {
+                case CP_TID_ISO: /* Greek */
+                                 /* Hebrew */
+                                 /* Cyrillic */
+                                 /* Thai */
+                                 /* Arabic */
+                                 /* Turkish */
+                                 /* Latin Extended */
+                                 /* Central European */
+                                 /* Western European */
                     tmp = ((cp-1)*128) + (*iso++ - 128);
-                    ucs = codepage_table[tmp];
+                    ucs = table[tmp];
                     break;
 
 #ifdef HAVE_LCD_BITMAP
-                case SJIS: /* Japanese */
+                case CP_TID_932: /* Japanese */
                     if (*iso > 0xA0 && *iso < 0xE0) {
                         tmp = *iso++ | (0xA100 - 0x8000);
-                        ucs = codepage_table[tmp];
+                        ucs = table[tmp];
                         break;
                     }
 
-                case GB_2312:  /* Simplified Chinese */
-                case KSX_1001: /* Korean */
-                case BIG_5:    /* Traditional Chinese */
+                case CP_TID_936: /* Simplified Chinese */
+                case CP_TID_949: /* Korean */
+                case CP_TID_950: /* Traditional Chinese */
                     if (count < 1 || !iso[1]) {
                         ucs = *iso++;
                         break;
@@ -256,7 +371,7 @@ unsigned char* iso_decode(const unsigned char *iso, unsigned char *utf8,
                     tmp = *iso++ << 8;
                     tmp |= *iso++;
                     tmp -= 0x8000;
-                    ucs = codepage_table[tmp];
+                    ucs = table[tmp];
                     count--;
                     break;
 #endif /* HAVE_LCD_BITMAP */
@@ -271,6 +386,17 @@ unsigned char* iso_decode(const unsigned char *iso, unsigned char *utf8,
             utf8 = utf8encode(ucs, utf8);
         }
     }
+
+    if (table) {
+        cp_lock_enter();
+        if (table == codepage_table) {
+            cp_table_ref--;
+        } else {
+            default_cp_table_ref--;
+        }
+        cp_lock_leave();
+    }
+
     return utf8;
 }
 
@@ -288,7 +414,7 @@ unsigned char* utf16LEdecode(const unsigned char *utf16, unsigned char *utf8,
             utf16 += 4;
             count -= 2;
         } else {
-            ucs = (utf16[0] | (utf16[1] << 8));
+            ucs = getle16(utf16);
             utf16 += 2;
             count -= 1;
         }
@@ -310,7 +436,7 @@ unsigned char* utf16BEdecode(const unsigned char *utf16, unsigned char *utf8,
             utf16 += 4;
             count -= 2;
         } else {
-            ucs = (utf16[0] << 8) | utf16[1];
+            ucs = getbe16(utf16);
             utf16 += 2;
             count -= 1;
         }
@@ -400,8 +526,50 @@ const unsigned char* utf8decode(const unsigned char *utf8, unsigned short *ucs)
 
 void set_codepage(int cp)
 {
-    default_codepage = cp;
-    return;
+    if (cp < 0 || cp >= NUM_CODEPAGES)
+        cp = NUM_CODEPAGES;
+
+    /* load first then swap if load is successful, else just leave it; if
+       handle is 0 then we just free the current one; this won't happen often
+       thus we don't worry about reusing it and consequently avoid possible
+       clobbering of the existing one */
+
+    int handle = -1;
+    int tid = cp_info[cp].tid;
+
+    while (1) {
+        cp_lock_enter();
+
+        if (default_cp_tid == tid)
+            break;
+
+        if (handle >= 0 && default_cp_table_ref == 0) {
+            int hold = default_cp_handle;
+            default_cp_handle = handle;
+            handle = hold;
+            default_cp_tid = tid;
+            break;
+        }
+
+        /* alloc and load must be done outside the lock */
+        cp_lock_leave();
+
+        if (handle < 0 && (handle = alloc_and_load_cp_table(cp, NULL)) < 0)
+            return; /* OOM; change nothing */
+
+        yield();
+    }
+
+    default_cp = cp;
+    cp_lock_leave();
+
+    if (handle > 0)
+        core_free(handle);
+}
+
+int get_codepage(void)
+{
+    return default_cp;
 }
 
 /* seek to a given char in a utf8 string and
@@ -418,9 +586,16 @@ int utf8seek(const unsigned char* utf8, int offset)
     return pos;
 }
 
-const char* get_codepage_name(int cp)
+const char * get_codepage_name(int cp)
 {
-    if (cp < 0 || cp>= NUM_CODEPAGES)
-        return name_codepages[NUM_CODEPAGES];
-    return name_codepages[cp];
+    if (cp < 0 || cp >= NUM_CODEPAGES)
+        cp = NUM_CODEPAGES;
+    return cp_info[cp].name;
 }
+
+#if 0 /* not needed just now */
+void unicode_init(void)
+{
+    cp_lock_init();
+}
+#endif