12 files changed, 6732 insertions, 2983 deletions

diff --git a/firmware/common/dir.c b/firmware/common/dir.c
new file mode 100644
index 0000000000..da798c71d5
--- /dev/null
+++ b/firmware/common/dir.c
@@ -0,0 +1,413 @@
+/***************************************************************************
+ *             __________               __   ___.
+ *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
+ *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
+ *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
+ *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
+ *                     \/            \/     \/    \/            \/
+ * $Id$
+ *
+ * Copyright (C) 2002 by Björn Stenberg
+ * Copyright (C) 2014 by Michael Sevakis
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ****************************************************************************/
+#define DIRFUNCTIONS_DEFINED
+#include "config.h"
+#include <errno.h>
+#include <string.h>
+#include "debug.h"
+#include "dir.h"
+#include "pathfuncs.h"
+#include "fileobj_mgr.h"
+#include "dircache_redirect.h"
+
+/* structure used for open directory streams */
+static struct dirstr_desc
+{
+    struct filestr_base stream; /* basic stream info (first!) */
+    struct dirscan_info scan;   /* directory scan cursor */
+    struct dirent       entry;  /* current parsed entry information */
+#ifdef HAVE_MULTIVOLUME
+    int                 volumecounter; /* counter for root volume entries */
+#endif
+} open_streams[MAX_OPEN_DIRS];
+
+/* check and return a struct dirstr_desc* from a DIR* */
+static struct dirstr_desc * get_dirstr(DIR *dirp)
+{
+    struct dirstr_desc *dir = (struct dirstr_desc *)dirp;
+
+    if (!PTR_IN_ARRAY(open_streams, dir, MAX_OPEN_DIRS))
+        dir = NULL;
+    else if (dir->stream.flags & FDO_BUSY)
+        return dir;
+
+    int errnum;
+
+    if (!dir)
+    {
+        errnum = EFAULT;
+    }
+    else if (dir->stream.flags == FV_NONEXIST)
+    {
+        DEBUGF("dir #%d: nonexistant device\n", (int)(dir - open_streams));
+        errnum = ENXIO;
+    }
+    else
+    {
+        DEBUGF("dir #%d: dir not open\n", (int)(dir - open_streams));
+        errnum = EBADF;
+    }
+
+    errno = errnum;
+    return NULL;
+}
+
+#define GET_DIRSTR(type, dirp) \
+    ({                                               \
+        file_internal_lock_##type();                 \
+        struct dirstr_desc *_dir = get_dirstr(dirp); \
+        if (_dir)                                    \
+            FILESTR_LOCK(type, &_dir->stream);       \
+        else                                         \
+            file_internal_unlock_##type();           \
+        _dir;                                        \
+    })
+
+/* release the lock on the dirstr_desc* */
+#define RELEASE_DIRSTR(type, dir) \
+    ({                                        \
+        FILESTR_UNLOCK(type, &(dir)->stream); \
+        file_internal_unlock_##type();        \
+    })
+
+
+/* find a free dir stream descriptor */
+static struct dirstr_desc * alloc_dirstr(void)
+{
+    for (unsigned int dd = 0; dd < MAX_OPEN_DIRS; dd++)
+    {
+        struct dirstr_desc *dir = &open_streams[dd];
+        if (!dir->stream.flags)
+            return dir;
+    }
+
+    DEBUGF("Too many dirs open\n");
+    return NULL;
+}
+
+#ifdef HAVE_MULTIVOLUME
+static int readdir_volume_inner(struct dirstr_desc *dir, struct dirent *entry)
+{
+    /* Volumes (secondary file systems) get inserted into the system root
+     * directory. If the path specified volume 0, enumeration will not
+     * include other volumes, but just its own files and directories.
+     *
+     * Fake special directories, which don't really exist, that will get
+     * redirected upon opendir()
+     */
+    while (++dir->volumecounter < NUM_VOLUMES)
+    {
+        /* on the system root */
+        if (!fat_ismounted(dir->volumecounter))
+            continue;
+
+        get_volume_name(dir->volumecounter, entry->d_name);
+        dir->entry.info.attr    = ATTR_MOUNT_POINT;
+        dir->entry.info.size    = 0;
+        dir->entry.info.wrtdate = 0;
+        dir->entry.info.wrttime = 0;
+        return 1;
+    }
+
+    /* do normal directory entry fetching */
+    return 0;
+}
+#endif /* HAVE_MULTIVOLUME */
+
+static inline int readdir_volume(struct dirstr_desc *dir,
+                                 struct dirent *entry)
+{
+#ifdef HAVE_MULTIVOLUME
+    /* fetch virtual volume entries? */
+    if (dir->volumecounter < NUM_VOLUMES)
+        return readdir_volume_inner(dir, entry);
+#endif /* HAVE_MULTIVOLUME */
+
+    /* do normal directory entry fetching */
+    return 0;
+    (void)dir; (void)entry;
+}
+
+
+/** POSIX interface **/
+
+/* open a directory */
+DIR * opendir(const char *dirname)
+{
+    DEBUGF("opendir(dirname=\"%s\"\n", dirname);
+
+    DIR *dirp = NULL;
+
+    file_internal_lock_WRITER();
+
+    int rc;
+
+    struct dirstr_desc * const dir = alloc_dirstr();
+    if (!dir)
+        FILE_ERROR(EMFILE, RC);
+
+    rc = open_stream_internal(dirname, FF_DIR, &dir->stream, NULL);
+    if (rc < 0)
+    {
+        DEBUGF("Open failed: %d\n", rc);
+        FILE_ERROR(ERRNO, RC);
+    }
+
+#ifdef HAVE_MULTIVOLUME
+    /* volume counter is relevant only to the system root */
+    dir->volumecounter = rc > 1 ? 0 : INT_MAX;
+#endif /* HAVE_MULTIVOLUME */
+
+    fat_rewind(&dir->stream.fatstr);
+    rewinddir_dirent(&dir->scan);
+
+    dirp = (DIR *)dir;
+file_error:
+    file_internal_unlock_WRITER();
+    return dirp;
+}
+
+/* close a directory stream */
+int closedir(DIR *dirp)
+{
+    int rc;
+
+    file_internal_lock_WRITER();
+
+    /* needs to work even if marked "nonexistant" */
+    struct dirstr_desc * const dir = (struct dirstr_desc *)dirp;
+    if (!PTR_IN_ARRAY(open_streams, dir, MAX_OPEN_DIRS))
+        FILE_ERROR(EFAULT, -1);
+
+    if (!dir->stream.flags)
+    {
+        DEBUGF("dir #%d: dir not open\n", (int)(dir - open_streams));
+        FILE_ERROR(EBADF, -2);
+    }
+
+    rc = close_stream_internal(&dir->stream);
+    if (rc < 0)
+        FILE_ERROR(ERRNO, rc * 10 - 3);
+
+file_error:
+    file_internal_unlock_WRITER();
+    return rc;
+}
+
+/* read a directory */
+struct dirent * readdir(DIR *dirp)
+{
+    struct dirstr_desc * const dir = GET_DIRSTR(READER, dirp);
+    if (!dir)
+        FILE_ERROR_RETURN(ERRNO, NULL);
+
+    struct dirent *res = NULL;
+
+    int rc = readdir_volume(dir, &dir->entry);
+    if (rc == 0)
+    {
+        rc = readdir_dirent(&dir->stream, &dir->scan, &dir->entry);
+        if (rc < 0)
+            FILE_ERROR(EIO, RC);
+    }
+
+    if (rc > 0)
+        res = &dir->entry;
+
+file_error:
+    RELEASE_DIRSTR(READER, dir);
+
+    if (rc > 1)
+        iso_decode_d_name(res->d_name);
+
+    return res;
+}
+
+#if 0 /* not included now but probably should be */
+/* read a directory (reentrant) */
+int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result)
+{
+    if (!result)
+        FILE_ERROR_RETURN(EFAULT, -2);
+
+    *result = NULL;
+
+    if (!entry)
+        FILE_ERROR_RETURN(EFAULT, -3);
+
+    struct dirstr_desc * const dir = GET_DIRSTR(READER, dirp);
+    if (!dir)
+        FILE_ERROR_RETURN(ERRNO, -1);
+
+    int rc = readdir_volume(dir, entry);
+    if (rc == 0)
+    {
+        rc = readdir_dirent(&dir->stream, &dir->scan, entry);
+        if (rc < 0)
+            FILE_ERROR(EIO, rc * 10 - 4);
+    }
+
+file_error:
+    RELEASE_DIRSTR(READER, dir);
+
+    if (rc > 0)
+    {
+        if (rc > 1)
+            iso_decode_d_name(entry->d_name);
+
+        *result = entry;
+        rc = 0;
+    }
+
+    return rc;
+}
+
+/* reset the position of a directory stream to the beginning of a directory */
+void rewinddir(DIR *dirp)
+{
+    struct dirstr_desc * const dir = GET_DIRSTR(READER, dirp);
+    if (!dir)
+        FILE_ERROR_RETURN(ERRNO);
+
+    rewinddir_dirent(&dir->scan);
+
+#ifdef HAVE_MULTIVOLUME
+    if (dir->volumecounter != INT_MAX)
+        dir->volumecounter = 0;
+#endif /* HAVE_MULTIVOLUME */
+
+    RELEASE_DIRSTR(READER, dir);
+}
+
+#endif /* 0 */
+
+/* make a directory */
+int mkdir(const char *path)
+{
+    DEBUGF("mkdir(path=\"%s\")\n", path);
+
+    int rc;
+
+    file_internal_lock_WRITER();
+
+    struct filestr_base stream;
+    struct path_component_info compinfo;
+    rc = open_stream_internal(path, FF_DIR, &stream, &compinfo);
+    if (rc < 0)
+    {
+        DEBUGF("Can't open parent dir or path is not a directory\n");
+        FILE_ERROR(ERRNO, rc * 10 - 1);
+    }
+    else if (rc > 0)
+    {
+        DEBUGF("File exists\n");
+        FILE_ERROR(EEXIST, -2);
+    }
+
+    rc = create_stream_internal(&compinfo.parentinfo, compinfo.name,
+                                compinfo.length, ATTR_NEW_DIRECTORY,
+                                FO_DIRECTORY, &stream);
+    if (rc < 0)
+        FILE_ERROR(ERRNO, rc * 10 - 3);
+
+    rc = 0;
+file_error:
+    close_stream_internal(&stream);
+    file_internal_unlock_WRITER();
+    return rc;
+}
+
+/* remove a directory */
+int rmdir(const char *name)
+{
+    DEBUGF("rmdir(name=\"%s\")\n", name);
+
+    if (name)
+    {
+        /* path may not end with "." */
+        const char *basename;
+        size_t len = path_basename(name, &basename);
+        if (basename[0] == '.' && len == 1)
+        {
+            DEBUGF("Invalid path; last component is \".\"\n");
+            FILE_ERROR_RETURN(EINVAL, -9);
+        }
+    }
+
+    file_internal_lock_WRITER();
+    int rc = remove_stream_internal(name, NULL, FF_DIR);
+    file_internal_unlock_WRITER();
+    return rc;
+}
+
+
+/** Extended interface **/
+
+/* return if two directory streams refer to the same directory */
+int samedir(DIR *dirp1, DIR *dirp2)
+{
+    struct dirstr_desc * const dir1 = GET_DIRSTR(WRITER, dirp1);
+    if (!dir1)
+        FILE_ERROR_RETURN(ERRNO, -1);
+
+    int rc = -2;
+
+    struct dirstr_desc * const dir2 = get_dirstr(dirp2);
+    if (dir2)
+        rc = dir1->stream.bindp == dir2->stream.bindp ? 1 : 0;
+
+    RELEASE_DIRSTR(WRITER, dir1);
+    return rc;
+}
+
+/* test directory existence (returns 'false' if a file) */
+bool dir_exists(const char *dirname)
+{
+    file_internal_lock_WRITER();
+    bool rc = test_stream_exists_internal(dirname, FF_DIR) > 0;
+    file_internal_unlock_WRITER();
+    return rc;
+}
+
+/* get the portable info from the native entry */
+struct dirinfo dir_get_info(DIR *dirp, struct dirent *entry)
+{
+    int rc;
+    if (!dirp || !entry)
+        FILE_ERROR(EFAULT, RC);
+
+    if (entry->d_name[0] == '\0')
+        FILE_ERROR(ENOENT, RC);
+
+    if ((file_size_t)entry->info.size > FILE_SIZE_MAX)
+        FILE_ERROR(EOVERFLOW, RC);
+
+    return (struct dirinfo)
+    {
+        .attribute = entry->info.attr,
+        .size      = entry->info.size,
+        .mtime     = fattime_mktime(entry->info.wrtdate, entry->info.wrttime),
+    };
+
+file_error:
+    return (struct dirinfo){ .attribute = 0 };
+}
diff --git a/firmware/common/dir_uncached.c b/firmware/common/dir_uncached.c
deleted file mode 100644
index b850a514e7..0000000000
--- a/firmware/common/dir_uncached.c
+++ /dev/null
@@ -1,312 +0,0 @@
-/***************************************************************************
- *             __________               __   ___.
- *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
- *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
- *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
- *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
- *                     \/            \/     \/    \/            \/
- * $Id: dir.c 13741 2007-06-30 02:08:27Z jethead71 $
- *
- * Copyright (C) 2002 by Björn Stenberg
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
- * KIND, either express or implied.
- *
- ****************************************************************************/
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
-#include <stdbool.h>
-#include <stdlib.h>
-#include "fat.h"
-#include "dir.h"
-#include "debug.h"
-#include "filefuncs.h"
-
-#if (MEMORYSIZE > 8)
-#define MAX_OPEN_DIRS 12
-#else
-#define MAX_OPEN_DIRS 8
-#endif
-
-static DIR_UNCACHED opendirs[MAX_OPEN_DIRS];
-
-// release all dir handles on a given volume "by force", to avoid leaks
-int release_dirs(int volume)
-{
-    DIR_UNCACHED* pdir = opendirs;
-    int dd;
-    int closed = 0;
-    for ( dd=0; dd<MAX_OPEN_DIRS; dd++, pdir++)
-    {
-#ifdef HAVE_MULTIVOLUME
-        if (pdir->fatdir.file.volume == volume)
-#else
-        (void)volume;
-#endif
-        {
-            pdir->busy = false; /* mark as available, no further action */
-            closed++;
-        }
-    }
-    return closed; /* return how many we did */
-}
-
-DIR_UNCACHED* opendir_uncached(const char* name)
-{
-    char namecopy[MAX_PATH];
-    char* part;
-    char* end;
-    struct fat_direntry entry;
-    int dd;
-    DIR_UNCACHED* pdir = opendirs;
-#ifdef HAVE_MULTIVOLUME
-    int volume;
-#endif
-
-    if ( name[0] != '/' ) {
-        DEBUGF("Only absolute paths supported right now\n");
-        return NULL;
-    }
-
-    /* find a free dir descriptor */
-    for ( dd=0; dd<MAX_OPEN_DIRS; dd++, pdir++)
-        if ( !pdir->busy )
-            break;
-
-    if ( dd == MAX_OPEN_DIRS ) {
-        DEBUGF("Too many dirs open\n");
-        errno = EMFILE;
-        return NULL;
-    }
-
-    pdir->busy = true;
-
-#ifdef HAVE_MULTIVOLUME
-    /* try to extract a heading volume name, if present */
-    volume = strip_volume(name, namecopy);
-    pdir->volumecounter = 0;
-#else
-    strlcpy(namecopy, name, sizeof(namecopy)); /* just copy */
-#endif
-
-    if ( fat_opendir(IF_MV(volume,) &pdir->fatdir, 0, NULL) < 0 ) {
-        DEBUGF("Failed opening root dir\n");
-        pdir->busy = false;
-        return NULL;
-    }
-
-    for ( part = strtok_r(namecopy, "/", &end); part;
-          part = strtok_r(NULL, "/", &end)) {
-        /* scan dir for name */
-        while (1) {
-            if ((fat_getnext(&pdir->fatdir,&entry) < 0) ||
-                (!entry.name[0])) {
-                pdir->busy = false;
-                return NULL;
-            }
-            if ( (entry.attr & FAT_ATTR_DIRECTORY) &&
-                 (!strcasecmp(part, entry.name)) ) {
-                /* In reality, the parent_dir parameter of fat_opendir seems
-                 * useless because it's sole purpose it to have a way to
-                 * update the file metadata, but here we are only reading
-                 * a directory so there's no need for that kind of stuff.
-                 * However, the rmdir_uncached function uses a ugly hack to
-                 * avoid opening a directory twice when deleting it and thus
-                 * needs those information. That's why we pass pdir->fatdir both
-                 * as the parent directory and the resulting one (this is safe,
-                 * in doubt, check fat_open(dir) code) which will allow this kind of
-                 * (ugly) things */
-                if ( fat_opendir(IF_MV(volume,)
-                                 &pdir->fatdir,
-                                 entry.firstcluster,
-                                 &pdir->fatdir) < 0 ) {
-                    DEBUGF("Failed opening dir '%s' (%ld)\n",
-                           part, entry.firstcluster);
-                    pdir->busy = false;
-                    return NULL;
-                }
-#ifdef HAVE_MULTIVOLUME
-                pdir->volumecounter = -1; /* n.a. to subdirs */
-#endif
-                break;
-            }
-        }
-    }
-
-    return pdir;
-}
-
-int closedir_uncached(DIR_UNCACHED* dir)
-{
-    dir->busy=false;
-    return 0;
-}
-
-struct dirent_uncached* readdir_uncached(DIR_UNCACHED* dir)
-{
-    struct fat_direntry entry;
-    struct dirent_uncached* theent = &(dir->theent);
-
-    if (!dir->busy)
-        return NULL;
-
-#ifdef HAVE_MULTIVOLUME
-    /* Volumes (secondary file systems) get inserted into the root directory
-        of the first volume, since we have no separate top level. */
-    if (dir->volumecounter >= 0 /* on a root dir */
-     && dir->volumecounter < NUM_VOLUMES /* in range */
-     && dir->fatdir.file.volume == 0) /* at volume 0 */
-    {   /* fake special directories, which don't really exist, but
-           will get redirected upon opendir_uncached() */
-        while (++dir->volumecounter < NUM_VOLUMES)
-        {
-            if (fat_ismounted(dir->volumecounter))
-            {
-                memset(theent, 0, sizeof(*theent));
-                theent->info.attribute = FAT_ATTR_DIRECTORY | FAT_ATTR_VOLUME;
-                snprintf(theent->d_name, sizeof(theent->d_name), 
-                         VOL_NAMES, dir->volumecounter);
-                return theent;
-            }
-        }
-    }
-#endif
-    /* normal directory entry fetching follows here */
-    if (fat_getnext(&(dir->fatdir),&entry) < 0)
-        return NULL;
-
-    if ( !entry.name[0] )
-        return NULL;
-
-    strlcpy(theent->d_name, entry.name, sizeof(theent->d_name));
-    theent->info.attribute = entry.attr;
-    theent->info.wrtdate = entry.wrtdate;
-    theent->info.wrttime = entry.wrttime;
-    theent->info.size = entry.filesize;
-    theent->startcluster = entry.firstcluster;
-
-    return theent;
-}
-
-int mkdir_uncached(const char *name)
-{
-    DIR_UNCACHED *dir;
-    char namecopy[MAX_PATH];
-    char* end;
-    char *basename;
-    char *parent;
-    struct dirent_uncached *entry;
-    int dd;
-    DIR_UNCACHED* pdir = opendirs;
-    struct fat_dir *newdir;
-    int rc;
-
-    if ( name[0] != '/' ) {
-        DEBUGF("mkdir: Only absolute paths supported right now\n");
-        return -1;
-    }
-    /* find a free dir descriptor */
-    for ( dd=0; dd<MAX_OPEN_DIRS; dd++, pdir++)
-        if ( !pdir->busy )
-            break;
-
-    if ( dd == MAX_OPEN_DIRS ) {
-        DEBUGF("Too many dirs open\n");
-        errno = EMFILE;
-        return -5;
-    }
-
-    pdir->busy = true;
-    newdir = &pdir->fatdir;
-
-    strlcpy(namecopy, name, sizeof(namecopy));
-
-    /* Split the base name and the path */
-    end = strrchr(namecopy, '/');
-    *end = 0;
-    basename = end+1;
-
-    if(namecopy == end) /* Root dir? */
-        parent = "/";
-    else
-        parent = namecopy;
-        
-    DEBUGF("mkdir: parent: %s, name: %s\n", parent, basename);
-
-    dir = opendir_uncached(parent);
-    
-    if(!dir) {
-        DEBUGF("mkdir: can't open parent dir\n");
-        pdir->busy = false;
-        return -2;
-    }    
-
-    if(basename[0] == 0) {
-        DEBUGF("mkdir: Empty dir name\n");
-        pdir->busy = false;
-        errno = EINVAL;
-        return -3;
-    }
-    
-    /* Now check if the name already exists */
-    while ((entry = readdir_uncached(dir))) {
-        if ( !strcasecmp(basename, entry->d_name) ) {
-            DEBUGF("mkdir error: file exists\n");
-            errno = EEXIST;
-            closedir_uncached(dir);
-            pdir->busy = false;
-            return - 4;
-        }
-    }
-
-    memset(newdir, 0, sizeof(struct fat_dir));
-    
-    rc = fat_create_dir(basename, newdir, &(dir->fatdir));
-    closedir_uncached(dir);
-    pdir->busy = false;
-    
-    return rc;
-}
-
-int rmdir_uncached(const char* name)
-{
-    int rc;
-    DIR_UNCACHED* dir;
-    struct dirent_uncached* entry;
-    
-    dir = opendir_uncached(name);
-    if (!dir)
-    {
-        errno = ENOENT; /* open error */
-        return -1;
-    }
-
-    /* check if the directory is empty */
-    while ((entry = readdir_uncached(dir)))
-    {
-        if (strcmp(entry->d_name, ".") &&
-            strcmp(entry->d_name, ".."))
-        {
-            DEBUGF("rmdir error: not empty\n");
-            errno = ENOTEMPTY;
-            closedir_uncached(dir);
-            return -2;
-        }
-    }
-
-    rc = fat_remove(&(dir->fatdir.file));
-    if ( rc < 0 ) {
-        DEBUGF("Failed removing dir: %d\n", rc);
-        errno = EIO;
-        rc = rc * 10 - 3;
-    }
-
-    closedir_uncached(dir);
-    return rc;
-}
diff --git a/firmware/common/dircache.c b/firmware/common/dircache.c
index b53fc4d7a6..1e580bf3af 100644
--- a/firmware/common/dircache.c
+++ b/firmware/common/dircache.c
@@ -8,6 +8,7 @@
  * $Id$
  *
  * Copyright (C) 2005 by Miika Pekkarinen
+ * Copyright (C) 2014 by Michael Sevakis
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -18,13 +19,7 @@
  * KIND, either express or implied.
  *
  ****************************************************************************/
-
-/* TODO:
-   - Allow cache live updating while transparent rebuild is running.
-*/
-
 #include "config.h"
-
 #include <stdio.h>
 #include <errno.h>
 #include "string-extra.h"
@@ -33,6 +28,8 @@
 #include "debug.h"
 #include "system.h"
 #include "logf.h"
+#include "fileobj_mgr.h"
+#include "pathfuncs.h"
 #include "dircache.h"
 #include "thread.h"
 #include "kernel.h"
@@ -42,393 +39,1626 @@
 #include "dir.h"
 #include "storage.h"
 #include "audio.h"
-#if CONFIG_RTC
-#include "time.h"
-#include "timefuncs.h"
-#endif
 #include "rbpaths.h"
+#include "linked_list.h"
+#ifdef HAVE_EEPROM_SETTINGS
+#include "crc32.h"
+#endif
 
+/**
+ * Cache memory layout:
+ * x - array of struct dircache_entry
+ * r - reserved buffer
+ * d - name buffer for the name entry of the struct dircache_entry
+ * 0 - zero bytes to assist free name block sentinel scanning (not 0xfe or 0xff)
+ * |xxxxxx|rrrrrrrrr|0|dddddd|0|
+ *
+ * Subsequent x are allocated from the front, d are allocated from the back,
+ * using the reserve buffer for entries added after initial scan.
+ *
+ * After a while the cache may look like:
+ * |xxxxxxxx|rrrrr|0|dddddddd|0|
+ *
+ * After a reboot, the reserve buffer is restored in it's size, so that the
+ * total allocation size grows:
+ * |xxxxxxxx|rrrrrrrrr|0|dddddddd|0|
+ *
+ *
+ * Cache structure:
+ * Format is memory position independent and uses only indexes as links. The
+ * buffer pointers are offset back by one entry to make the array 1-based so
+ * that an index of 0 may be considered an analog of a NULL pointer.
+ *
+ * Entry elements are linked together analagously to the filesystem directory
+ * structure with minor variations that are helpful to the cache's algorithms.
+ * Each volume has a special root structure in the dircache structure, not an
+ * entry in the cache, comprising a forest of volume trees which facilitates
+ * mounting or dismounting of specified volumes on the fly.
+ *
+ * Indexes identifying a volume are computed as: index = -volume - 1
+ * Returning the volume from these indexes is thus: volume = -index - 1
+ * Such indexes are used in root binding and as the 'up' index for an entry
+ * who's parent is the root directory.
+ *
+ * Open files list:
+ * When dircache is made it is the maintainer of the main volume open files
+ * lists, even when it is off. Any files open before dircache is enabled or
+ * initialized must be bound to cache entries by the scan and build operation.
+ * It maintains these lists in a special way.
+ *
+ * Queued (unresolved) bindings are at the back and resolved at the front.
+ * A pointer to the first of each kind of binding is provided to skip to help
+ * iterating one sublist or another.
+ *
+ * r0->r1->r2->q0->q1->q2->NULL
+ * ^resolved0  ^queued0
+ */
 
-/* Queue commands. */
-#define DIRCACHE_BUILD 1
-#define DIRCACHE_STOP  2
+#ifdef DIRCACHE_NATIVE
+#define dircache_lock()   file_internal_lock_WRITER()
+#define dircache_unlock() file_internal_unlock_WRITER()
 
-#if (MEMORYSIZE > 8)
-#define MAX_OPEN_DIRS 12
-#else
-#define MAX_OPEN_DIRS 8
-#endif
-static DIR_CACHED opendirs[MAX_OPEN_DIRS];
-static char       opendir_dnames[MAX_OPEN_DIRS][MAX_PATH];
+/* scan and build parameter data */
+struct sab_component;
+struct sab
+{
+    struct filestr_base   stream;    /* scan directory stream */
+    struct file_base_info info;      /* scanned entry info */
+    bool volatile         quit;      /* halt all scanning */
+    struct sab_component  *stackend; /* end of stack pointer */
+    struct sab_component  *top;      /* current top of stack */
+    struct sab_component
+    {
+        int volatile      idx;       /* cache index of directory */
+        int               *downp;    /* pointer to ce->down */
+        int *volatile     prevp;     /* previous item accessed */
+    } stack[];                       /* "recursion" stack */
+};
+
+#else /* !DIRCACHE_NATIVE */
+
+#error need locking scheme
+#define FILESYS_WRITER_LOCK()
+#define FILESYS_WRITER_UNLOCK()
 
-#define MAX_PENDING_BINDINGS 2
-struct fdbind_queue {
+struct sab_component
+{
+};
+
+struct sab
+{
+#ifdef HAVE_MUTLIVOLUME
+    int volume;
+#endif /* HAVE_MULTIVOLUME */
     char path[MAX_PATH];
-    int fd;
+    unsigned int append;
+};
+#endif /* DIRCACHE_NATIVE */
+
+enum
+{
+    FRONTIER_SETTLED = 0x0, /* dir entry contents are complete */
+    FRONTIER_NEW     = 0x1, /* dir entry contents are in-progress */
+    FRONTIER_ZONED   = 0x2, /* frontier entry permanent mark (very sticky!) */
+    FRONTIER_RENEW   = 0x4, /* override FRONTIER_ZONED sticky (not stored) */
 };
 
-/* Unions with char to make pointer arithmetic simpler and avoid casting */
-struct dircache_entry {
-    struct dirinfo info;
+enum
+{
+    DCM_BUILD,              /* build a volume */
+    DCM_PROCEED,            /* merged DCM_BUILD messages */
+    DCM_FIRST = DCM_BUILD,
+    DCM_LAST  = DCM_PROCEED,
+};
+
+#define MAX_TINYNAME   sizeof (uint32_t)
+#define DC_MAX_NAME    MIN(MAX_NAME, UINT8_MAX)
+
+/* Throw some warnings if about the limits if things may not work */
+#if MAX_NAME > UINT8_MAX
+#warning Need more than 8 bits in name length bitfield
+#endif
+
+#if DIRCACHE_LIMIT > ((1 << 24)-255)
+#warning Names may not be addressable with 24 bits
+#endif
+
+/* data structure used by cache entries */
+struct dircache_entry
+{
+    int         next;              /* next at same level */
     union {
-        struct dircache_entry *next;
-        char* next_char;
+    int         down;              /* first at child level (if directory) */
+    file_size_t filesize;          /* size of file in bytes (if file) */
     };
+    int         up;                /* parent index (-volume-1 if root) */
     union {
-        struct dircache_entry *up;
-        char* up_char;
+    struct {
+    uint32_t    name         : 24; /* indirect storage (.tinyname == 0) */
+    uint32_t    length       :  8; /* length of name indexed by 'name' */
     };
-    union {
-        struct dircache_entry *down;
-        char* down_char;
+    unsigned char namebuf[MAX_TINYNAME]; /* direct storage (.tinyname == 1) */
     };
-    long startcluster;
-    char *d_name;
-};
-
-/* Cache Layout:
- *
- * x - array of struct dircache_entry
- * r - reserved buffer
- * d - name buffer for the d_name entry of the struct dircache_entry
- * |xxxxxx|rrrrrrrrr|dddddd|
- *
- * subsequent x are allocated from the front, d are allocated from the back,
- * using the reserve buffer for entries added after initial scan
- *
- * after a while the cache may look like:
- * |xxxxxxxx|rrrrr|dddddddd|
- *
- * after a reboot, the reserve buffer is restored in it's size, so that the
- * total allocation size grows
- * |xxxxxxxx|rrrrrrrrr|dddddddd|
- */
-/* this points to the beginnging of the buffer and the first entry */
-static struct dircache_entry *dircache_root;
-/* these point to the start and end of the name buffer (d above) */
-static char                  *d_names_start, *d_names_end;
-/* put "." and ".." into the d_names buffer to enable easy pointer logic */
-static char                  *dot, *dotdot;
-#ifdef HAVE_MULTIVOLUME
-static struct dircache_entry *append_position;
+    uint32_t    direntry     : 16; /* entry # in parent - max 0xffff */
+    uint32_t    direntries   :  5; /* # of entries used - max 21 */
+    uint32_t    tinyname     :  1; /* if == 1, name fits in .namebuf */
+    uint32_t    frontier     :  2; /* (FRONTIER_* bitflags) */
+    uint32_t    attr         :  8; /* entry file attributes */
+#ifdef DIRCACHE_NATIVE
+    long        firstcluster;      /* first file cluster - max 0x0ffffff4 */
+    uint16_t    wrtdate;           /* FAT write date */
+    uint16_t    wrttime;           /* FAT write time */
+#else
+    time_t      mtime;             /* file last-modified time */
 #endif
+    dc_serial_t serialnum;         /* entry serial number */
+};
 
-static DIR_CACHED opendirs[MAX_OPEN_DIRS];
-static struct dircache_entry *fd_bindings[MAX_OPEN_FILES];
-
-static bool dircache_initialized = false;
-static bool dircache_initializing = false;
-static bool thread_enabled = false;
-static unsigned long allocated_size = 0;
-static unsigned long dircache_size = 0;
-static unsigned long entry_count = 0;
-static unsigned long reserve_used = 0;
-static unsigned int  cache_build_ticks = 0;
-static unsigned long appflags = 0;
+/* spare us some tedium */
+#define ENTRYSIZE   (sizeof (struct dircache_entry))
 
+/* thread and kernel stuff */
 static struct event_queue dircache_queue SHAREDBSS_ATTR;
-static long dircache_stack[(DEFAULT_STACK_SIZE + 0x400)/sizeof(long)];
+static uintptr_t dircache_stack[DIRCACHE_STACK_SIZE / sizeof (uintptr_t)];
 static const char dircache_thread_name[] = "dircache";
 
-static struct fdbind_queue fdbind_cache[MAX_PENDING_BINDINGS];
-static int fdbind_idx = 0;
+/* struct that is both used during run time and for persistent storage */
+static struct dircache
+{
+    /* cache-wide data */
+    int          free_list;           /* first index of free entry list */
+    size_t       size;                /* total size of data (including holes) */
+    size_t       sizeused;            /* bytes of .size bytes actually used */
+    union {
+    unsigned int numentries;          /* entry count (including holes) */
+#ifdef HAVE_EEPROM_SETTINGS
+    size_t       sizeentries;         /* used when persisting */
+#endif
+    };
+    int          names;               /* index of first name in name block */
+    size_t       sizenames;           /* size of all names (including holes) */
+    size_t       namesfree;           /* amount of wasted name space */
+    int          nextnamefree;        /* hint of next free name in buffer */
+    /* per-volume data */
+    struct dircache_volume            /* per volume cache data */
+    {
+        uint32_t       status   : 2;  /* cache status of this volume */
+        uint32_t       frontier : 2;  /* (FRONTIER_* bitflags) */
+        dc_serial_t    serialnum;     /* dircache serial number of root */
+        int            root_down;     /* index of first entry of volume root */
+        union {
+        long           start_tick;    /* when did scan start (scanning) */
+        long           build_ticks;   /* how long to build volume? (ready) */
+        };
+    } dcvol[NUM_VOLUMES];
+    /* these remain unchanged between cache resets */
+    size_t       last_size;           /* last reported size at boot */
+    size_t       reserve_used;        /* reserved used at last build */
+    dc_serial_t  last_serialnum;      /* last serialnumber generated */
+} dircache;
+
+/* struct that is used only for the cache in main memory */
+struct dircache_runinfo
+{
+    /* cache setting and build info */
+    int          suspended;        /* dircache suspend count */
+    bool         enabled;          /* dircache master enable switch */
+    unsigned int thread_id;        /* current/last thread id */
+    bool         thread_done;      /* thread has exited */
+    /* cache buffer info */
+    int          handle;           /* buflib buffer handle */
+    size_t       bufsize;          /* size of buflib allocation - 1 */
+    int          buflocked;        /* don't move due to other allocs */
+    union {
+    void                  *p;      /* address of buffer - ENTRYSIZE */
+    struct dircache_entry *pentry; /* alias of .p to assist entry resolution */
+    unsigned char         *pname;  /* alias of .p to assist name resolution */
+    };
+    struct buflib_callbacks ops;   /* buflib ops callbacks */
+    /* per-volume data */
+    struct dircache_runinfo_volume
+    {
+        struct file_base_binding *resolved0; /* first resolved binding in list */
+        struct file_base_binding *queued0;   /* first queued binding in list */
+        struct sab               *sabp;      /* if building, struct sab in use */
+    } dcrivol[NUM_VOLUMES];
+} dircache_runinfo;
+
+#define BINDING_NEXT(bindp) \
+    ((struct file_base_binding *)(bindp)->node.next)
+
+#define FOR_EACH_BINDING(start, p) \
+    for (struct file_base_binding *p = (start); p; p = BINDING_NEXT(p))
+
+#define FOR_EACH_CACHE_ENTRY(ce) \
+    for (struct dircache_entry *ce = &dircache_runinfo.pentry[1],  \
+                               *_ceend = ce + dircache.numentries; \
+         ce < _ceend; ce++) if (ce->serialnum)
+
+#define FOR_EACH_SAB_COMP(sabp, p) \
+    for (struct sab_component *p = (sabp)->top; p < (sabp)->stackend; p++)
+
+/* "overloaded" macros to get volume structures */
+#define DCVOL_i(i)               (&dircache.dcvol[i])
+#define DCVOL_volume(volume)     (&dircache.dcvol[volume])
+#define DCVOL_infop(infop)       (&dircache.dcvol[BASEINFO_VOL(infop)])
+#define DCVOL_dirinfop(dirinfop) (&dircache.dcvol[BASEINFO_VOL(dirinfop)])
+#define DCVOL(x)                 DCVOL_##x(x)
+
+#define DCRIVOL_i(i)             (&dircache_runinfo.dcrivol[i])
+#define DCRIVOL_infop(infop)     (&dircache_runinfo.dcrivol[BASEINFO_VOL(infop)])
+#define DCRIVOL_bindp(bindp)     (&dircache_runinfo.dcrivol[BASEBINDING_VOL(bindp)])
+#define DCRIVOL(x)               DCRIVOL_##x(x)
+
+/* reserve over 75% full? */
+#define DIRCACHE_STUFFED(reserve_used) \
+    ((reserve_used) > 3*DIRCACHE_RESERVE / 4)
 
-/* --- Internal cache structure control functions --- */
+#ifdef HAVE_EEPROM_SETTINGS
+/**
+ * remove the snapshot file
+ */
+static int remove_dircache_file(void)
+{
+    return remove(DIRCACHE_FILE);
+}
+
+/**
+ * open or create the snapshot file
+ */
+static int open_dircache_file(int oflag)
+{
+    return open(DIRCACHE_FILE, oflag, 0666);
+}
+#endif /* HAVE_EEPROM_SETTINGS */
 
-static inline struct dircache_entry* get_entry(int id)
+#ifdef DIRCACHE_DUMPSTER
+/**
+ * clean up the memory allocation to make viewing in a hex editor more friendly
+ * and highlight problems
+ */
+static inline void dumpster_clean_buffer(void *p, size_t size)
 {
-    return &dircache_root[id];
+    memset(p, 0xAA, size);
 }
+#endif /* DIRCACHE_DUMPSTER */
 
-/* flag to make sure buffer doesn't move due to other allocs.
- * this is set to true completely during dircache build */
-static bool dont_move = false;
-static int dircache_handle;
-static int move_callback(int handle, void* current, void* new)
+/**
+ * relocate the cache when the buffer has moved
+ */
+static int move_callback(int handle, void *current, void *new)
 {
-    (void)handle;
-    if (dont_move)
+    if (dircache_runinfo.buflocked)
         return BUFLIB_CB_CANNOT_MOVE;
 
-#define UPDATE(x) if (x) { x = PTR_ADD(x, diff); }
-    /* relocate the cache */
-    ptrdiff_t diff = new - current;
-    for(unsigned i = 0; i < entry_count; i++)
+    dircache_runinfo.p = new - ENTRYSIZE;
+
+    return BUFLIB_CB_OK;
+    (void)handle; (void)current;
+}
+
+/**
+ * add a "don't move" lock count
+ */
+static inline void buffer_lock(void)
+{
+    dircache_runinfo.buflocked++;
+}
+
+/**
+ * remove a "don't move" lock count
+ */
+static inline void buffer_unlock(void)
+{
+    dircache_runinfo.buflocked--;
+}
+
+
+/** Open file bindings management **/
+
+/* compare the basic file information and return 'true' if they are logically
+   equivalent or the same item, else return 'false' if not */
+static inline bool binding_compare(const struct file_base_info *infop1,
+                                   const struct file_base_info *infop2)
+{
+#ifdef DIRCACHE_NATIVE
+    return fat_file_is_same(&infop1->fatfile, &infop2->fatfile);
+#else
+    #error hey watch it!
+#endif
+}
+
+/**
+ * bind a file to the cache; "queued" or "resolved" depending upon whether or
+ * not it has entry information
+ */
+static void binding_open(struct file_base_binding *bindp)
+{
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(bindp);
+    if (bindp->info.dcfile.serialnum)
     {
-        UPDATE(dircache_root[i].d_name);
-        UPDATE(dircache_root[i].next_char);
-        UPDATE(dircache_root[i].up_char);
-        UPDATE(dircache_root[i].down_char);
+        /* already resolved */
+        dcrivolp->resolved0 = bindp;
+        file_binding_insert_first(bindp);
     }
-    dircache_root = new;
-    UPDATE(d_names_start);
-    UPDATE(d_names_end);
-    UPDATE(dot);
-    UPDATE(dotdot);
+    else
+    {
+        if (dcrivolp->queued0 == NULL)
+            dcrivolp->queued0 = bindp;
 
-    for(unsigned i = 0; i < MAX_OPEN_FILES; i++)
-        UPDATE(fd_bindings[i]);
+        file_binding_insert_last(bindp);
+    }
+}
 
-#ifdef HAVE_MULTIVOLUME
-    UPDATE(append_position);
-#endif
-    return BUFLIB_CB_OK;
+/**
+ * remove a binding from the cache
+ */
+static void binding_close(struct file_base_binding *bindp)
+{
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(bindp);
+
+    if (bindp == dcrivolp->queued0)
+        dcrivolp->queued0 = BINDING_NEXT(bindp);
+    else if (bindp == dcrivolp->resolved0)
+    {
+        struct file_base_binding *nextp = BINDING_NEXT(bindp);
+        dcrivolp->resolved0 = (nextp == dcrivolp->queued0) ? NULL : nextp;
+    }
+
+    file_binding_remove(bindp);
+    /* no need to reset it */
 }
 
-static struct buflib_callbacks ops = {
-    .move_callback = move_callback,
-    .shrink_callback = NULL,
-};
+/**
+ * resolve a queued binding with the information from the given source file
+ */
+static void binding_resolve(const struct file_base_info *infop)
+{
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(infop);
+
+    /* quickly check the queued list to see if it's there */
+    struct file_base_binding *prevp = NULL;
+    FOR_EACH_BINDING(dcrivolp->queued0, p)
+    {
+        if (!binding_compare(infop, &p->info))
+        {
+            prevp = p;
+            continue;
+        }
+
+        if (p == dcrivolp->queued0)
+            dcrivolp->queued0 = BINDING_NEXT(p);
+        else
+        {
+            file_binding_remove_next(prevp, p);
+            file_binding_insert_first(p);
+        }
+
+        dcrivolp->resolved0 = p;
+
+        /* srcinfop may be the actual one */
+        if (&p->info != infop)
+            p->info.dcfile = infop->dcfile;
+
+        break;
+    }
+}
 
-#ifdef HAVE_EEPROM_SETTINGS
 /**
- * Open the dircache file to save a snapshot on disk
+ * dissolve a resolved binding on its volume
  */
-static int open_dircache_file(unsigned flags, int permissions)
+static void binding_dissolve(struct file_base_binding *prevp,
+                             struct file_base_binding *bindp)
 {
-    if (permissions != 0)
-        return open(DIRCACHE_FILE, flags, permissions);
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(bindp);
 
-    return open(DIRCACHE_FILE, flags);
+    if (bindp == dcrivolp->resolved0)
+    {
+        struct file_base_binding *nextp = BINDING_NEXT(bindp);
+        dcrivolp->resolved0 = (nextp == dcrivolp->queued0) ? NULL : nextp;
+    }
+
+    if (dcrivolp->queued0 == NULL)
+        dcrivolp->queued0 = bindp;
+
+    file_binding_remove_next(prevp, bindp);
+    file_binding_insert_last(bindp);
+
+    dircache_dcfile_init(&bindp->info.dcfile);
 }
 
 /**
- * Remove the snapshot file
+ * dissolve all resolved bindings on a given volume
  */
-static int remove_dircache_file(void)
+static void binding_dissolve_volume(struct dircache_runinfo_volume *dcrivolp)
 {
-    return remove(DIRCACHE_FILE);
+    if (!dcrivolp->resolved0)
+        return;
+
+    FOR_EACH_BINDING(dcrivolp->resolved0, p)
+    {
+        if (p == dcrivolp->queued0)
+            break;
+
+        dircache_dcfile_init(&p->info.dcfile);
+    }
+
+    dcrivolp->queued0 = dcrivolp->resolved0;
+    dcrivolp->resolved0 = NULL;
 }
-#endif
-/** 
- * Internal function to allocate a new dircache_entry from memory.
+
+
+/** Dircache buffer management **/
+
+/**
+ * allocate the cache's memory block
  */
-static struct dircache_entry* allocate_entry(void)
+static int alloc_cache(size_t size)
 {
-    struct dircache_entry *next_entry;
-    
-    if (dircache_size > allocated_size - MAX_PATH*2)
+    /* pad with one extra-- see alloc_name() and free_name() */
+    return core_alloc_ex("dircache", size + 1, &dircache_runinfo.ops);
+}
+
+/**
+ * put the allocation in dircache control
+ */
+static void set_buffer(int handle, size_t size)
+{
+    void *p = core_get_data(handle);
+
+#ifdef DIRCACHE_DUMPSTER
+    dumpster_clean_buffer(p, size);
+#endif /* DIRCACHE_DUMPSTER */
+
+    /* set it up as a 1-based array */
+    dircache_runinfo.p = p - ENTRYSIZE;
+
+    if (dircache_runinfo.handle != handle)
     {
-        logf("size limit reached");
-        return NULL;
+        /* new buffer */
+        dircache_runinfo.handle = handle;
+        dircache_runinfo.bufsize = size;
+        dircache.names = size + ENTRYSIZE;
+        dircache_runinfo.pname[dircache.names - 1] = 0;
+        dircache_runinfo.pname[dircache.names    ] = 0;
     }
-    
-    next_entry = &dircache_root[entry_count++];
-    next_entry->d_name = NULL;
-    next_entry->up = NULL;
-    next_entry->down = NULL;
-    next_entry->next = NULL;
+}
 
-    dircache_size += sizeof(struct dircache_entry);
+/**
+ * remove the allocation from dircache control and return the handle
+ */
+static int reset_buffer(void)
+{
+    int handle = dircache_runinfo.handle;
+    if (handle > 0)
+    {
+        /* don't mind .p; it might get changed by the callback even after
+           this call; buffer presence is determined by the following: */
+        dircache_runinfo.handle  = 0;
+        dircache_runinfo.bufsize = 0;
+    }
+
+    return handle;
+}
+
+/**
+ * return the number of bytes remaining in the buffer
+ */
+static size_t dircache_buf_remaining(void)
+{
+    if (!dircache_runinfo.handle)
+        return 0;
+
+    return dircache_runinfo.bufsize - dircache.size;
+}
+
+/**
+ * return the amount of reserve space used
+ */
+static size_t reserve_buf_used(void)
+{
+    size_t remaining = dircache_buf_remaining();
+    return (remaining < DIRCACHE_RESERVE) ?
+                DIRCACHE_RESERVE - remaining : 0;
+}
+
+
+/** Internal cache structure control functions **/
+
+/**
+ * generate the next serial number in the sequence
+ */
+static dc_serial_t next_serialnum(void)
+{
+    dc_serial_t serialnum = MAX(dircache.last_serialnum + 1, 1);
+    dircache.last_serialnum = serialnum;
+    return serialnum;
+}
+
+/**
+ * return the dircache volume pointer for the special index
+ */
+static struct dircache_volume * get_idx_dcvolp(int idx)
+{
+    if (idx >= 0)
+        return NULL;
+
+    return &dircache.dcvol[IF_MV_VOL(-idx - 1)];
+}
+
+/**
+ * return the cache entry referenced by idx (NULL if outside buffer)
+ */
+static struct dircache_entry * get_entry(int idx)
+{
+    if (idx <= 0 || (unsigned int)idx > dircache.numentries)
+        return NULL;
 
-    return next_entry;
+    return &dircache_runinfo.pentry[idx];
 }
 
 /**
- * Internal function to allocate a dircache_entry and set 
- * ->next entry pointers.
+ * return the index of the cache entry (0 if outside buffer)
  */
-static struct dircache_entry* dircache_gen_next(struct dircache_entry *ce)
+static int get_index(const struct dircache_entry *ce)
 {
-    struct dircache_entry *next_entry;
+    if (!PTR_IN_ARRAY(dircache_runinfo.pentry + 1, ce,
+                      dircache.numentries + 1))
+    {
+        return 0;
+    }
+
+    return ce - dircache_runinfo.pentry;
+}
 
-    if ( (next_entry = allocate_entry()) == NULL)
+/**
+ *  return the sublist down pointer for the sublist that contains entry 'idx'
+ */
+static int * get_downidxp(int idx)
+{
+    /* NOTE: 'idx' must refer to a directory or the result is undefined */
+    if (idx == 0 || idx < -NUM_VOLUMES)
         return NULL;
-    next_entry->up = ce->up;
-    ce->next = next_entry;
-    
-    return next_entry;
+
+    if (idx > 0)
+    {
+        /* a normal entry */
+        struct dircache_entry *ce = get_entry(idx);
+        return ce ? &ce->down : NULL;
+    }
+    else
+    {
+        /* a volume root */
+        return &get_idx_dcvolp(idx)->root_down;
+    }
 }
 
-/*
- * Internal function to allocate a dircache_entry and set
- * ->down entry pointers.
+/**
+ * return a pointer to the index referencing the cache entry that 'idx'
+ * references
  */
-static struct dircache_entry* dircache_gen_down(struct dircache_entry *ce)
+static int * get_previdxp(int idx)
 {
-    struct dircache_entry *next_entry;
+    struct dircache_entry *ce = get_entry(idx);
 
-    if ( (next_entry = allocate_entry()) == NULL)
+    int *prevp = get_downidxp(ce->up);
+    if (!prevp)
         return NULL;
-    next_entry->up = ce;
-    ce->down = next_entry;
-    
-    return next_entry;
+
+    while (1)
+    {
+        int next = *prevp;
+        if (!next || next == idx)
+            break;
+
+        prevp = &get_entry(next)->next;
+    }
+
+    return prevp;
+}
+
+/**
+ * if required, adjust the lists and directory read of any scan and build in
+ * progress
+ */
+static void sab_sync_scan(struct sab *sabp, int *prevp, int *nextp)
+{
+    struct sab_component *abovep = NULL;
+    FOR_EACH_SAB_COMP(sabp, p)
+    {
+        if (nextp != p->prevp)
+        {
+            abovep = p;
+            continue;
+        }
+
+        /* removing an item being scanned; set the component position to the
+           entry before this */
+        p->prevp = prevp;
+
+        if (p == sabp->top)
+        {
+            /* removed at item in the directory who's immediate contents are
+               being scanned */
+            if (prevp == p->downp)
+            {
+                /* was first item; rewind it */
+                dircache_rewinddir_internal(&sabp->info);
+            }
+            else
+            {
+                struct dircache_entry *ceprev =
+                    container_of(prevp, struct dircache_entry, next);
+            #ifdef DIRCACHE_NATIVE
+                sabp->info.fatfile.e.entry   = ceprev->direntry;
+                sabp->info.fatfile.e.entries = ceprev->direntries;
+            #endif
+            }
+        }
+        else if (abovep)
+        {
+            /* the directory being scanned or a parent of it has been removed;
+               abort its build or cache traversal */
+            abovep->idx = 0;
+        }
+
+        break;
+    }
+}
+
+/**
+ * get a pointer to an allocated name given a cache index
+ */
+static inline unsigned char * get_name(int nameidx)
+{
+    return &dircache_runinfo.pname[nameidx];
+}
+
+/**
+ * get the cache buffer index of the given name
+ */
+static inline int get_nameidx(const unsigned char *pname)
+{
+    return pname - dircache_runinfo.pname;
+}
+
+/**
+ * copy the entry's name to a buffer (which assumed to be of sufficient size)
+ */
+static void entry_name_copy(char *dst, const struct dircache_entry *ce)
+{
+    if (LIKELY(!ce->tinyname))
+    {
+        strmemcpy(dst, get_name(ce->name), ce->length);
+        return;
+    }
+
+    const unsigned char *src = ce->namebuf;
+    size_t len = 0;
+    while (len++ < MAX_TINYNAME && *src)
+        *dst++ = *src++;
+
+    *dst = '\0';
+}
+
+/**
+ * set the namesfree hint to a new position
+ */
+static void set_namesfree_hint(const unsigned char *hintp)
+{
+    int hintidx = get_nameidx(hintp);
+
+    if (hintidx >= (int)(dircache.names + dircache.sizenames))
+        hintidx = dircache.names;
+
+    dircache.nextnamefree = hintidx;
 }
 
 /**
- * Returns true if there is an event waiting in the queue
- * that requires the current operation to be aborted.
+ * allocate a buffer to use for a new name
  */
-static bool check_event_queue(void)
+static int alloc_name(size_t size)
+{
+    int nameidx = 0;
+
+    if (dircache.namesfree >= size)
+    {
+        /* scan for a free gap starting at the hint point - first fit */
+        unsigned char *start = get_name(dircache.nextnamefree), *p = start;
+        unsigned char *namesend = get_name(dircache.names + dircache.sizenames);
+        size_t gapsize = 0;
+
+        while (gapsize < size)
+        {
+            if ((p = memchr(p, 0xff, namesend - p)))
+            {
+                /* found a sentinel; see if there are enough in a row */
+                gapsize = 1;
+                while (*++p == 0xff && gapsize < size)
+                    gapsize++;
+            }
+            else
+            {
+                if (namesend == start)
+                    break; /* exhausted */
+
+                /* wrap */
+                namesend = start;
+                p = get_name(dircache.names);
+
+                if (p == namesend)
+                    break; /* initial hint was at names start */
+            }
+        }
+
+        if (gapsize >= size)
+        {
+            unsigned char *namep = p - gapsize;
+            nameidx = get_nameidx(namep);
+
+            if (*p == 0xff)
+            {
+                /* if only a tiny block remains after buffer, claim it too */
+                size_t tinysize = 1;
+                while (*++p == 0xff && tinysize <= MAX_TINYNAME)
+                    tinysize++;
+
+                if (tinysize <= MAX_TINYNAME)
+                {
+                    /* mark with tiny block sentinel */
+                    memset(p - tinysize, 0xfe, tinysize);
+                    size += tinysize;
+                }
+            }
+
+            dircache.namesfree -= size;
+            dircache.sizeused += size;
+            set_namesfree_hint(namep + size);
+        }
+    }
+
+    if (!nameidx)
+    {
+        /* no sufficiently long free gaps; allocate anew */
+        if (dircache_buf_remaining() <= size)
+        {
+            dircache.last_size = 0;
+            return 0;
+        }
+
+        dircache.names     -= size;
+        dircache.sizenames += size;
+        nameidx             = dircache.names;
+        dircache.size      += size;
+        dircache.sizeused  += size;
+        *get_name(dircache.names - 1) = 0;
+    }
+
+    return nameidx;
+}
+
+/**
+ * mark a name as free and note that its bytes are available
+ */
+static void free_name(int nameidx, size_t size)
+{
+    unsigned char *beg = get_name(nameidx);
+    unsigned char *end = beg + size;
+
+    /* merge with any adjacent tiny blocks */
+    while (beg[-1] == 0xfe)
+        --beg;
+
+    while (end[1] == 0xfe)
+        ++end;
+
+    size = end - beg;
+    memset(beg, 0xff, size);
+    dircache.namesfree += size;
+    dircache.sizeused -= size;
+    set_namesfree_hint(beg);
+}
+
+/**
+ * allocate and assign a name to the entry
+ */
+static bool entry_assign_name(struct dircache_entry *ce,
+                              const unsigned char *name, size_t size)
+{
+    unsigned char *copyto;
+
+    if (size <= MAX_TINYNAME)
+    {
+        copyto = ce->namebuf;
+
+        if (size < MAX_TINYNAME)
+            copyto[size] = '\0';
+
+        ce->tinyname = 1;
+    }
+    else
+    {
+        if (size > DC_MAX_NAME)
+            size = DC_MAX_NAME;
+
+        int nameidx = alloc_name(size);
+        if (!nameidx)
+            return false;
+
+        copyto = get_name(nameidx);
+
+        ce->tinyname = 0;
+        ce->name     = nameidx;
+        ce->length   = size;
+    }
+
+    memcpy(copyto, name, size);
+    return true;
+}
+
+/**
+ * free the name for the entry
+ */
+static void entry_unassign_name(struct dircache_entry *ce)
+{
+    if (!ce->tinyname)
+        free_name(ce->name, ce->length);
+}
+
+/**
+ * assign a new name to the entry
+ */
+static bool entry_reassign_name(struct dircache_entry *ce,
+                                const unsigned char *newname)
+{
+    size_t oldlen = ce->tinyname ? 0 : ce->length;
+    size_t newlen = strlen(newname);
+
+    if (oldlen == newlen || (oldlen == 0 && newlen <= MAX_TINYNAME))
+    {
+        char *p = mempcpy(oldlen == 0 ? ce->namebuf : get_name(ce->name),
+                          newname, newlen);
+        if (newlen < MAX_TINYNAME)
+            *p = '\0';
+        return true;
+    }
+
+    /* needs a new name allocation; if the new name fits in the freed block,
+       it will use it immediately without a lengthy search */
+    entry_unassign_name(ce);
+    return entry_assign_name(ce, newname, newlen);
+}
+
+/**
+ * allocate a dircache_entry from memory using freed ones if available
+ */
+static int alloc_entry(struct dircache_entry **res)
+{
+    struct dircache_entry *ce;
+    int idx = dircache.free_list;
+
+    if (idx)
+    {
+        /* reuse a freed entry */
+        ce = get_entry(idx);
+        dircache.free_list = ce->next;
+    }
+    else if (dircache_buf_remaining() > ENTRYSIZE)
+    {
+        /* allocate a new one */
+        idx = ++dircache.numentries;
+        dircache.size += ENTRYSIZE;
+        ce = get_entry(idx);
+    }
+    else
+    {
+        dircache.last_size = 0;
+        *res = NULL;
+        return 0;
+    }
+
+    dircache.sizeused += ENTRYSIZE;
+
+    ce->next      = 0;
+    ce->up        = 0;
+    ce->down      = 0;
+    ce->length    = 0;
+    ce->frontier  = FRONTIER_SETTLED;
+    ce->serialnum = next_serialnum();
+
+    *res = ce;
+    return idx;
+}
+
+/**
+ * free an entry's allocations in the cache; must not be linked to anything
+ * by this time (orphan!)
+ */
+static void free_orphan_entry(struct dircache_runinfo_volume *dcrivolp,
+                              struct dircache_entry *ce, int idx)
+{
+    if (dcrivolp)
+    {
+        /* was an established entry; find any associated resolved binding and
+           dissolve it;  bindings are kept strictly synchronized with changes
+           to the storage so a simple serial number comparison is sufficient */
+        struct file_base_binding *prevp = NULL;
+        FOR_EACH_BINDING(dcrivolp->resolved0, p)
+        {
+            if (p == dcrivolp->queued0)
+                break;
+
+            if (ce->serialnum == p->info.dcfile.serialnum)
+            {
+                binding_dissolve(prevp, p);
+                break;
+            }
+
+            prevp = p;
+        }
+    }
+
+    entry_unassign_name(ce);
+
+    /* no serialnum says "it's free" (for cache-wide iterators) */
+    ce->serialnum = 0;
+
+    /* add to free list */
+    ce->next = dircache.free_list;
+    dircache.free_list = idx;
+    dircache.sizeused -= ENTRYSIZE;
+}
+
+/**
+ * allocates a new entry of with the name specified by 'basename'
+ */
+static int create_entry(const char *basename,
+                        struct dircache_entry **res)
+{
+    int idx = alloc_entry(res);
+    if (!idx)
+    {
+        logf("size limit reached (entry)");
+        return 0; /* full */
+    }
+
+    if (!entry_assign_name(*res, basename, strlen(basename)))
+    {
+        free_orphan_entry(NULL, *res, idx);
+        logf("size limit reached (name)");
+        return 0; /* really full! */
+    }
+
+    return idx;
+}
+
+/**
+ * unlink the entry at *prevp and adjust the scanner if needed
+ */
+static void remove_entry(struct dircache_runinfo_volume *dcrivolp,
+                         struct dircache_entry *ce, int *prevp)
+{
+    /* unlink it from its list */
+    *prevp = ce->next;
+
+    if (dcrivolp)
+    {
+        /* adjust scanner iterator if needed */
+        struct sab *sabp = dcrivolp->sabp;
+        if (sabp)
+            sab_sync_scan(sabp, prevp, &ce->next);
+    }
+}
+
+/**
+ * free the entire subtree in the referenced parent down index
+ */
+static void free_subentries(struct dircache_runinfo_volume *dcrivolp, int *downp)
+{
+    while (1)
+    {
+        int idx = *downp;
+        struct dircache_entry *ce = get_entry(idx);
+        if (!ce)
+            break;
+
+        if ((ce->attr & ATTR_DIRECTORY) && ce->down)
+            free_subentries(dcrivolp, &ce->down);
+
+        remove_entry(dcrivolp, ce, downp);
+        free_orphan_entry(dcrivolp, ce, idx);
+    }
+}
+
+/**
+ * free the specified file entry and its children
+ */
+static void free_file_entry(struct file_base_info *infop)
+{
+    int idx = infop->dcfile.idx;
+    if (idx <= 0)
+        return; /* can't remove a root/invalid */
+
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(infop);
+
+    struct dircache_entry *ce = get_entry(idx);
+    if ((ce->attr & ATTR_DIRECTORY) && ce->down)
+    {
+        /* gonna get all this contents (normally the "." and "..") */
+        free_subentries(dcrivolp, &ce->down);
+    }
+
+    remove_entry(dcrivolp, ce, get_previdxp(idx));
+    free_orphan_entry(dcrivolp, ce, idx);
+}
+
+/**
+ * insert the new entry into the parent, sorted into position
+ */
+static void insert_file_entry(struct file_base_info *dirinfop,
+                              struct dircache_entry *ce)
+{
+    /* DIRCACHE_NATIVE: the entires are sorted into the spot it would be on
+     * the storage medium based upon the directory entry number, in-progress
+     * scans will catch it or miss it in just the same way they would if
+     * directly scanning the disk. If this is behind an init scan, it gets
+     * added anyway; if in front of it, then scanning will compare what it
+     * finds in order to avoid adding a duplicate.
+     *
+     * All others: the order of entries of the host filesystem is not known so
+     * this must be placed at the end so that a build scan won't miss it and
+     * add a duplicate since it will be comparing any entries it finds in front
+     * of it.
+     */
+    int diridx = dirinfop->dcfile.idx;
+    int *nextp = get_downidxp(diridx);
+
+    while (8675309)
+    {
+        struct dircache_entry *nextce = get_entry(*nextp);
+        if (!nextce)
+            break;
+
+#ifdef DIRCACHE_NATIVE
+        if (nextce->direntry > ce->direntry)
+            break;
+
+        /* now, nothing should be equal to ours or that is a bug since it
+           would already exist (and it shouldn't because it's just been
+           created or moved) */
+#endif /* DIRCACHE_NATIVE */
+
+        nextp = &nextce->next;
+    }
+
+    ce->up   = diridx;
+    ce->next = *nextp;
+    *nextp   = get_index(ce);
+}
+
+/**
+ * unlink the entry from its parent and return its pointer to the caller
+ */
+static struct dircache_entry * remove_file_entry(struct file_base_info *infop)
+{
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(infop);
+    struct dircache_entry *ce = get_entry(infop->dcfile.idx);
+    remove_entry(dcrivolp, ce, get_previdxp(infop->dcfile.idx));
+    return ce;
+}
+
+/**
+ * set the frontier indicator for the given cache index
+ */
+static void establish_frontier(int idx, uint32_t code)
+{
+    if (idx < 0)
+    {
+        int volume = IF_MV_VOL(-idx - 1);
+        uint32_t val = dircache.dcvol[volume].frontier;
+        if (code & FRONTIER_RENEW)
+            val &= ~FRONTIER_ZONED;
+        dircache.dcvol[volume].frontier = code | (val & FRONTIER_ZONED);
+    }
+    else if (idx > 0)
+    {
+        struct dircache_entry *ce = get_entry(idx);
+        uint32_t val = ce->frontier;
+        if (code & FRONTIER_RENEW)
+            val &= ~FRONTIER_ZONED;
+        ce->frontier = code | (val & FRONTIER_ZONED);
+    }
+}
+
+/**
+ * remove all messages from the queue, responding to anyone waiting
+ */
+static void clear_dircache_queue(void)
 {
     struct queue_event ev;
-    
-    if(!queue_peek(&dircache_queue, &ev))
-        return false;
-    
-    switch (ev.id)
-    {
-        case DIRCACHE_STOP:
-        case SYS_USB_CONNECTED:
-#ifdef HAVE_HOTSWAP
-        case SYS_FS_CHANGED:
-#endif
-            return true;
+
+    while (1)
+    {
+        queue_wait_w_tmo(&dircache_queue, &ev, 0);
+        if (ev.id == SYS_TIMEOUT)
+            break;
+
+        /* respond to any synchronous build queries; since we're already
+           building and thusly allocated, any additional requests can be
+           processed async */
+        if (ev.id == DCM_BUILD)
+        {
+            int *rcp = (int *)ev.data;
+            if (rcp)
+                *rcp = 0;
+        }
     }
-    
-    return false;
 }
 
-#if (CONFIG_PLATFORM & PLATFORM_NATIVE)
-/* scan and build static data (avoid redundancy on stack) */
-static struct
+/**
+ * service dircache_queue during a scan and build
+ */
+static void process_events(void)
 {
-#ifdef HAVE_MULTIVOLUME
-    int volume;
-#endif
-    struct fat_dir *dir;
-    struct fat_direntry *direntry;
-}sab;
-
-static int sab_process_dir(unsigned long startcluster, struct dircache_entry *ce)
-{
-    /* normally, opendir expects a full fat_dir as parent but in our case,
-     * it's completely useless because we don't modify anything
-     * WARNING: this heavily relies on current FAT implementation ! */
-    
-    /* those field are necessary to update the FAT entry in case of modification
-       here we don't touch anything so we put dummy values */
-    sab.dir->entry = 0;
-    sab.dir->entrycount = 0;
-    sab.dir->file.firstcluster = 0;
-    /* open directory */
-    int rc = fat_opendir(IF_MV(sab.volume,) sab.dir, startcluster, sab.dir);
-    if(rc < 0)
-    {
-        logf("fat_opendir failed: %d", rc);
-        return rc;
+    yield();
+
+    /* only count externally generated commands */
+    if (!queue_peek_ex(&dircache_queue, NULL, 0, QPEEK_FILTER1(DCM_BUILD)))
+        return;
+
+    clear_dircache_queue();
+
+    /* this reminds us to keep moving after we're done here; a volume we passed
+       up earlier could have been mounted and need refreshing; it just condenses
+       a slew of requests into one and isn't mistaken for an externally generated
+       command */
+    queue_post(&dircache_queue, DCM_PROCEED, 0);
+}
+
+#if defined (DIRCACHE_NATIVE)
+/**
+ * scan and build the contents of a subdirectory
+ */
+static void sab_process_sub(struct sab *sabp)
+{
+    struct fat_direntry *const fatentp = get_dir_fatent();
+    struct filestr_base *const streamp = &sabp->stream;
+    struct file_base_info *const infop = &sabp->info;
+
+    int idx = infop->dcfile.idx;
+    int *downp = get_downidxp(idx);
+    if (!downp)
+        return;
+
+    while (1)
+    {
+        struct sab_component *compp = --sabp->top;
+        compp->idx   = idx;
+        compp->downp = downp;
+        compp->prevp = downp;
+
+        /* open directory stream */
+        filestr_base_init(streamp);
+        fileobj_fileop_open(streamp, infop, FO_DIRECTORY);
+        fat_rewind(&streamp->fatstr);
+        uncached_rewinddir_internal(infop);
+
+        const long dircluster = streamp->infop->fatfile.firstcluster;
+
+        /* first pass: read directory */
+        while (1)
+        {
+            if (sabp->stack + 1 < sabp->stackend)
+            {
+                /* release control and process queued events */
+                dircache_unlock();
+                process_events();
+                dircache_lock();
+
+                if (sabp->quit || !compp->idx)
+                    break;
+            }
+            /* else an immediate-contents directory scan */
+
+            int rc = uncached_readdir_internal(streamp, infop, fatentp);
+            if (rc <= 0)
+            {
+                if (rc < 0)
+                    sabp->quit = true;
+                else
+                    compp->prevp = downp; /* rewind list */
+
+                break;
+            }
+
+            struct dircache_entry *ce;
+            int prev = *compp->prevp;
+
+            if (prev)
+            {
+                /* there are entries ahead of us; they will be what was just
+                   read or something to be subsequently read; if it belongs
+                   ahead of this one, insert a new entry before it; if it's
+                   the entry just scanned, do nothing further and continue
+                   with the next */
+                ce = get_entry(prev);
+                if (ce->direntry == infop->fatfile.e.entry)
+                {
+                    compp->prevp = &ce->next;
+                    continue; /* already there */
+                }
+            }
+
+            int idx = create_entry(fatentp->name, &ce);
+            if (!idx)
+            {
+                sabp->quit = true;
+                break;
+            }
+
+            /* link it in */
+            ce->up = compp->idx;
+            ce->next = prev;
+            *compp->prevp = idx;
+            compp->prevp = &ce->next;
+
+            if (!(fatentp->attr & ATTR_DIRECTORY))
+                ce->filesize = fatentp->filesize;
+            else if (!is_dotdir_name(fatentp->name))
+                ce->frontier = FRONTIER_NEW; /* this needs scanning */
+
+            /* copy remaining FS info */
+            ce->direntry     = infop->fatfile.e.entry;
+            ce->direntries   = infop->fatfile.e.entries;
+            ce->attr         = fatentp->attr;
+            ce->firstcluster = fatentp->firstcluster;
+            ce->wrtdate      = fatentp->wrtdate;
+            ce->wrttime      = fatentp->wrttime;
+
+            /* resolve queued user bindings */
+            infop->fatfile.firstcluster = fatentp->firstcluster;
+            infop->fatfile.dircluster   = dircluster;
+            infop->dcfile.idx           = idx;
+            infop->dcfile.serialnum     = ce->serialnum;
+            binding_resolve(infop);
+        } /* end while */
+
+        close_stream_internal(streamp);
+
+        if (sabp->quit)
+            return;
+
+        establish_frontier(compp->idx, FRONTIER_SETTLED);
+
+        /* second pass: "recurse!" */
+        struct dircache_entry *ce = NULL;
+
+        while (1)
+        {
+            idx = compp->idx && compp > sabp->stack ? *compp->prevp : 0;
+            if (idx)
+            {
+                ce = get_entry(idx);
+                compp->prevp = &ce->next;
+
+                if (ce->frontier != FRONTIER_SETTLED)
+                    break;
+            }
+            else
+            {
+                /* directory completed or removed/deepest level */
+                compp = ++sabp->top;
+                if (compp >= sabp->stackend)
+                    return; /* scan completed/initial directory removed */
+            }
+        }
+
+        /* even if it got zoned from outside it is about to be scanned in
+           its entirety and may be considered new again */
+        ce->frontier = FRONTIER_NEW;
+        downp = &ce->down;
+
+        /* set up info for next open
+         * IF_MV: "volume" was set when scan began */
+        infop->fatfile.firstcluster = ce->firstcluster;
+        infop->fatfile.dircluster   = dircluster;
+        infop->fatfile.e.entry      = ce->direntry;
+        infop->fatfile.e.entries    = ce->direntries;
+    } /* end while */
+}
+
+/**
+ * scan and build the contents of a directory or volume root
+ */
+static void sab_process_dir(struct file_base_info *infop, bool issab)
+{
+    /* infop should have been fully opened meaning that all its parent
+       directory information is filled in and intact; the binding information
+       should also filled in beforehand */
+
+    /* allocate the stack right now to the max demand */
+    struct dirsab
+    {
+        struct sab           sab;
+        struct sab_component stack[issab ? DIRCACHE_MAX_DEPTH : 1];
+    } dirsab;
+    struct sab *sabp = &dirsab.sab;
+
+    sabp->quit     = false;
+    sabp->stackend = &sabp->stack[ARRAYLEN(dirsab.stack)];
+    sabp->top      = sabp->stackend;
+    sabp->info     = *infop;
+
+    if (issab)
+        DCRIVOL(infop)->sabp = sabp;
+
+    establish_frontier(infop->dcfile.idx, FRONTIER_NEW | FRONTIER_RENEW);
+    sab_process_sub(sabp);
+
+    if (issab)
+        DCRIVOL(infop)->sabp = NULL;
+}
+
+/**
+ * scan and build the entire tree for a volume
+ */
+static void sab_process_volume(struct dircache_volume *dcvolp)
+{
+    int rc;
+
+    int volume = IF_MV_VOL(dcvolp - dircache.dcvol);
+    int idx = -volume - 1;
+
+    logf("dircache - building volume %d", volume);
+
+    /* gather everything sab_process_dir() needs in order to begin a scan */
+    struct file_base_info info;
+    rc = fat_open_rootdir(IF_MV(volume,) &info.fatfile);
+    if (rc < 0)
+    {
+        /* probably not mounted */
+        logf("SAB - no root %d: %d", volume, rc);
+        establish_frontier(idx, FRONTIER_NEW);
+        return;
     }
-    
-    /* first pass : read dir */
-    struct dircache_entry *first_ce = ce;
-    
-    /* read through directory */
-    while((rc = fat_getnext(sab.dir, sab.direntry)) >= 0 && sab.direntry->name[0])
+
+    info.dcfile.idx       = idx;
+    info.dcfile.serialnum = dcvolp->serialnum;
+    sab_process_dir(&info, true);
+}
+
+/**
+ * this function is the back end to the public API's like readdir()
+ */
+int dircache_readdir_dirent(struct filestr_base *stream,
+                            struct dirscan_info *scanp,
+                            struct dirent *entry)
+{
+    struct file_base_info *dirinfop = stream->infop;
+
+    if (!dirinfop->dcfile.serialnum)
+        goto read_uncached; /* no parent cached => no entries cached */
+
+    struct dircache_volume *dcvolp = DCVOL(dirinfop);
+
+    int diridx = dirinfop->dcfile.idx;
+    unsigned int frontier = diridx <= 0 ? dcvolp->frontier :
+                                          get_entry(diridx)->frontier;
+
+    /* if not settled, just readthrough; no binding information is needed for
+       this; if it becomes settled, we'll get scan->dcfile caught up and do
+       subsequent reads with the cache */
+    if (frontier != FRONTIER_SETTLED)
+        goto read_uncached;
+
+    int idx = scanp->dcscan.idx;
+    struct dircache_entry *ce;
+
+    unsigned int direntry = scanp->fatscan.entry;
+    while (1)
     {
-        if(!strcmp(".", sab.direntry->name) ||
-                !strcmp("..", sab.direntry->name))
-            continue;
+        if (idx == 0 || direntry == FAT_RW_VAL) /* rewound? */
+        {
+            idx = diridx <= 0 ? dcvolp->root_down : get_entry(diridx)->down;
+            break;
+        }
 
-        size_t size = strlen(sab.direntry->name) + 1;
-        ce->d_name = (d_names_start -= size);
-        ce->startcluster = sab.direntry->firstcluster;
-        ce->info.size = sab.direntry->filesize;
-        ce->info.attribute = sab.direntry->attr;
-        ce->info.wrtdate = sab.direntry->wrtdate;
-        ce->info.wrttime = sab.direntry->wrttime;
+        /* validate last entry scanned; it might have been replaced between
+           calls or not there at all any more; if so, start the cache reader
+           at the beginning and fast-forward to the correct point as indicated
+           by the FS scanner */
+        ce = get_entry(idx);
+        if (ce && ce->serialnum == scanp->dcscan.serialnum)
+        {
+            idx = ce->next;
+            break;
+        }
 
-        strcpy(ce->d_name, sab.direntry->name); 
-        dircache_size += size;
-        
-        if(ce->info.attribute & FAT_ATTR_DIRECTORY)
-            dircache_gen_down(ce);
-                
-        ce = dircache_gen_next(ce);
-        if(ce == NULL)
-            return -5;
-        
-        /* When simulator is used, it's only safe to yield here. */
-        if(thread_enabled)
+        idx = 0;
+    }
+
+    while (1)
+    {
+        ce = get_entry(idx);
+        if (!ce)
         {
-            /* Stop if we got an external signal. */
-            if(check_event_queue())
-                return -6;
-            yield();
+            empty_dirent(entry);
+            scanp->fatscan.entries = 0;
+            return 0; /* end of dir */
         }
+
+        if (ce->direntry > direntry || direntry == FAT_RW_VAL)
+            break; /* cache reader is caught up to FS scan */
+
+        idx = ce->next;
     }
-    
-    /* add "." and ".." */
-    ce->d_name = dot;
-    ce->info.attribute = FAT_ATTR_DIRECTORY;
-    ce->startcluster = startcluster;
-    ce->info.size = 0;
-    ce->down = first_ce;
-    
-    ce = dircache_gen_next(ce);
-    
-    ce->d_name = dotdot;
-    ce->info.attribute = FAT_ATTR_DIRECTORY;
-    ce->startcluster = (first_ce->up ? first_ce->up->startcluster : 0);
-    ce->info.size = 0;
-    ce->down = first_ce->up;
-    
-    /* second pass: recurse ! */
-    ce = first_ce;
-    
-    while(rc >= 0 && ce)
-    {
-        if(ce->d_name != NULL && ce->down != NULL && strcmp(ce->d_name, ".")
-                && strcmp(ce->d_name, ".."))
-            rc = sab_process_dir(ce->startcluster, ce->down);
-        
-        ce = ce->next;
-    }
-    
+
+    /* basic dirent information */
+    entry_name_copy(entry->d_name, ce);
+    entry->info.attr    = ce->attr;
+    entry->info.size    = (ce->attr & ATTR_DIRECTORY) ? 0 : ce->filesize;
+    entry->info.wrtdate = ce->wrtdate;
+    entry->info.wrttime = ce->wrttime;
+
+    /* FS scan information */
+    scanp->fatscan.entry    = ce->direntry;
+    scanp->fatscan.entries  = ce->direntries;
+
+    /* dircache scan information */
+    scanp->dcscan.idx       = idx;
+    scanp->dcscan.serialnum = ce->serialnum;
+
+    /* return whether this needs decoding */
+    int rc = ce->direntries == 1 ? 2 : 1;
+
+    yield();
     return rc;
+
+read_uncached:
+    dircache_dcfile_init(&scanp->dcscan);
+    return uncached_readdir_dirent(stream, scanp, entry);
 }
 
-/* used during the generation */
-static struct fat_dir sab_fat_dir;
+/**
+ * rewind the directory scan cursor
+ */
+void dircache_rewinddir_dirent(struct dirscan_info *scanp)
+{
+    uncached_rewinddir_dirent(scanp);
+    dircache_dcfile_init(&scanp->dcscan);
+}
 
-static int dircache_scan_and_build(IF_MV(int volume,) struct dircache_entry *ce)
+/**
+ * this function is the back end to file API internal scanning, which requires
+ * much more detail about the directory entries; this is allowed to make
+ * assumptions about cache state because the cache will not be altered during
+ * the scan process; an additional important property of internal scanning is
+ * that any available binding information is not ignored even when a scan
+ * directory is frontier zoned.
+ */
+int dircache_readdir_internal(struct filestr_base *stream,
+                              struct file_base_info *infop,
+                              struct fat_direntry *fatent)
 {
-    memset(ce, 0, sizeof(struct dircache_entry));
+    /* call with writer exclusion */
+    struct file_base_info *dirinfop = stream->infop;
+    struct dircache_volume *dcvolp = DCVOL(dirinfop);
 
-#ifdef HAVE_MULTIVOLUME
-    if (volume > 0)
-    {
-        /* broken for 100+ volumes because the format string is too small
-         * and we use that for size calculation */
-        const size_t max_len = VOL_ENUM_POS + 3;
-        ce->d_name = (d_names_start -= max_len);
-        snprintf(ce->d_name, max_len, VOL_NAMES, volume);
-        dircache_size += max_len;
-        ce->info.attribute = FAT_ATTR_DIRECTORY | FAT_ATTR_VOLUME;
-        ce->info.size = 0;
-        append_position = dircache_gen_next(ce);
-        ce = dircache_gen_down(ce);
+    /* assume binding "not found" */
+    infop->dcfile.serialnum = 0;
+
+    /* is parent cached? if not, readthrough because nothing is here yet */
+    if (!dirinfop->dcfile.serialnum)
+        return uncached_readdir_internal(stream, infop, fatent);
+
+    int diridx = dirinfop->dcfile.idx;
+    unsigned int frontier = diridx < 0 ?
+        dcvolp->frontier : get_entry(diridx)->frontier;
+
+    int idx = infop->dcfile.idx;
+    if (idx == 0) /* rewound? */
+        idx = diridx <= 0 ? dcvolp->root_down : get_entry(diridx)->down;
+    else
+        idx = get_entry(idx)->next;
+
+    struct dircache_entry *ce = get_entry(idx);
+    if (frontier != FRONTIER_SETTLED)
+    {
+        /* the directory being read is reported to be incompletely cached;
+           readthrough and if the entry exists, return it with its binding
+           information; otherwise return the uncached read result while
+           maintaining the last index */
+        int rc = uncached_readdir_internal(stream, infop, fatent);
+        if (rc <= 0 || !ce || ce->direntry > infop->fatfile.e.entry)
+            return rc;
+
+        /* entry matches next one to read */
+    }
+    else if (!ce)
+    {
+        /* end of dir */
+        fat_empty_fat_direntry(fatent);
+        infop->fatfile.e.entries = 0;
+        return 0;
     }
-#endif
 
-    struct fat_direntry direntry; /* ditto */
-#ifdef HAVE_MULTIVOLUME
-    sab.volume = volume;
-#endif
-    sab.dir = &sab_fat_dir;
-    sab.direntry = &direntry;
-    
-    return sab_process_dir(0, ce);
+    /* FS entry information that we maintain */
+    entry_name_copy(fatent->name, ce);
+    fatent->shortname[0]     = '\0';
+    fatent->attr             = ce->attr;
+    /* file code file scanning does not need time information */
+    fatent->filesize         = (ce->attr & ATTR_DIRECTORY) ? 0 : ce->filesize;
+    fatent->firstcluster     = ce->firstcluster;
+
+    /* FS entry directory information */
+    infop->fatfile.e.entry   = ce->direntry;
+    infop->fatfile.e.entries = ce->direntries;
+
+    /* dircache file binding information */
+    infop->dcfile.idx        = idx;
+    infop->dcfile.serialnum  = ce->serialnum;
+
+    /* return whether this needs decoding */
+    int rc = ce->direntries == 1 ? 2 : 1;
+
+    if (frontier == FRONTIER_SETTLED)
+    {
+        static long next_yield;
+        if (TIME_AFTER(current_tick, next_yield))
+        {
+            yield();
+            next_yield = current_tick + HZ/50;
+        }
+    }
+
+    return rc;
 }
-#elif (CONFIG_PLATFORM & PLATFORM_HOSTED) /* PLATFORM_HOSTED */
+
+/**
+ * rewind the scan position for an internal scan
+ */
+void dircache_rewinddir_internal(struct file_base_info *infop)
+{
+    uncached_rewinddir_internal(infop);
+    dircache_dcfile_init(&infop->dcfile);
+}
+
+#else /* !DIRCACHE_NATIVE (for all others) */
+
+#####################
+/* we require access to the host functions */
+#undef opendir
+#undef readdir
+#undef closedir
+#undef rewinddir
+
 static char sab_path[MAX_PATH];
 
 static int sab_process_dir(struct dircache_entry *ce)
 {
     struct dirent_uncached *entry;
     struct dircache_entry *first_ce = ce;
-    DIR_UNCACHED *dir = opendir_uncached(sab_path);
+    DIR *dir = opendir(sab_path);
     if(dir == NULL)
     {
         logf("Failed to opendir_uncached(%s)", sab_path);
         return -1;
     }
-    
-    while((entry = readdir_uncached(dir)))
+
+    while (1)
     {
-        if(!strcmp(".", entry->d_name) ||
-                !strcmp("..", entry->d_name))
+        if (!(entry = readdir(dir)))
+            break;
+
+        if (IS_DOTDIR_NAME(entry->d_name))
+        {
+            /* add "." and ".." */
+            ce->info.attribute = ATTR_DIRECTORY;
+            ce->info.size = 0;
+            ce->down = entry->d_name[1] == '\0' ? first_ce : first_ce->up;
+            strcpy(ce->dot_d_name, entry->d_name);
             continue;
+        }
 
         size_t size = strlen(entry->d_name) + 1;
         ce->d_name = (d_names_start -= size);
@@ -436,10 +1666,10 @@ static int sab_process_dir(struct dircache_entry *ce)
 
         strcpy(ce->d_name, entry->d_name);
         dircache_size += size;
-        
+
         if(entry->info.attribute & ATTR_DIRECTORY)
         {
-            dircache_gen_down(ce);
+            dircache_gen_down(ce, ce);
             if(ce->down == NULL)
             {
                 closedir_uncached(dir);
@@ -450,1189 +1680,1504 @@ static int sab_process_dir(struct dircache_entry *ce)
             /* append entry */
             strlcpy(&sab_path[pathpos], "/", sizeof(sab_path) - pathpos);
             strlcpy(&sab_path[pathpos+1], entry->d_name, sizeof(sab_path) - pathpos - 1);
-            
+
             int rc = sab_process_dir(ce->down);
             /* restore path */
             sab_path[pathpos] = '\0';
-            
+
             if(rc < 0)
             {
                 closedir_uncached(dir);
                 return rc;
             }
         }
-        
-        ce = dircache_gen_next(ce);
-        if(ce == NULL)
+
+        ce = dircache_gen_entry(ce);
+        if (ce == NULL)
             return -5;
-        
-        /* When simulator is used, it's only safe to yield here. */
-        if(thread_enabled)
-        {
-            /* Stop if we got an external signal. */
-            if(check_event_queue())
-                return -1;
-            yield();
-        }
+
+        yield();
     }
-    
-    /* add "." and ".." */
-    ce->d_name = dot;
-    ce->info.attribute = ATTR_DIRECTORY;
-    ce->info.size = 0;
-    ce->down = first_ce;
-    
-    ce = dircache_gen_next(ce);
-    
-    ce->d_name = dotdot;
-    ce->info.attribute = ATTR_DIRECTORY;
-    ce->info.size = 0;
-    ce->down = first_ce->up;
-    
+
     closedir_uncached(dir);
-    return 0;
+    return 1;
 }
 
-static int dircache_scan_and_build(IF_MV(int volume,) struct dircache_entry *ce)
+static int sab_process_volume(IF_MV(int volume,) struct dircache_entry *ce)
 {
-    #ifdef HAVE_MULTIVOLUME
-    (void) volume;
-    #endif
     memset(ce, 0, sizeof(struct dircache_entry));
-    
     strlcpy(sab_path, "/", sizeof sab_path);
     return sab_process_dir(ce);
 }
-#endif /* PLATFORM_NATIVE */
 
-/**
- * Internal function to get a pointer to dircache_entry for a given filename.
- *   path: Absolute path to a file or directory (see comment)
- *   go_down: Returns the first entry of the directory given by the path (see comment)
- *
- * As a a special case, accept path="" as an alias for "/".
- * Also if the path omits the first '/', it will be accepted.
- *
- * * If get_down=true:
- *   If path="/", the returned entry is the first of root directory (ie dircache_root)
- *   Otherwise, if 'entry' is the returned value when get_down=false, 
- *   the functions returns entry->down (which can be NULL)
- *
- * * If get_down=false:
- *   If path="/chunk_1/chunk_2/.../chunk_n" then this functions returns the entry
- *   root_entry()->chunk_1->chunk_2->...->chunk_(n-1)
- *   Which means that
- *   dircache_get_entry(path)->d_name == chunk_n
- *
- *   If path="/", the returned entry is NULL.
- *   If the entry doesn't exist, return NULL
- *
- *  NOTE: this functions silently handles double '/'
- */
-static struct dircache_entry* dircache_get_entry(const char *path, bool go_down)
-{
-    char namecopy[MAX_PATH];
-    char* part;
-    char* end;
-    
-    bool at_root = true;
-    struct dircache_entry *cache_entry = dircache_root;
-    
-    strlcpy(namecopy, path, sizeof(namecopy));
-    
-    for(part = strtok_r(namecopy, "/", &end); part; part = strtok_r(NULL, "/", &end))
-    {
-        /* If request another chunk, the current entry has to be directory
-         * and so cache_entry->down has to be non-NULL/
-         * Special case of root because it's already the first entry of the root directory
-         *
-         * NOTE: this is safe even if cache_entry->down is NULL */
-        if(!at_root)
-            cache_entry = cache_entry->down;
-        else
-            at_root = false;
-        
-        /* scan dir for name */
-        while(cache_entry != NULL)
-        {
-            /* skip unused entries */
-            if(cache_entry->d_name == NULL)
-            {
-                cache_entry = cache_entry->next;
-                continue;
-            }
-            /* compare names */
-            if(!strcasecmp(part, cache_entry->d_name))
-                break;
-            /* go to next entry */
-            cache_entry = cache_entry->next;
-        }
-        
-        /* handle not found case */
-        if(cache_entry == NULL)
-            return NULL;
-    }
+int dircache_readdir_r(struct dircache_dirscan *dir, struct dirent *result)
+{
+    if (dircache_state != DIRCACHE_READY)
+        return readdir_r(dir->###########3, result, &result);
 
-    /* NOTE: here cache_entry!=NULL so taking ->down is safe */
-    if(go_down)
-        return at_root ? cache_entry : cache_entry->down;
-    else
-        return at_root ? NULL : cache_entry;
-}
+    bool first = dir->dcinfo.scanidx == REWIND_INDEX;
+    struct dircache_entry *ce = get_entry(first ? dir->dcinfo.index :
+                                                  dir->dcinfo.scanidx);
 
-#ifdef HAVE_EEPROM_SETTINGS
+    ce = first ? ce->down : ce->next;
 
-#define DIRCACHE_MAGIC  0x00d0c0a1
-struct dircache_maindata {
-    long magic;
-    long size;
-    long entry_count;
-    long appflags;
-    struct dircache_entry *root_entry;
-    char *d_names_start;
-};
+    if (ce == NULL)
+        return 0;
+
+    dir->scanidx = ce - dircache_root;
+
+    strlcpy(result->d_name, ce->d_name, sizeof (result->d_name));
+    result->info = ce->dirinfo;
+
+    return 1;
+}
+
+#endif /* DIRCACHE_* */
 
 /**
- * Function to load the internal cache structure from disk to initialize
- * the dircache really fast and little disk access.
+ * reset the cache for the specified volume
  */
-int dircache_load(void)
+static void reset_volume(IF_MV_NONVOID(int volume))
 {
-    struct dircache_maindata maindata;
-    ssize_t bytes_read;
-    int fd;
-        
-    if (dircache_initialized)
-        return -1;
-        
-    logf("Loading directory cache");
-    dircache_size = 0;
-    
-    fd = open_dircache_file(O_RDONLY, 0);
-    if (fd < 0)
-        return -2;
-        
-    bytes_read = read(fd, &maindata, sizeof(struct dircache_maindata));
-    if (bytes_read != sizeof(struct dircache_maindata)
-        || maindata.magic != DIRCACHE_MAGIC || maindata.size <= 0)
+    FOR_EACH_VOLUME(volume, i)
     {
-        logf("Dircache file header error");
-        close(fd);
-        remove_dircache_file();
-        return -3;
-    }
-    
-    allocated_size = maindata.size + DIRCACHE_RESERVE;
-    dircache_handle = core_alloc_ex("dircache", allocated_size, &ops);
-    /* block movement during upcoming I/O */
-    dont_move = true;
-    dircache_root = core_get_data(dircache_handle);
-    ALIGN_BUFFER(dircache_root, allocated_size, sizeof(struct dircache_entry*));
-    entry_count = maindata.entry_count;
-    appflags = maindata.appflags;
-
-    /* read the dircache file into memory,
-     * start with the struct dircache_entries */
-    ssize_t bytes_to_read = entry_count*sizeof(struct dircache_entry);
-    bytes_read = read(fd, dircache_root, bytes_to_read);
-    
-    if (bytes_read != bytes_to_read)
-    {
-        logf("Dircache read failed #1");
-        return -6;
-    }
-
-    /* continue with the d_names. Fix up pointers to them if needed */
-    bytes_to_read = maindata.size - bytes_to_read;
-    d_names_start = (char*)dircache_root + allocated_size - bytes_to_read;
-    bytes_read = read(fd, d_names_start, bytes_to_read);
-    close(fd);
-    remove_dircache_file();
-    if (bytes_read != bytes_to_read)
-    {
-        logf("Dircache read failed #2");
-        return -7;
+        struct dircache_volume *dcvolp = DCVOL(i);
+
+        if (dcvolp->status == DIRCACHE_IDLE)
+            continue; /* idle => nothing happening there */
+
+        struct dircache_runinfo_volume *dcrivolp = DCRIVOL(i);
+
+        /* stop any scan and build on this one */
+        if (dcrivolp->sabp)
+            dcrivolp->sabp->quit = true;
+
+    #ifdef HAVE_MULTIVOLUME
+        /* if this call is for all volumes, subsequent code will just reset
+           the cache memory usage and the freeing of individual entries may
+           be skipped */
+        if (volume >= 0)
+            free_subentries(NULL, &dcvolp->root_down);
+        else
+    #endif
+            binding_dissolve_volume(dcrivolp);
+
+        /* set it back to unscanned */
+        dcvolp->status      = DIRCACHE_IDLE;
+        dcvolp->frontier    = FRONTIER_NEW;
+        dcvolp->root_down   = 0;
+        dcvolp->build_ticks = 0;
+        dcvolp->serialnum   = 0;
     }
+}
 
-    d_names_end = d_names_start + bytes_read;
-    dot = d_names_end - sizeof(".");
-    dotdot = dot - sizeof("..");
+/**
+ * reset the entire cache state for all volumes
+ */
+static void reset_cache(void)
+{
+    if (!dircache_runinfo.handle)
+        return; /* no buffer => nothing cached */
+
+    /* blast all the volumes */
+    reset_volume(IF_MV(-1));
+
+#ifdef DIRCACHE_DUMPSTER
+    dumpster_clean_buffer(dircache_runinfo.p + ENTRYSIZE,
+                          dircache_runinfo.bufsize);
+#endif /* DIRCACHE_DUMPSTER */
+
+    /* reset the memory */
+    dircache.free_list    = 0;
+    dircache.size         = 0;
+    dircache.sizeused     = 0;
+    dircache.numentries   = 0;
+    dircache.names        = dircache_runinfo.bufsize + ENTRYSIZE;
+    dircache.sizenames    = 0;
+    dircache.namesfree    = 0;
+    dircache.nextnamefree = 0;
+    *get_name(dircache.names - 1) = 0;
+    /* dircache.last_serialnum stays */
+    /* dircache.reserve_used stays */
+    /* dircache.last_size stays */
+}
 
-    /* d_names are in reverse order, so the last entry points to the first string */
-    ptrdiff_t offset_d_names = maindata.d_names_start - d_names_start;
-    ptrdiff_t offset_entries = maindata.root_entry - dircache_root;
-    offset_entries *= sizeof(struct dircache_entry); /* make it bytes */
+/**
+ * checks each "idle" volume and builds it
+ */
+static void build_volumes(void)
+{
+    buffer_lock();
 
-    /* offset_entries is less likely to differ, so check if it's 0 in the loop
-     * offset_d_names however is almost always non-zero, since dircache_save()
-     * creates a file which causes the reserve buffer to be used. since
-     * we allocate a new, empty DIRCACHE_RESERVE here, the strings are
-     * farther behind */
-    if (offset_entries != 0 || offset_d_names != 0)
+    for (int i = 0; i < NUM_VOLUMES; i++)
     {
-        for(unsigned i = 0; i < entry_count; i++)
-        {
-            if (dircache_root[i].d_name)
-                dircache_root[i].d_name -= offset_d_names;
+        /* this does reader locking but we already own that */
+        if (!volume_ismounted(IF_MV(i)))
+            continue;
 
-            if (offset_entries == 0)
-                continue;
-            if (dircache_root[i].next_char)
-                dircache_root[i].next_char -= offset_entries;
-            if (dircache_root[i].up_char)
-                dircache_root[i].up_char -= offset_entries;
-            if (dircache_root[i].down_char)
-                dircache_root[i].down_char -= offset_entries;
-        }
+        struct dircache_volume *dcvolp = DCVOL(i);
+
+        /* can't already be "scanning" because that's us; doesn't retry
+           "ready" volumes */
+        if (dcvolp->status == DIRCACHE_READY)
+            continue;
+
+        /* measure how long it takes to build the cache for each volume */
+        if (!dcvolp->serialnum)
+            dcvolp->serialnum = next_serialnum();
+
+        dcvolp->status = DIRCACHE_SCANNING;
+        dcvolp->start_tick = current_tick;
+
+        sab_process_volume(dcvolp);
+
+        if (dircache_runinfo.suspended)
+            break;
+
+        /* whatever happened, it's ready unless reset */
+        dcvolp->build_ticks = current_tick - dcvolp->start_tick;
+        dcvolp->status = DIRCACHE_READY;
     }
 
-    /* Cache successfully loaded. */
-    dircache_size = maindata.size;
-    reserve_used = 0;
-    logf("Done, %ld KiB used", dircache_size / 1024);
-    dircache_initialized = true;
-    memset(fd_bindings, 0, sizeof(fd_bindings));
-    dont_move = false;
+    size_t reserve_used = reserve_buf_used();
+    if (reserve_used > dircache.reserve_used)
+        dircache.reserve_used = reserve_used;
+
+    if (DIRCACHE_STUFFED(reserve_used))
+        dircache.last_size = 0; /* reset */
+    else if (dircache.size > dircache.last_size ||
+        dircache.last_size - dircache.size > DIRCACHE_RESERVE)
+        dircache.last_size = dircache.size;
+
+    logf("Done, %ld KiB used", dircache.size / 1024);
 
-    return 0;
+    buffer_unlock();
 }
 
 /**
- * Function to save the internal cache stucture to disk for fast loading
- * on boot.
+ * allocate buffer and return whether or not a synchronous build should take
+ * place; if 'realloced' is NULL, it's just a query about what will happen
  */
-int dircache_save(void)
+static int prepare_build(bool *realloced)
 {
-    struct dircache_maindata maindata;
-    int fd;
-    unsigned long bytes_written;
+    /* called holding dircache lock */
+    size_t size = dircache.last_size;
 
-    remove_dircache_file();
-    
-    if (!dircache_initialized)
-        return -1;
+#ifdef HAVE_EEPROM_SETTINGS
+    if (realloced)
+    {
+        dircache_unlock();
+        remove_dircache_file();
+        dircache_lock();
 
-    logf("Saving directory cache");
-    dont_move = true;
-    fd = open_dircache_file(O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        if (dircache_runinfo.suspended)
+            return -1;
+    }
+#endif /* HAVE_EEPROM_SETTINGS */
+
+    bool stuffed = DIRCACHE_STUFFED(dircache.reserve_used);
+    if (dircache_runinfo.bufsize > size && !stuffed)
+    {
+        if (realloced)
+            *realloced = false;
 
-    maindata.magic = DIRCACHE_MAGIC;
-    maindata.size = dircache_size;
-    maindata.root_entry = dircache_root;
-    maindata.d_names_start = d_names_start;
-    maindata.entry_count = entry_count;
-    maindata.appflags = appflags;
+        return 0; /* start a transparent rebuild */
+    }
 
-    /* Save the info structure */
-    bytes_written = write(fd, &maindata, sizeof(struct dircache_maindata));
-    if (bytes_written != sizeof(struct dircache_maindata))
+    int syncbuild = size > 0 && !stuffed ? 0 : 1;
+
+    if (!realloced)
+        return syncbuild;
+
+    if (syncbuild)
     {
-        close(fd);
-        logf("dircache: write failed #1");
-        return -2;
+        /* start a non-transparent rebuild */
+        /* we'll use the entire audiobuf to allocate the dircache */
+        size = audio_buffer_available() + dircache_runinfo.bufsize;
+        /* try to allocate at least the min and no more than the limit */
+        size = MAX(DIRCACHE_MIN, MIN(size, DIRCACHE_LIMIT));
     }
+    else
+    {
+        /* start a transparent rebuild */
+        size = MAX(size, DIRCACHE_RESERVE) + DIRCACHE_RESERVE*2;
+    }
+
+    *realloced = true;
+    reset_cache();
 
-    /* Dump whole directory cache to disk
-     * start by writing the dircache_entries */
-    size_t bytes_to_write = entry_count*sizeof(struct dircache_entry);
-    bytes_written = write(fd, dircache_root, bytes_to_write);
-    if (bytes_written != bytes_to_write)
+    buffer_lock();
+
+    int handle = reset_buffer();
+    dircache_unlock();
+
+    if (handle > 0)
+        core_free(handle);
+
+    handle = alloc_cache(size);
+
+    dircache_lock();
+
+    if (dircache_runinfo.suspended && handle > 0)
     {
-        logf("dircache: write failed #2");
-        return -3;
+        /* if we got suspended, don't keep this huge buffer around */
+        dircache_unlock();
+        core_free(handle);
+        handle = 0;
+        dircache_lock();
     }
 
-    /* continue with the d_names */
-    bytes_to_write = d_names_end - d_names_start;
-    bytes_written = write(fd, d_names_start, bytes_to_write);
-    close(fd);
-    if (bytes_written != bytes_to_write)
+    if (handle <= 0)
     {
-        logf("dircache: write failed #3");
-        return -4;
+        buffer_unlock();
+        return -1;
     }
 
-    dont_move = false;
-    return 0;
+    set_buffer(handle, size);
+    buffer_unlock();
+
+    return syncbuild;
 }
-#endif /* HAVE_EEPROM_SETTINGS */
 
 /**
- * Internal function which scans the disk and creates the dircache structure.
+ * compact the dircache buffer after a successful full build
  */
-static int dircache_do_rebuild(void)
+static void compact_cache(void)
 {
-    struct dircache_entry* root_entry;
-    unsigned int start_tick;
-    int i;
-    
-    /* Measure how long it takes build the cache. */
-    start_tick = current_tick;
-    dircache_initializing = true;
-    appflags = 0;
+    /* called holding dircache lock */
+    if (dircache_runinfo.suspended)
+        return;
 
-    /* reset dircache and alloc root entry */
-    entry_count = 0;
-    root_entry = allocate_entry();
-    dont_move = true;
+    void *p = dircache_runinfo.p + ENTRYSIZE;
+    size_t leadsize = dircache.numentries * ENTRYSIZE + DIRCACHE_RESERVE;
 
-#ifdef HAVE_MULTIVOLUME
-    append_position = root_entry;
+    void *dst = p + leadsize;
+    void *src = get_name(dircache.names - 1);
+    if (dst >= src)
+        return; /* cache got bigger than expected; never mind that */
+
+    /* slide the names up in memory */
+    memmove(dst, src, dircache.sizenames + 2);
 
-    for (i = NUM_VOLUMES; i >= 0; i--)
+    /* fix up name indexes */
+    ptrdiff_t offset = dst - src;
+
+    FOR_EACH_CACHE_ENTRY(ce)
     {
-        if (fat_ismounted(i))
-        {
-#endif
-            cpu_boost(true);
-#ifdef HAVE_MULTIVOLUME
-            if (dircache_scan_and_build(IF_MV(i,) append_position) < 0)
-#else
-            if (dircache_scan_and_build(IF_MV(0,) root_entry) < 0)
-#endif /* HAVE_MULTIVOLUME */
-            {
-                logf("dircache_scan_and_build failed");
-                cpu_boost(false);
-                dircache_size = 0;
-                dircache_initializing = false;
-                dont_move = false;
-                return -2;
-            }
-            cpu_boost(false);
-#ifdef HAVE_MULTIVOLUME
-        }
+        if (!ce->tinyname)
+             ce->name += offset;
     }
-#endif
 
-    logf("Done, %ld KiB used", dircache_size / 1024);
-    
-    dircache_initialized = true;
-    dircache_initializing = false;
-    cache_build_ticks = current_tick - start_tick;
-    
-    /* Initialized fd bindings. */
-    memset(fd_bindings, 0, sizeof(fd_bindings));
-    for (i = 0; i < fdbind_idx; i++)
-        dircache_bind(fdbind_cache[i].fd, fdbind_cache[i].path);
-    fdbind_idx = 0;
-    
-    if (thread_enabled)
-    {
-        if (allocated_size - dircache_size < DIRCACHE_RESERVE)
-            reserve_used = DIRCACHE_RESERVE - (allocated_size - dircache_size);
-    }
-
-    dont_move = false;
-    return 1;
-}
+    dircache.names += offset;
 
-/*
- * Free all associated resources, if any */
-static void dircache_free(void)
-{
-    if (dircache_handle > 0)
-        dircache_handle = core_free(dircache_handle);
-    dircache_size = allocated_size = 0;
+    /* assumes beelzelib doesn't do things like calling callbacks or changing
+       the pointer as a result of the shrink operation; it doesn't as of now
+       but if it ever does that may very well cause deadlock problems since
+       we're holding filesystem locks */
+    size_t newsize = leadsize + dircache.sizenames + 1;
+    core_shrink(dircache_runinfo.handle, p, newsize + 1);
+    dircache_runinfo.bufsize = newsize;
+    dircache.reserve_used = 0;
 }
 
 /**
- * Internal thread that controls transparent cache building.
+ * internal thread that controls cache building; exits when no more requests
+ * are pending or the cache is suspended
  */
 static void dircache_thread(void)
 {
     struct queue_event ev;
 
+    /* calls made within the loop reopen the lock */
+    dircache_lock();
+
     while (1)
     {
-        queue_wait(&dircache_queue, &ev);
-
-        switch (ev.id)
+        queue_wait_w_tmo(&dircache_queue, &ev, 0);
+        if (ev.id == SYS_TIMEOUT || dircache_runinfo.suspended)
         {
-#ifdef HAVE_HOTSWAP
-            case SYS_FS_CHANGED:
-                if (!dircache_initialized)
-                    break;
-                dircache_initialized = false;
-#endif
-            case DIRCACHE_BUILD:
-                thread_enabled = true;
-                if (dircache_do_rebuild() < 0)
-                    dircache_free();
-                thread_enabled = false;
-                break ;
-                
-            case DIRCACHE_STOP:
-                logf("Stopped the rebuilding.");
-                dircache_initialized = false;
-                break ;
-            
-            case SYS_USB_CONNECTED:
-                usb_acknowledge(SYS_USB_CONNECTED_ACK);
-                usb_wait_for_disconnect(&dircache_queue);
-                break ;
+            /* nothing left to do/suspended */
+            if (dircache_runinfo.suspended)
+                clear_dircache_queue();
+            dircache_runinfo.thread_done = true;
+            break;
         }
-    }
-}
 
-static void generate_dot_d_names(void)
-{
-    dot = (d_names_start -= sizeof("."));
-    dotdot = (d_names_start -= sizeof(".."));
-    dircache_size += sizeof(".") + sizeof("..");
-    strcpy(dot, ".");
-    strcpy(dotdot, "..");
+        /* background-only builds are not allowed if a synchronous build is
+           required first; test what needs to be done and if it checks out,
+           do it for real below */
+        int *rcp = (int *)ev.data;
+
+        if (!rcp && prepare_build(NULL) > 0)
+            continue;
+
+        bool realloced;
+        int rc = prepare_build(&realloced);
+        if (rcp)
+            *rcp = rc;
+
+        if (rc < 0)
+            continue;
+
+        trigger_cpu_boost();
+        build_volumes();
+
+        /* if it was reallocated, compact it */
+        if (realloced)
+            compact_cache();
+     }
+
+     dircache_unlock();
 }
 
 /**
- * Start scanning the disk to build the dircache.
- * Either transparent or non-transparent build method is used.
+ * post a scan and build message to the thread, starting it if required
  */
-int dircache_build(int last_size)
+static bool dircache_thread_post(int volatile *rcp)
 {
-    if (dircache_initialized || thread_enabled)
-        return -3;
+    if (dircache_runinfo.thread_done)
+    {
+        /* mustn't recreate until it exits so that the stack isn't reused */
+        thread_wait(dircache_runinfo.thread_id);
+        dircache_runinfo.thread_done = false;
+        dircache_runinfo.thread_id = create_thread(
+            dircache_thread, dircache_stack, sizeof (dircache_stack), 0,
+            dircache_thread_name IF_PRIO(, PRIORITY_BACKGROUND)
+            IF_COP(, CPU));
+    }
 
-    logf("Building directory cache");
-#ifdef HAVE_EEPROM_SETTINGS
-    remove_dircache_file();
-#endif
+    bool started = dircache_runinfo.thread_id != 0;
 
-    /* Background build, dircache has been previously allocated and */
-    if (allocated_size > MAX(last_size, 0))
-    {
-        d_names_start = d_names_end;
-        dircache_size = 0;
-        reserve_used = 0;
-        thread_enabled = true;
-        dircache_initializing = true;
-        generate_dot_d_names();
-        
-        queue_post(&dircache_queue, DIRCACHE_BUILD, 0);
-        return 2;
-    }
-
-    /* start by freeing, as we possibly re-allocate */
-    dircache_free();
-
-    if (last_size > DIRCACHE_RESERVE && last_size < DIRCACHE_LIMIT )
-    {
-        allocated_size = last_size + DIRCACHE_RESERVE;
-        dircache_handle = core_alloc_ex("dircache", allocated_size, &ops);
-        dircache_root = core_get_data(dircache_handle);
-        ALIGN_BUFFER(dircache_root, allocated_size, sizeof(struct dircache_entry*));
-        d_names_start = d_names_end = ((char*)dircache_root)+allocated_size-1;
-        dircache_size = 0;
-        thread_enabled = true;
-        generate_dot_d_names();
-
-        /* Start a transparent rebuild. */
-        queue_post(&dircache_queue, DIRCACHE_BUILD, 0);
-        return 3;
-    }
-
-    /* We'll use the entire audiobuf to allocate the dircache
-     * struct dircache_entrys are allocated from the beginning
-     * and their corresponding d_name from the end
-     * after generation the buffer will be compacted with DIRCACHE_RESERVE
-     * free bytes inbetween */
-    size_t available = audio_buffer_available();
-    /* try to allocate at least 1MB, the more the better */
-    if (available < 1<<20) available = 1<<20;
-    if (available > DIRCACHE_LIMIT) available = DIRCACHE_LIMIT;
-
-    dircache_handle = core_alloc_ex("dircache", available, &ops);
-    if (dircache_handle <= 0)
-        return -1; /* that was not successful, should try rebooting */
-    char* buf = core_get_data(dircache_handle);
-    dircache_root = (struct dircache_entry*)ALIGN_UP(buf,
-                                                sizeof(struct dircache_entry*));
-    d_names_start = d_names_end = buf + available - 1;
-    dircache_size = 0;
-    generate_dot_d_names();
-
-    /* Start a non-transparent rebuild. */
-    int res = dircache_do_rebuild();
-    if (res < 0)
-        goto fail;
-
-    /* now compact the dircache buffer */
-    char* dst = ((char*)&dircache_root[entry_count] + DIRCACHE_RESERVE);
-    ptrdiff_t offset = d_names_start - dst;
-    if (offset <= 0) /* something went wrong */
-    {
-        res = -1;
-        goto fail;
-    }
-
-    /* memmove d_names down, there's a possibility of overlap
-     * equivaent to dircache_size - entry_count*sizeof(struct dircache_entry) */
-    ptrdiff_t size_to_move = d_names_end - d_names_start;
-    memmove(dst, d_names_start, size_to_move);
-    
-    /* fix up pointers to the d_names */
-    for(unsigned i = 0; i < entry_count; i++)
-        dircache_root[i].d_name -= offset;
-
-    d_names_start -= offset;
-    d_names_end -= offset;
-    dot -= offset;
-    dotdot -= offset;
-    
-    /* equivalent to dircache_size + DIRCACHE_RESERVE + align */
-    allocated_size = (d_names_end - buf);
-    reserve_used = 0;
-
-    core_shrink(dircache_handle, dircache_root, allocated_size);
-    return res;
-fail:
-    dircache_disable();
-    return res;
-}
-
-/**
- * Main initialization function that must be called before any other
- * operations within the dircache.
- */
-void dircache_init(void)
-{
-    int i;
-    int thread_id __attribute__((unused));
-    
-    dircache_initialized = false;
-    dircache_initializing = false;
-    
-    memset(opendirs, 0, sizeof(opendirs));
-    for (i = 0; i < MAX_OPEN_DIRS; i++)
-    {
-        opendirs[i].theent.d_name = opendir_dnames[i];
-    }
-    
-    queue_init(&dircache_queue, true);
-    thread_id = create_thread(dircache_thread, dircache_stack,
-                sizeof(dircache_stack), 0, dircache_thread_name
-                IF_PRIO(, PRIORITY_BACKGROUND)
-                IF_COP(, CPU));
-#ifdef HAVE_IO_PRIORITY
-    thread_set_io_priority(thread_id,IO_PRIORITY_BACKGROUND);
-#endif
+    if (started)
+        queue_post(&dircache_queue, DCM_BUILD, (intptr_t)rcp);
 
+    return started;
 }
 
 /**
- * Returns true if dircache has been initialized and is ready to be used.
+ * wait for the dircache thread to finish building; intended for waiting for a
+ * non-transparent build to finish when dircache_resume() returns > 0
  */
-bool dircache_is_enabled(void)
+void dircache_wait(void)
 {
-    return dircache_initialized;
+    thread_wait(dircache_runinfo.thread_id);
 }
 
 /**
- * Returns true if dircache is being initialized.
+ * call after mounting a volume or all volumes
  */
-bool dircache_is_initializing(void)
+void dircache_mount(void)
 {
-    return dircache_initializing || thread_enabled;
+    /* call with writer exclusion */
+    if (dircache_runinfo.suspended)
+        return;
+
+    dircache_thread_post(NULL);
 }
 
 /**
- * Set application flags used to determine if dircache is still intact.
+ * call after unmounting a volume; specifying < 0 for all or >= 0 for the
+ * specific one
  */
-void dircache_set_appflag(long mask)
+void dircache_unmount(IF_MV_NONVOID(int volume))
 {
-    appflags |= mask;
+    /* call with writer exclusion */
+    if (dircache_runinfo.suspended)
+        return;
+
+#ifdef HAVE_MULTIVOLUME
+    if (volume >= 0)
+        reset_volume(volume);
+    else
+#endif /* HAVE_MULTIVOLUME */
+        reset_cache();
+}
+
+/* backend to dircache_suspend() and dircache_disable() */
+static void dircache_suspend_internal(bool freeit)
+{
+    if (dircache_runinfo.suspended++ > 0 &&
+        (!freeit || dircache_runinfo.handle <= 0))
+        return;
+
+    unsigned int thread_id = dircache_runinfo.thread_id;
+
+    reset_cache();
+    clear_dircache_queue();
+
+    /* grab the buffer away into our control; the cache won't need it now */
+    int handle = 0;
+    if (freeit)
+        handle = reset_buffer();
+
+    dircache_unlock();
+
+    if (handle > 0)
+        core_free(handle);
+
+    thread_wait(thread_id);
+
+    dircache_lock();
+}
+
+/* backend to dircache_resume() and dircache_enable() */
+static int dircache_resume_internal(bool build_now)
+{
+    int volatile rc = 0;
+
+    if (dircache_runinfo.suspended == 0 || --dircache_runinfo.suspended == 0)
+        rc = build_now && dircache_runinfo.enabled ? 1 : 0;
+
+    if (rc)
+        rc = dircache_thread_post(&rc) ? INT_MIN : -1;
+
+    if (rc == INT_MIN)
+    {
+        dircache_unlock();
+
+        while (rc == INT_MIN) /* poll for response */
+            sleep(0);
+
+        dircache_lock();
+    }
+
+    return rc < 0 ? rc * 10 - 2 : rc;
 }
 
 /**
- * Get application flags used to determine if dircache is still intact.
+ * service to dircache_enable() and dircache_load(); "build_now" starts a build
+ * immediately if the cache was not enabled
  */
-bool dircache_get_appflag(long mask)
+static int dircache_enable_internal(bool build_now)
 {
-    return dircache_is_enabled() && (appflags & mask);
+    int rc = 0;
+
+    if (!dircache_runinfo.enabled)
+    {
+        dircache_runinfo.enabled = true;
+        rc = dircache_resume_internal(build_now);
+    }
+
+    return rc;
 }
 
 /**
- * Returns the current number of entries (directories and files) in the cache.
+ * service to dircache_disable()
  */
-int dircache_get_entry_count(void)
+static void dircache_disable_internal(void)
 {
-    return entry_count;
+    if (dircache_runinfo.enabled)
+    {
+        dircache_runinfo.enabled = false;
+        dircache_suspend_internal(true);
+    }
 }
 
 /**
- * Returns the allocated space for dircache (without reserve space).
+ * disables dircache without freeing the buffer (so it can be re-enabled
+ * afterwards with dircache_resume(); usually called when accepting an USB
+ * connection
  */
-int dircache_get_cache_size(void)
+void dircache_suspend(void)
 {
-    return dircache_is_enabled() ? dircache_size : 0;
+    dircache_lock();
+    dircache_suspend_internal(false);
+    dircache_unlock();
 }
 
 /**
- * Returns how many bytes of the reserve allocation for live cache
- * updates have been used.
+ * re-enables the dircache if previously suspended by dircache_suspend
+ * or dircache_steal_buffer(), re-using the already allocated buffer if
+ * available
+ *
+ * returns: 0 if the background build is started or dircache is still
+ *          suspended
+ *          > 0 if the build is non-background
+ *          < 0 upon failure
  */
-int dircache_get_reserve_used(void)
+int dircache_resume(void)
 {
-    return dircache_is_enabled() ? reserve_used : 0;
+    dircache_lock();
+    int rc = dircache_resume_internal(true);
+    dircache_unlock();
+    return rc;
 }
 
 /**
- * Returns the time in kernel ticks that took to build the cache.
+ * as dircache_resume() but globally enables it; called by settings and init
  */
-int dircache_get_build_ticks(void)
+int dircache_enable(void)
 {
-    return dircache_is_enabled() ? cache_build_ticks : 0;
+    dircache_lock();
+    int rc = dircache_enable_internal(true);
+    dircache_unlock();
+    return rc;
 }
 
 /**
- * Disables dircache without freeing the buffer (so it can be re-enabled
- * afterwards with dircache_resume() or dircache_build()), usually
- * called when accepting an usb connection */
-void dircache_suspend(void)
+ * as dircache_suspend() but also frees the buffer; usually called on shutdown
+ * or when deactivated
+ */
+void dircache_disable(void)
 {
-    int i;
-    bool cache_in_use;
-    
-    if (thread_enabled)
-        queue_post(&dircache_queue, DIRCACHE_STOP, 0);
-    
-    while (thread_enabled)
-        sleep(1);
-    dircache_initialized = false;
-
-    logf("Waiting for cached dirs to release");
-    do {
-        cache_in_use = false;
-        for (i = 0; i < MAX_OPEN_DIRS; i++) {
-            if (!opendirs[i].regulardir && opendirs[i].busy)
-            {
-                cache_in_use = true;
-                sleep(1);
-                break ;
-            }
-        }
-    } while (cache_in_use) ;
-    
-    logf("Cache released");
-    entry_count = 0;
+    dircache_lock();
+    dircache_disable_internal();
+    dircache_unlock();
 }
 
 /**
- * Re-enables the dircache if previous suspended by dircache_suspend
- * or dircache_steal_buffer(), re-using the already allocated buffer
- *
- * Returns true if the background build is started, false otherwise
- * (e.g. if no buffer was previously allocated)
+ * have dircache give up its allocation; call dircache_resume() to restart it
  */
-bool dircache_resume(void)
+void dircache_free_buffer(void)
 {
-    bool ret = allocated_size > 0;
-    if (ret) /* only resume if already allocated */
-        ret = (dircache_build(0) > 0);
+    dircache_lock();
+    dircache_suspend_internal(true);
+    dircache_unlock();
+}
+
+
+/** Dircache live updating **/
 
-    return (allocated_size > 0);
+/**
+ * obtain binding information for the file's root volume; this is the starting
+ * point for internal path parsing and binding
+ */
+void dircache_get_rootinfo(struct file_base_info *infop)
+{
+    int volume = BASEINFO_VOL(infop);
+    struct dircache_volume *dcvolp = DCVOL(volume);
+
+    if (dcvolp->serialnum)
+    {
+        /* root has a binding */
+        infop->dcfile.idx       = -volume - 1;
+        infop->dcfile.serialnum = dcvolp->serialnum;
+    }
+    else
+    {
+        /* root is idle */
+        dircache_dcfile_init(&infop->dcfile);
+    }
 }
 
 /**
- * Disables the dircache entirely. Usually called on shutdown or when
- * deactivated
+ * called by file code when the first reference to a file or directory is
+ * opened
  */
-void dircache_disable(void)
+void dircache_bind_file(struct file_base_binding *bindp)
+{
+    /* requires write exclusion */
+    logf("dc open: %u", (unsigned int)bindp->info.dcfile.serialnum);
+    binding_open(bindp);
+}
+
+/**
+ * called by file code when the last reference to a file or directory is
+ * closed
+ */
+void dircache_unbind_file(struct file_base_binding *bindp)
 {
-    dircache_suspend();
-    dircache_free();
+    /* requires write exclusion */
+    logf("dc close: %u", (unsigned int)bindp->info.dcfile.serialnum);
+    binding_close(bindp);
 }
 
 /**
- * Steal the allocated dircache buffer and disable dircache.
+ * called by file code when a file is newly created
  */
-void* dircache_steal_buffer(size_t *size)
+void dircache_fileop_create(struct file_base_info *dirinfop,
+                            struct file_base_binding *bindp,
+                            const char *basename,
+                            const struct dirinfo_native *dinp)
 {
-    dircache_suspend();
-    if (dircache_size == 0)
+    /* requires write exclusion */
+    logf("dc create: %u \"%s\"",
+         (unsigned int)bindp->info.dcfile.serialnum, basename);
+
+    if (!dirinfop->dcfile.serialnum)
     {
-        *size = 0;
-        return NULL;
+        /* no parent binding => no child binding */
+        return;
     }
 
-    /* since we give up the buffer (without freeing), it must not move anymore */
-    dont_move = true;
-    *size = dircache_size + (DIRCACHE_RESERVE-reserve_used);
-    
-    return dircache_root;
+    struct dircache_entry *ce;
+    int idx = create_entry(basename, &ce);
+    if (idx == 0)
+    {
+        /* failed allocation; parent cache contents are not complete */
+        establish_frontier(dirinfop->dcfile.idx, FRONTIER_ZONED);
+        return;
+    }
+
+    struct file_base_info *infop = &bindp->info;
+
+#ifdef DIRCACHE_NATIVE
+    ce->firstcluster = infop->fatfile.firstcluster;
+    ce->direntry     = infop->fatfile.e.entry;
+    ce->direntries   = infop->fatfile.e.entries;
+    ce->wrtdate      = dinp->wrtdate;
+    ce->wrttime      = dinp->wrttime;
+#else
+    ce->mtime        = dinp->mtime;
+#endif
+    ce->attr         = dinp->attr;
+    if (!(dinp->attr & ATTR_DIRECTORY))
+        ce->filesize = dinp->size;
+
+    insert_file_entry(dirinfop, ce);
+
+    /* file binding will have been queued when it was opened; just resolve */
+    infop->dcfile.idx       = idx;
+    infop->dcfile.serialnum = ce->serialnum;
+    binding_resolve(infop);
+
+    if ((dinp->attr & ATTR_DIRECTORY) && !is_dotdir_name(basename))
+    {
+        /* scan-in the contents of the new directory at this level only */
+        buffer_lock();
+        sab_process_dir(infop, false);
+        buffer_unlock();
+    }
 }
 
 /**
- * Usermode function to return dircache_entry index to the given path.
+ * called by file code when a file or directory is removed
  */
-static int dircache_get_entry_id_ex(const char *filename, bool go_down)
+void dircache_fileop_remove(struct file_base_binding *bindp)
 {
-    if (!dircache_initialized || filename == NULL)
-        return -1;
-    
-    struct dircache_entry* res = dircache_get_entry(filename, go_down);
-    return res ? res - dircache_root : -1;
-}
+    /* requires write exclusion */
+    logf("dc remove: %u\n", (unsigned int)bindp->info.dcfile.serialnum);
 
-int dircache_get_entry_id(const char* filename)
-{
-    return dircache_get_entry_id_ex(filename, false);
+    if (!bindp->info.dcfile.serialnum)
+        return; /* no binding yet */
+
+    free_file_entry(&bindp->info);
+
+    /* if binding was resolved; it should now be queued via above call */
 }
 
 /**
- * Internal: Get the startcluster for the index
+ * called by file code when a file is renamed
  */
-long _dircache_get_entry_startcluster(int id)
+void dircache_fileop_rename(struct file_base_info *dirinfop,
+                            struct file_base_binding *bindp,
+                            const char *basename)
 {
-    return get_entry(id)->startcluster;
+    /* requires write exclusion */
+    logf("dc rename: %u \"%s\"",
+         (unsigned int)bindp->info.dcfile.serialnum, basename);
+
+    if (!dirinfop->dcfile.serialnum)
+    {
+        /* new parent directory not cached; there is nowhere to put it so
+           nuke it */
+        if (bindp->info.dcfile.serialnum)
+            free_file_entry(&bindp->info);
+        /* else no entry anyway */
+
+        return;
+    }
+
+    if (!bindp->info.dcfile.serialnum)
+    {
+        /* binding not resolved on the old file but it's going into a resolved
+           parent which means the parent would be missing an entry in the cache;
+           downgrade the parent */
+        establish_frontier(dirinfop->dcfile.idx, FRONTIER_ZONED);
+        return;
+    }
+
+    /* unlink the entry but keep it; it needs to be re-sorted since the
+       underlying FS probably changed the order */
+    struct dircache_entry *ce = remove_file_entry(&bindp->info);
+
+#ifdef DIRCACHE_NATIVE
+    /* update other name-related information before inserting */
+    ce->direntry   = bindp->info.fatfile.e.entry;
+    ce->direntries = bindp->info.fatfile.e.entries;
+#endif
+
+    /* place it into its new home */
+    insert_file_entry(dirinfop, ce);
+
+    /* lastly, update the entry name itself */
+    if (entry_reassign_name(ce, basename))
+    {
+        /* it's not really the same one now so re-stamp it */
+        dc_serial_t serialnum = next_serialnum();
+        ce->serialnum = serialnum;
+        bindp->info.dcfile.serialnum = serialnum;
+    }
+    else
+    {
+        /* it cannot be kept around without a valid name */
+        free_file_entry(&bindp->info);
+        establish_frontier(dirinfop->dcfile.idx, FRONTIER_ZONED);
+    }
 }
 
 /**
- * Internal: Get the struct dirinfo for the index
+ * called by file code to synchronize file entry information
  */
-struct dirinfo* _dircache_get_entry_dirinfo(int id)
+void dircache_fileop_sync(struct file_base_binding *bindp,
+                          const struct dirinfo_native *dinp)
 {
-    return &get_entry(id)->info;
+    /* requires write exclusion */
+    struct file_base_info *infop = &bindp->info;
+    logf("dc sync: %u\n", (unsigned int)infop->dcfile.serialnum);
+
+    if (!infop->dcfile.serialnum)
+        return; /* binding unresolved */
+
+    struct dircache_entry *ce = get_entry(infop->dcfile.idx);
+    if (!ce)
+    {
+        logf("  bad index %d", infop->dcfile.idx);
+        return; /* a root (should never be called for this) */
+    }
+
+#ifdef DIRCACHE_NATIVE
+    ce->firstcluster = infop->fatfile.firstcluster;
+    ce->wrtdate      = dinp->wrtdate;
+    ce->wrttime      = dinp->wrttime;
+#else
+    ce->mtime        = dinp->mtime;
+#endif
+    ce->attr         = dinp->attr;
+    if (!(dinp->attr & ATTR_DIRECTORY))
+        ce->filesize = dinp->size;
 }
 
-/*
- * build a path from an entry upto the root using recursion
- *
- * it appends '/' after strlcat, therefore buf[0] needs to be prepared with '/'
- * and it will leave a trailing '/'
- *
- * returns the position of that trailing '/' so it can be deleted afterwards
- * (or, in case of truncation, the position of the nul byte */
-static size_t copy_path_helper(const struct dircache_entry *entry, char *buf, size_t size)
+
+/** Dircache paths and files **/
+
+#ifdef DIRCACHE_DUMPSTER
+/* helper for dircache_get_path() */
+static ssize_t get_path_sub(int idx, char *buf, size_t size)
 {
-    int offset = 1;
-    /* has parent? */
-    if (entry->up)
-        offset += copy_path_helper(entry->up, buf, size);
+    if (idx == 0)
+        return -2; /* entry is an orphan split from any root */
 
-    size_t len = strlcpy(buf+offset, entry->d_name, size - offset) + offset;
-    if (len < size)
+    ssize_t offset;
+    char *cename;
+
+    if (idx > 0)
     {
-        buf[len++] = '/';
-        buf[len]   = '\0';
+        /* go all the way up then move back down from the root */
+        struct dircache_entry *ce = get_entry(idx);
+        offset = get_path_sub(ce->up, buf, size) - 1;
+        if (offset < 0)
+            return -3;
+
+        cename = alloca(MAX_NAME + 1);
+        entry_name_copy(cename, ce);
     }
-    return len-1;
+    else /* idx < 0 */
+    {
+        offset = 0;
+        cename = "";
+
+    #ifdef HAVE_MULTIVOLUME
+        int volume = IF_MV_VOL(-idx - 1);
+        if (volume > 0)
+        {
+            /* prepend the volume specifier for volumes > 0 */
+            cename = alloca(VOL_MAX_LEN+1);
+            get_volume_name(volume, cename);
+        }
+    #endif /* HAVE_MULTIVOLUME */
+    }
+
+    return offset + path_append(buf + offset, PA_SEP_HARD, cename,
+                                size > (size_t)offset ? size - offset : 0);
 }
+#endif /* DIRCACHE_DUMPSTER */
+
+#if 0
+
 /**
- * Function to copy the full absolute path from dircache to the given buffer
- * using the given dircache_entry pointer.
- *
- * Returns the size of the resulting string, or 0 if an error occured
+ * retrieve and validate the file's entry/binding serial number
+ * the dircache file's serial number must match the indexed entry's or the
+ * file reference is stale
  */
-size_t dircache_copy_path(int index, char *buf, size_t size)
+static dc_serial_t get_file_serialnum(const struct dircache_file *dcfilep)
 {
-    if (!size || !buf || index < 0)
+    int idx = dcfilep->idx;
+
+    if (idx == 0 || idx < -NUM_VOLUMES)
         return 0;
 
-    buf[0] = '/';
-    size_t res = copy_path_helper(&dircache_root[index], buf, size - 1);
-    /* fixup trailing '/' */
-    buf[res] = '\0';
-    return res;
+    dc_serial_t serialnum;
+
+    if (idx > 0)
+    {
+        struct dircache_entry *ce = get_entry(idx);
+        serialnum = ce ? ce->serialnum : 0;
+    }
+    else
+    {
+        serialnum = get_idx_dcvolp(idx)->serialnum;
+    }
+
+    return serialnum == dcfilep->serialnum ? serialnum : 0;
 }
 
-/* --- Directory cache live updating functions --- */
-static int block_until_ready(void)
+/**
+ * usermode function to construct a full absolute path from dircache into the
+ * given buffer given the dircache file info
+ *
+ * returns:
+ *   success - the length of the string, not including the trailing null
+ *   failure - a negative value
+ *
+ * successful return value is as strlcpy()
+ *
+ * errors:
+ *   ENOENT - the file or directory does not exist
+ */
+ssize_t dircache_get_path(const struct dircache_file *dcfilep, char *buf,
+                          size_t size)
 {
-    /* Block until dircache has been built. */
-    while (!dircache_initialized && dircache_is_initializing())
-        sleep(1);
-    
-    if (!dircache_initialized)
-        return -1;
-    
-    return 0;
+    /* if missing buffer space, still return what's needed a la strlcpy */
+    if (!buf)
+        size = 0;
+    else if (size)
+        *buf = '\0';
+
+    ssize_t len = -1;
+
+    dircache_lock();
+
+    /* first and foremost, there must be a cache and the serial number must
+       check out */
+    if (dircache_runinfo.handle && get_file_serialnum(dcfilep))
+        len = get_path_sub(dcfilep->idx, buf, size);
+
+    if (len < 0)
+        errno = ENOENT;
+
+    dircache_unlock();
+    return len;
 }
 
-static struct dircache_entry* dircache_new_entry(const char *path, int attribute)
-{
-    struct dircache_entry *entry;
-    char basedir[MAX_PATH*2];
-    char *new;
-    long last_cache_size = dircache_size;
+/**
+ * searches the sublist starting at 'idx' for the named component
+ */
 
-    strlcpy(basedir, path, sizeof(basedir));
-    new = strrchr(basedir, '/');
-    if (new == NULL)
+/* helper for get_file_sub() */
+static struct dircache_entry *
+get_file_sub_scan(int idx, const char *name, size_t length, int *idxp)
+{
+    struct dircache_entry *ce = get_entry(idx);
+    if (ce)
     {
-        logf("error occurred");
-        dircache_initialized = false;
-        return NULL;
+        char entname[MAX_NAME+1];
+        name = strmemdupa(name, length);
+
+        do
+        {
+            entry_name_copy(entname, ce);
+            if (!strcasecmp(entname, name))
+            {
+                *idxp = idx;
+                break;
+            }
+
+            idx = ce->next;
+        }
+        while ((ce = get_entry(idx)));
     }
 
-    *new = '\0';
-    new++;
+    return ce;
+}
+
+/**
+ * searches for the subcomponent of *pathp
+ */
+
+/* helper for dircache_get_file() */
+static int get_file_sub(const char **pathp, int *downp, int *idxp)
+{
+    int rc;
+    const char *name;
+    rc = parse_path_component(pathp, &name, false);
+    if (rc <= 0)
+        return rc;
+    else if (rc >= MAX_PATH)
+        return ENAMETOOLONG; /* that's just unpossible, man */
+
+    struct dircache_entry *ce = get_file_sub_scan(*downp, name, rc, idxp);
 
-    entry = dircache_get_entry(basedir, true);
-    if (entry == NULL)
+    if (!ce)
+        rc = RC_NOT_FOUND;   /* not there; tellibry solly */
+    else if (!*pathp)
+        rc = RC_PATH_ENDED;  /* done */
+    else if (!(ce->attr & ATTR_DIRECTORY))
+        rc = ENOTDIR;        /* a parent component must be a directory */
+    else
+        while ((rc = get_file_sub(pathp, &ce->down, idxp)) == RC_CONTINUE);
+
+    switch (rc)
     {
-        logf("basedir not found!");
-        logf("%s", basedir);
-        dircache_initialized = false;
-        return NULL;
+    case RC_GO_UP:           /* hit ".."; drop to previous level */
+        return RC_CONTINUE;
+    case RC_PATH_ENDED:      /* success! */
+        return RC_FOUND;
+    default:                 /* component not found or error */
+        return rc;
     }
+}
 
-    if (reserve_used + 2*sizeof(struct dircache_entry) + strlen(new)+1
-        >= DIRCACHE_RESERVE)
+/**
+ * usermode function to return dircache file info for the given path
+ *
+ * returns:
+ *   success: the volume number that is specified for the file
+ *   failure: a negative value
+ *
+ * errors:
+ *   ENOENT       - the file or directory does not exist or path is empty
+ *   ENAMETOOLONG - a component of the path is too long
+ *   ENOTDIR      - a component of the path is not a directory
+ */
+int dircache_get_file(const char *path, struct dircache_file *dcfilep)
+{
+    if (!path_is_absolute(path) || !dcfilep)
     {
-        logf("not enough space");
-        dircache_initialized = false;
-        return NULL;
+        errno = ENOENT;
+        return -1;
     }
-    
-    while (entry->next != NULL)
-        entry = entry->next;
 
-    if (entry->d_name != NULL)
+    dircache_lock();
+
+    if (!dircache_runinfo.handle)
     {
-        entry = dircache_gen_next(entry);
-        if (entry == NULL)
+        dircache_unlock();
+        errno = ENOENT;
+        return -2;
+    }
+
+    int volume = 0;
+    int idx = 0;
+    dc_serial_t serialnum = 0;
+    struct dircache_volume *dcvolp = NULL;
+    struct dircache_entry *ce = NULL;
+
+    int rc = RC_GO_UP;
+
+    while (rc == RC_CONTINUE || rc == RC_GO_UP)
+    {
+    #ifdef HAVE_MULTIVOLUME
+        if (rc == RC_GO_UP)
         {
-            dircache_initialized = false;
-            return NULL;
+            volume = path_strip_volume(path, &path, false);
+            if (!CHECK_VOL(volume))
+            {
+                rc = ENXIO;
+                break;
+            }
         }
-    }
+    #endif /* HAVE_MULTIVOLUME */
+
+        dcvolp = DCVOL(volume);
 
-    size_t size = strlen(new) + 1;
-    entry->d_name = (d_names_start -= size);
-    entry->startcluster = 0;
-    memset(&entry->info, 0, sizeof(entry->info));
-    entry->info.attribute = attribute;
+        int *downp = &dcvolp->root_down;
+        if (*downp <= 0)
+        {
+            rc = ENXIO;
+            break;
+        }
 
-    strcpy(entry->d_name, new);
-    dircache_size += size;
+        rc = get_file_sub(&path, downp, &idx);
+    }
 
-    if (attribute & ATTR_DIRECTORY)
+    switch (rc)
     {
-        logf("gen_down");
-        dircache_gen_down(entry);
+    case RC_FOUND:      /* hit: component found */
+        serialnum = ce->serialnum;
+        rc = volume;
+        break;
+    case RC_PATH_ENDED: /* hit: it's a root (volume or system) */
+        idx = -volume - 1;
+        serialnum = dcvolp->serialnum;
+        rc = volume;
+        break;
+    case RC_NOT_FOUND:  /* miss */
+        rc = ENOENT;
+    default:
+        idx = 0;
+        errno = rc;
+        rc = -3;
+        break;
     }
-        
-    reserve_used += dircache_size - last_cache_size;
 
-    return entry;
+    dcfilep->idx       = idx;
+    dcfilep->serialnum = serialnum;
+
+    dircache_unlock();
+    return rc;
 }
+#endif /* 0 */
+
 
-void dircache_bind(int fd, const char *path)
+/** Debug screen/info stuff **/
+
+/**
+ * return cache state parameters
+ */
+void dircache_get_info(struct dircache_info *info)
 {
-    struct dircache_entry *entry;
-    
-    /* Queue requests until dircache has been built. */
-    if (!dircache_initialized && dircache_is_initializing())
+    static const char * const status_descriptions[] =
+    {
+        [DIRCACHE_IDLE]     = "Idle",
+        [DIRCACHE_SCANNING] = "Scanning",
+        [DIRCACHE_READY]    = "Ready",
+    };
+
+    if (!info)
+        return;
+
+    memset(info, 0, sizeof (*info));
+
+    dircache_lock();
+
+    enum dircache_status status = DIRCACHE_IDLE;
+
+    for (unsigned int volume = 0; volume < NUM_VOLUMES; volume++)
     {
-        if (fdbind_idx >= MAX_PENDING_BINDINGS)
-            return ;
-        strlcpy(fdbind_cache[fdbind_idx].path, path, 
-                sizeof(fdbind_cache[fdbind_idx].path));
-        fdbind_cache[fdbind_idx].fd = fd;
-        fdbind_idx++;
-        return ;
+        struct dircache_volume *dcvolp = DCVOL(volume);
+        enum dircache_status volstatus = dcvolp->status;
+
+        switch (volstatus)
+        {
+        case DIRCACHE_SCANNING:
+            /* if any one is scanning then overall status is "scanning" */
+            status = volstatus;
+
+            /* sum the time the scanning has taken so far */
+            info->build_ticks += current_tick - dcvolp->start_tick;
+            break;
+        case DIRCACHE_READY:
+            /* if all the rest are idle and at least one is ready, then
+               status is "ready". */
+            if (status == DIRCACHE_IDLE)
+                status = DIRCACHE_READY;
+
+            /* sum the build ticks of all "ready" volumes */
+            info->build_ticks += dcvolp->build_ticks;
+            break;
+        case DIRCACHE_IDLE:
+            /* if all are idle; then the whole cache is "idle" */
+            break;
+        }
     }
-    
-    if (!dircache_initialized)
-        return ;
 
-    logf("bind: %d/%s", fd, path);
-    entry = dircache_get_entry(path, false);
-    if (entry == NULL)
+    info->status     = status;
+    info->statusdesc = status_descriptions[status];
+    info->last_size  = dircache.last_size;
+    info->size_limit = DIRCACHE_LIMIT;
+    info->reserve    = DIRCACHE_RESERVE;
+
+    /* report usage only if there is something ready or being built */
+    if (status != DIRCACHE_IDLE)
     {
-        logf("not found!");
-        dircache_initialized = false;
-        return ;
+        info->reserve_used = reserve_buf_used();
+        info->size         = dircache.size;
+        info->sizeused     = dircache.sizeused;
+        info->entry_count  = dircache.numentries;
     }
 
-    fd_bindings[fd] = entry;
+    dircache_unlock();
 }
 
-void dircache_update_filesize(int fd, long newsize, long startcluster)
+#ifdef DIRCACHE_DUMPSTER
+/**
+ * dump RAW binary of buffer and CSV of all valid paths and volumes to disk
+ */
+void dircache_dump(void)
 {
-    if (!dircache_initialized || fd < 0)
-        return ;
-
-    if (fd_bindings[fd] == NULL)
+    /* open both now so they're in the cache */
+    int fdbin = open(DIRCACHE_DUMPSTER_BIN, O_WRONLY|O_CREAT|O_TRUNC, 0666);
+    int fdcsv = open(DIRCACHE_DUMPSTER_CSV, O_WRONLY|O_CREAT|O_TRUNC, 0666);
+    if (fdbin < 0 || fdcsv < 0)
     {
-        logf("dircache fd(%d) access error", fd);
-        dircache_initialized = false;
-        return ;
+        close(fdbin);
+        return;
     }
-    
-    fd_bindings[fd]->info.size = newsize;
-    fd_bindings[fd]->startcluster = startcluster;
-}
-void dircache_update_filetime(int fd)
-{
-#if CONFIG_RTC == 0
-    (void)fd;
-#else
-    short year;
-    struct tm *now = get_time();
-    if (!dircache_initialized || fd < 0)
-        return ;
-
-    if (fd_bindings[fd] == NULL)
-    {
-        logf("dircache fd access error");
-        dircache_initialized = false;
-        return ;
-    }
-    year = now->tm_year+1900-1980;
-    fd_bindings[fd]->info.wrtdate = (((year)&0x7f)<<9)           |
-                                    (((now->tm_mon+1)&0xf)<<5)   |
-                                    (((now->tm_mday)&0x1f));
-    fd_bindings[fd]->info.wrttime = (((now->tm_hour)&0x1f)<<11)  |
-                                    (((now->tm_min)&0x3f)<<5)    |
-                                    (((now->tm_sec/2)&0x1f));
-#endif
-}
 
-void dircache_mkdir(const char *path)
-{ /* Test ok. */
-    if (block_until_ready())
-        return ;
-        
-        
-    logf("mkdir: %s", path);
-    dircache_new_entry(path, ATTR_DIRECTORY);
-}
-
-void dircache_rmdir(const char *path)
-{ /* Test ok. */
-    struct dircache_entry *entry;
-    
-    if (block_until_ready())
-        return ;
-        
-    logf("rmdir: %s", path);
-    entry = dircache_get_entry(path, false);
-    if (entry == NULL || entry->down == NULL)
-    {
-        logf("not found or not a directory!");
-        dircache_initialized = false;
-        return ;
-    }
-
-    entry->down = NULL;
-    entry->d_name = NULL;
-}
-
-/* Remove a file from cache */
-void dircache_remove(const char *name)
-{ /* Test ok. */
-    struct dircache_entry *entry;
-    
-    if (block_until_ready())
-        return ;
-        
-    logf("remove: %s", name);
-    
-    entry = dircache_get_entry(name, false);
-
-    if (entry == NULL)
-    {
-        logf("not found!");
-        dircache_initialized = false;
-        return ;
-    }
-    
-    entry->d_name = NULL;
-}
-
-void dircache_rename(const char *oldpath, const char *newpath)
-{ /* Test ok. */
-    struct dircache_entry *entry, *newentry;
-    struct dircache_entry oldentry;
-    char absolute_path[MAX_PATH*2];
-    char *p;
-    
-    if (block_until_ready())
-        return ;
-        
-    logf("rename: %s->%s", oldpath, newpath);
-    
-    entry = dircache_get_entry(oldpath, false);
-    if (entry == NULL)
-    {
-        logf("not found!");
-        dircache_initialized = false;
-        return ;
-    }
-
-    /* Delete the old entry. */
-    entry->d_name = NULL;
-
-    /** If we rename the same filename twice in a row, we need to
-     * save the data, because the entry will be re-used. */
-    oldentry = *entry;
-
-    /* Generate the absolute path for destination if necessary. */
-    if (newpath[0] != '/')
-    {
-        strlcpy(absolute_path, oldpath, sizeof(absolute_path));
-        p = strrchr(absolute_path, '/');
-        if (!p)
+    trigger_cpu_boost();
+    dircache_lock();
+
+    if (dircache_runinfo.handle)
+    {
+        buffer_lock();
+
+        /* bin */
+        write(fdbin, dircache_runinfo.p + ENTRYSIZE,
+              dircache_runinfo.bufsize + 1);
+
+        /* CSV */
+        fdprintf(fdcsv, "\"Index\",\"Serialnum\","
+                        "\"Path\",\"Frontier\","
+                        "\"Attribute\",\"File Size\","
+                        "\"Mod Date\",\"Mod Time\"\n");
+
+        FOR_EACH_VOLUME(-1, volume)
         {
-            logf("Invalid path");
-            dircache_initialized = false;
-            return ;
+            struct dircache_volume *dcvolp = DCVOL(volume);
+            if (dcvolp->status == DIRCACHE_IDLE)
+                continue;
+
+        #ifdef HAVE_MULTIVOLUME
+            char name[VOL_MAX_LEN+1];
+            get_volume_name(volume, name);
+        #endif
+            fdprintf(fdcsv,
+                     "%d,%lu,"
+                     "\"%c" IF_MV("%s") "\",%u,"
+                     "0x%08X,0,"
+                     "\"\",\"\"\n",
+                     -volume-1, dcvolp->serialnum,
+                     PATH_SEPCH, IF_MV(name,) dcvolp->frontier,
+                     ATTR_DIRECTORY | ATTR_VOLUME);
         }
-        
-        *p = '\0';
-        strlcpy(p, absolute_path, sizeof(absolute_path)-strlen(p));
-        newpath = absolute_path;
-    }
-    
-    newentry = dircache_new_entry(newpath, entry->info.attribute);
-    if (newentry == NULL)
-    {
-        dircache_initialized = false;
-        return ;
+
+        FOR_EACH_CACHE_ENTRY(ce)
+        {
+        #ifdef DIRCACHE_NATIVE
+            time_t mtime = fattime_mktime(ce->wrtdate, ce->wrttime);
+        #else
+            time_t mtime = ce->mtime;
+        #endif
+            struct tm tm;
+            gmtime_r(&mtime, &tm);
+
+            char buf[DC_MAX_NAME + 2];
+            *buf = '\0';
+            int idx = get_index(ce);
+            get_path_sub(idx, buf, sizeof (buf));
+
+            fdprintf(fdcsv,
+                     "%d,%lu,"
+                     "\"%s\",%u,"
+                     "0x%08X,%lu,"
+                     "%04d/%02d/%02d,"
+                     "%02d:%02d:%02d\n",
+                     idx, ce->serialnum,
+                     buf, ce->frontier,
+                     ce->attr, (ce->attr & ATTR_DIRECTORY) ?
+                                0ul : (unsigned long)ce->filesize,
+                     tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+                     tm.tm_hour, tm.tm_min, tm.tm_sec);
+        }
+
+        buffer_unlock();
     }
 
-    newentry->down = oldentry.down;
-    newentry->startcluster = oldentry.startcluster;
-    newentry->info.size    = oldentry.info.size;
-    newentry->info.wrtdate = oldentry.info.wrtdate;
-    newentry->info.wrttime = oldentry.info.wrttime;
+    dircache_unlock();
+    cancel_cpu_boost();
+
+    close(fdbin);
+    close(fdcsv);
 }
+#endif /* DIRCACHE_DUMPSTER */
+
 
-void dircache_add_file(const char *path, long startcluster)
+/** Misc. stuff **/
+
+/**
+ * set the dircache file to initial values
+ */
+void dircache_dcfile_init(struct dircache_file *dcfilep)
 {
-    struct dircache_entry *entry;
-    
-    if (block_until_ready())
-        return ;
-    
-    logf("add file: %s", path);
-    entry = dircache_new_entry(path, 0);
-    if (entry == NULL)
-        return ;
-    
-    entry->startcluster = startcluster;
+    dcfilep->idx       = 0;
+    dcfilep->serialnum = 0;
 }
 
-static bool is_disable_msg_pending(void)
+#ifdef HAVE_EEPROM_SETTINGS
+
+#ifdef HAVE_HOTSWAP
+/* NOTE: This is hazardous to the filesystem of any sort of removable
+         storage unless it may be determined that the filesystem from save
+         to load is identical. If it's not possible to do so in a timely
+         manner, it's not worth persisting the cache. */
+  #warning "Don't do this; you'll find the consequences unpleasant."
+#endif
+
+/* dircache persistence file header magic */
+#define DIRCACHE_MAGIC  0x00d0c0a1
+
+/* dircache persistence file header */
+struct dircache_maindata
 {
-    return check_event_queue();
+    uint32_t        magic;      /* DIRCACHE_MAGIC */
+    struct dircache dircache;   /* metadata of the cache! */
+    uint32_t        datacrc;    /* CRC32 of data */
+    uint32_t        hdrcrc;     /* CRC32 of header through datacrc */
+} __attribute__((packed, aligned (4)));
+
+/**
+ * verify that the clean status is A-ok
+ */
+static bool dircache_is_clean(bool saving)
+{
+    if (saving)
+        return dircache.dcvol[0].status == DIRCACHE_READY;
+    else
+    {
+        return dircache.dcvol[0].status == DIRCACHE_IDLE &&
+               !dircache_runinfo.enabled;
+    }
 }
 
-DIR_CACHED* opendir_cached(const char* name)
+/**
+ * function to load the internal cache structure from disk to initialize
+ * the dircache really fast with little disk access.
+ */
+int dircache_load(void)
 {
-    int dd;
-    DIR_CACHED* pdir = opendirs;
+    logf("Loading directory cache");
+    int fd = open_dircache_file(O_RDONLY);
+    if (fd < 0)
+        return -1;
+
+    int rc = -1;
+
+    ssize_t size;
+    struct dircache_maindata maindata;
+    uint32_t crc;
+    int handle = 0;
+    bool hasbuffer = false;
 
-    if ( name[0] != '/' )
+    size = sizeof (maindata);
+    if (read(fd, &maindata, size) != size)
     {
-        DEBUGF("Only absolute paths supported right now\n");
-        return NULL;
+        logf("dircache: header read failed");
+        goto error_nolock;
     }
 
-    /* find a free dir descriptor */
-    for ( dd=0; dd<MAX_OPEN_DIRS; dd++, pdir++)
-        if ( !pdir->busy )
-            break;
+    /* sanity check the header */
+    if (maindata.magic != DIRCACHE_MAGIC)
+    {
+        logf("dircache: invalid header magic");
+        goto error_nolock;
+    }
 
-    if ( dd == MAX_OPEN_DIRS )
+    crc = crc_32(&maindata, offsetof(struct dircache_maindata, hdrcrc),
+                 0xffffffff);
+    if (crc != maindata.hdrcrc)
     {
-        DEBUGF("Too many dirs open\n");
-        errno = EMFILE;
-        return NULL;
+        logf("dircache: invalid header CRC32");
+        goto error_nolock;
     }
-    
-    pdir->busy = true;
 
-    if (!dircache_initialized || is_disable_msg_pending())
+    if (maindata.dircache.size !=
+            maindata.dircache.sizeentries + maindata.dircache.sizenames ||
+        ALIGN_DOWN(maindata.dircache.size, ENTRYSIZE) != maindata.dircache.size ||
+        filesize(fd) - sizeof (maindata) != maindata.dircache.size)
     {
-        pdir->internal_entry = -1;
-        pdir->regulardir = opendir_uncached(name);   
+        logf("dircache: file header error");
+        goto error_nolock;
     }
-    else
+
+    /* allocate so that exactly the reserve size remains */
+    size_t bufsize = maindata.dircache.size + DIRCACHE_RESERVE + 1;
+    handle = alloc_cache(bufsize);
+    if (handle <= 0)
+    {
+        logf("dircache: failed load allocation");
+        goto error_nolock;
+    }
+
+    dircache_lock();
+    buffer_lock();
+
+    if (!dircache_is_clean(false))
+        goto error;
+
+    /* from this point on, we're actually dealing with the cache in RAM */
+    dircache = maindata.dircache;
+
+    set_buffer(handle, bufsize);
+    hasbuffer = true;
+
+    /* convert back to in-RAM representation */
+    dircache.numentries = maindata.dircache.sizeentries / ENTRYSIZE;
+
+    /* read the dircache file into memory; start with the entries */
+    size = maindata.dircache.sizeentries;
+    if (read(fd, dircache_runinfo.pentry + 1, size) != size)
     {
-        pdir->regulardir = NULL;
-        pdir->internal_entry = dircache_get_entry_id_ex(name, true);
-        pdir->theent.info.attribute = -1; /* used to make readdir_cached aware of the first call */
+        logf("dircache read failed #1");
+        goto error;
     }
 
-    if (pdir->internal_entry == -1 && pdir->regulardir == NULL)
+    crc = crc_32(dircache_runinfo.pentry + 1, size, 0xffffffff);
+
+    /* continue with the names; fix up indexes to them if needed */
+    dircache.names -= maindata.dircache.sizenames;
+    *get_name(dircache.names - 1) = 0;
+
+    size = maindata.dircache.sizenames;
+    if (read(fd, get_name(dircache.names), size) != size)
     {
-        pdir->busy = false;
-        return NULL;
+        logf("dircache read failed #2");
+        goto error;
     }
 
-    return pdir;
-}
+    crc = crc_32(get_name(dircache.names), size, crc);
+    if (crc != maindata.datacrc)
+    {
+        logf("dircache: data failed CRC32");
+        goto error;
+    }
 
-struct dirent_cached* readdir_cached(DIR_CACHED* dir)
-{
-    struct dircache_entry *ce = get_entry(dir->internal_entry);
-    struct dirent_uncached *regentry;
-    
-    if (!dir->busy)
-        return NULL;
+    /* only names will be changed in relative position so fix up those
+       references */
+    ssize_t offset = dircache.names - maindata.dircache.names;
+    if (offset != 0)
+    {
+        /* nothing should be open besides the dircache file itself therefore
+           no bindings need be resolved; the cache will have its own entry
+           but that should get cleaned up when removing the file */
+        FOR_EACH_CACHE_ENTRY(ce)
+        {
+            if (!ce->tinyname)
+                ce->name += offset;
+        }
+    }
 
-    if (dir->regulardir != NULL)
-    {
-        regentry = readdir_uncached(dir->regulardir);
-        if (regentry == NULL)
-            return NULL;
-
-        strlcpy(dir->theent.d_name, regentry->d_name, MAX_PATH);
-        dir->theent.startcluster = regentry->startcluster;
-        dir->theent.info = regentry->info;
-        
-        return &dir->theent;
-    }
-    
-    /* if theent.attribute=-1 then this is the first call */
-    /* otherwise, this is is not so we first take the entry's ->next */
-    /* NOTE: normal file can't have attribute=-1 */
-    if(dir->theent.info.attribute != -1)
-        ce = ce->next;
-    /* skip unused entries */
-    while(ce != NULL && ce->d_name == NULL)
-        ce = ce->next;
-    
-    if (ce == NULL)
-            return NULL;
+    dircache.reserve_used = 0;
+
+    /* enable the cache but do not try to build it */
+    dircache_enable_internal(false);
+
+    /* cache successfully loaded */
+    logf("Done, %ld KiB used", dircache.size / 1024);
+    rc = 0;
+error:
+    if (rc < 0 && hasbuffer)
+        reset_buffer();
 
-    strlcpy(dir->theent.d_name, ce->d_name, MAX_PATH);
-    /* Can't do `dir->theent = *ce`
-       because that modifies the d_name pointer. */
-    dir->theent.startcluster = ce->startcluster;
-    dir->theent.info = ce->info;
-    dir->internal_entry = ce - dircache_root;
+    buffer_unlock();
+    dircache_unlock();
 
-    //logf("-> %s", ce->d_name);
-    return &dir->theent;
+error_nolock:
+    if (rc < 0 && handle > 0)
+        core_free(handle);
+
+    if (fd >= 0)
+        close(fd);
+
+    remove_dircache_file();
+    return rc;
 }
 
-int closedir_cached(DIR_CACHED* dir)
+/**
+ * function to save the internal cache stucture to disk for fast loading
+ * on boot
+ */
+int dircache_save(void)
 {
-    if (!dir->busy)
+    logf("Saving directory cache");
+
+    int fd = open_dircache_file(O_WRONLY|O_CREAT|O_TRUNC|O_APPEND);
+    if (fd < 0)
         return -1;
-        
-    dir->busy=false;
-    if (dir->regulardir != NULL)
-        return closedir_uncached(dir->regulardir);
-    
-    return 0;
-}
 
-int mkdir_cached(const char *name)
-{
-    int rc=mkdir_uncached(name);
-    if (rc >= 0)
-        dircache_mkdir(name);
-    return(rc);
+    dircache_lock();
+    buffer_lock();
+
+    int rc = -1;
+
+    if (!dircache_is_clean(true))
+        goto error;
+
+    /* save the header structure along with the cache metadata */
+    ssize_t size;
+    uint32_t crc;
+    struct dircache_maindata maindata =
+    {
+        .magic    = DIRCACHE_MAGIC,
+        .dircache = dircache,
+    };
+
+    /* store the size since it better detects an invalid header */
+    maindata.dircache.sizeentries = maindata.dircache.numentries * ENTRYSIZE;
+
+    /* write the template header */
+    size = sizeof (maindata);
+    if (write(fd, &maindata, size) != size)
+    {
+        logf("dircache: write failed #1");
+        goto error;
+    }
+
+    /* write the dircache entries */
+    size = maindata.dircache.sizeentries;
+    if (write(fd, dircache_runinfo.pentry + 1, size) != size)
+    {
+        logf("dircache: write failed #2");
+        goto error;
+    }
+
+    crc = crc_32(dircache_runinfo.pentry + 1, size, 0xffffffff);
+
+    /* continue with the names */
+    size = maindata.dircache.sizenames;
+    if (write(fd, get_name(dircache.names), size) != size)
+    {
+        logf("dircache: write failed #3");
+        goto error;
+    }
+
+    crc = crc_32(get_name(dircache.names), size, crc);
+    maindata.datacrc = crc;
+
+    /* rewrite the header with CRC info */
+    if (lseek(fd, 0, SEEK_SET) != 0)
+    {
+        logf("dircache: seek failed");
+        goto error;
+    }
+
+    crc = crc_32(&maindata, offsetof(struct dircache_maindata, hdrcrc),
+                 0xffffffff);
+    maindata.hdrcrc = crc;
+
+    if (write(fd, &maindata, sizeof (maindata)) != sizeof (maindata))
+    {
+        logf("dircache: write failed #4");
+        goto error;
+    }
+
+    /* as of now, no changes to the volumes should be allowed at all since
+       that makes what was saved completely invalid */
+    rc = 0;
+error:
+    buffer_unlock();
+    dircache_unlock();
+
+    if (rc < 0)
+        remove_dircache_file();
+
+    close(fd);
+    return rc;
 }
+#endif /* HAVE_EEPROM_SETTINGS */
 
-int rmdir_cached(const char* name)
+/**
+ * main one-time initialization function that must be called before any other
+ * operations within the dircache
+ */
+void dircache_init(size_t last_size)
 {
-    int rc=rmdir_uncached(name);
-    if(rc >= 0)
-        dircache_rmdir(name);
-    return(rc);
+    queue_init(&dircache_queue, false);
+
+    dircache.last_size = MIN(last_size, DIRCACHE_LIMIT);
+
+    struct dircache_runinfo *dcrip = &dircache_runinfo;
+    dcrip->suspended         = 1;
+    dcrip->thread_done       = true;
+    dcrip->ops.move_callback = move_callback;
 }
diff --git a/firmware/common/disk.c b/firmware/common/disk.c
index 5a55a3b6ac..3a2d27e0d7 100644
--- a/firmware/common/disk.c
+++ b/firmware/common/disk.c
@@ -19,14 +19,25 @@
  *
  ****************************************************************************/
 #include <stdio.h>
+#include <string.h>
+#include "config.h"
 #include "kernel.h"
 #include "storage.h"
 #include "debug.h"
-#include "fat.h"
-#include "dir.h" /* for release_dirs() */
-#include "file.h" /* for release_files() */
+#include "disk_cache.h"
+#include "fileobj_mgr.h"
+#include "dir.h"
+#include "dircache_redirect.h"
 #include "disk.h"
-#include <string.h>
+
+#ifndef CONFIG_DEFAULT_PARTNUM
+#define CONFIG_DEFAULT_PARTNUM 0
+#endif
+
+#define disk_reader_lock()      file_internal_lock_READER()
+#define disk_reader_unlock()    file_internal_unlock_READER()
+#define disk_writer_lock()      file_internal_lock_WRITER()
+#define disk_writer_unlock()    file_internal_unlock_WRITER()
 
 /* Partition table entry layout:
    -----------------------
@@ -42,11 +53,18 @@
    12-15: nr of sectors in partition
 */
 
-#define BYTES2INT32(array,pos)                  \
-    ((long)array[pos] | ((long)array[pos+1] << 8 ) |        \
-     ((long)array[pos+2] << 16 ) | ((long)array[pos+3] << 24 ))
+#define BYTES2INT32(array, pos) \
+    (((uint32_t)array[pos+0] <<  0) | \
+     ((uint32_t)array[pos+1] <<  8) | \
+     ((uint32_t)array[pos+2] << 16) | \
+     ((uint32_t)array[pos+3] << 24))
+
+#define BYTES2INT16(array, pos) \
+    (((uint32_t)array[pos+0] << 0) | \
+     ((uint32_t)array[pos+1] << 8))
 
-static const unsigned char fat_partition_types[] = {
+static const unsigned char fat_partition_types[] =
+{
     0x0b, 0x1b, /* FAT32 + hidden variant */
     0x0c, 0x1c, /* FAT32 (LBA) + hidden variant */
 #ifdef HAVE_FAT16SUPPORT
@@ -56,159 +74,135 @@ static const unsigned char fat_partition_types[] = {
 #endif
 };
 
-static struct partinfo part[NUM_DRIVES*4]; /* space for 4 partitions on 2 drives */
-static int vol_drive[NUM_VOLUMES]; /* mounted to which drive (-1 if none) */
-static struct mutex disk_mutex;
+/* space for 4 partitions on 2 drives */
+static struct partinfo part[NUM_DRIVES*4];
+/* mounted to which drive (-1 if none) */
+static int vol_drive[NUM_VOLUMES];
+
+static int get_free_volume(void)
+{
+    for (int i = 0; i < NUM_VOLUMES; i++)
+    {
+        if (vol_drive[i] == -1) /* unassigned? */
+            return i;
+    }
+
+    return -1; /* none found */
+}
 
 #ifdef MAX_LOG_SECTOR_SIZE
-static int disk_sector_multiplier[NUM_DRIVES] = {[0 ... NUM_DRIVES-1] = 1};
+static int disk_sector_multiplier[NUM_DRIVES] =
+    { [0 ... NUM_DRIVES-1] = 1 };
 
 int disk_get_sector_multiplier(IF_MD_NONVOID(int drive))
 {
-    #ifdef HAVE_MULTIDRIVE
-    return disk_sector_multiplier[drive];
-    #else
-    return disk_sector_multiplier[0];
-    #endif
+    if (!CHECK_DRV(drive))
+        return 0;
+
+    disk_reader_lock();
+    int multiplier = disk_sector_multiplier[IF_MD_DRV(drive)];
+    disk_reader_unlock();
+    return multiplier;
 }
-#endif
+#endif /* MAX_LOG_SECTOR_SIZE */
 
-struct partinfo* disk_init(IF_MD_NONVOID(int drive))
+bool disk_init(IF_MD_NONVOID(int drive))
 {
-    int i;
-#ifdef HAVE_MULTIDRIVE
-    /* For each drive, start at a different position, in order not to destroy 
-       the first entry of drive 0. 
-       That one is needed to calculate config sector position. */
-    struct partinfo* pinfo = &part[drive*4];
-    if ((size_t)drive >= sizeof(part)/sizeof(*part)/4)
-        return NULL; /* out of space in table */
-#else
-    struct partinfo* pinfo = part;
-    const int drive = 0;
-    (void)drive;
-#endif
+    if (!CHECK_DRV(drive))
+        return false; /* out of space in table */
 
-    unsigned char* sector = fat_get_sector_buffer();
-    storage_read_sectors(IF_MD(drive,) 0,1, sector);
-    /* check that the boot sector is initialized */
-    if ( (sector[510] != 0x55) ||
-         (sector[511] != 0xaa)) {
-        fat_release_sector_buffer();
-        DEBUGF("Bad boot sector signature\n");
-        return NULL;
-    }
+    unsigned char *sector = dc_get_buffer();
+    if (!sector)
+        return false;
 
-    /* parse partitions */
-    for ( i=0; i<4; i++ ) {
-        unsigned char* ptr = sector + 0x1be + 16*i;
-        pinfo[i].type  = ptr[4];
-        pinfo[i].start = BYTES2INT32(ptr, 8);
-        pinfo[i].size  = BYTES2INT32(ptr, 12);
+    memset(sector, 0, SECTOR_SIZE);
+    storage_read_sectors(IF_MD(drive,) 0, 1, sector);
 
-        DEBUGF("Part%d: Type %02x, start: %08lx size: %08lx\n",
-               i,pinfo[i].type,pinfo[i].start,pinfo[i].size);
+    bool init = false;
 
-        /* extended? */
-        if ( pinfo[i].type == 5 ) {
-            /* not handled yet */
-        }
-    }
-    fat_release_sector_buffer();
-    return pinfo;
-}
+    /* check that the boot sector is initialized */
+    if (BYTES2INT16(sector, 510) == 0xaa55)
+    {
+        /* For each drive, start at a different position, in order not to
+           destroy the first entry of drive 0. That one is needed to calculate
+           config sector position. */
+        struct partinfo *pinfo = &part[IF_MD_DRV(drive)*4];
 
-struct partinfo* disk_partinfo(int partition)
-{
-    return &part[partition];
-}
+        disk_writer_lock();
 
-void disk_init_subsystem(void)
-{
-   mutex_init(&disk_mutex);
-}
+        /* parse partitions */
+        for (int i = 0; i < 4; i++)
+        {
+            unsigned char* ptr = sector + 0x1be + 16*i;
+            pinfo[i].type  = ptr[4];
+            pinfo[i].start = BYTES2INT32(ptr, 8);
+            pinfo[i].size  = BYTES2INT32(ptr, 12);
 
-int disk_mount_all(void)
-{
-    int mounted=0;
-    int i;
-    
-#ifdef HAVE_HOTSWAP
-    mutex_lock(&disk_mutex);
-#endif
+            DEBUGF("Part%d: Type %02x, start: %08lx size: %08lx\n",
+                   i,pinfo[i].type,pinfo[i].start,pinfo[i].size);
 
-    fat_init(); /* reset all mounted partitions */
-    for (i=0; i<NUM_VOLUMES; i++)
-        vol_drive[i] = -1; /* mark all as unassigned */
+            /* extended? */
+            if ( pinfo[i].type == 5 )
+            {
+                /* not handled yet */
+            }
+        }
 
-#ifndef HAVE_MULTIDRIVE
-    mounted = disk_mount(0);
-#else
-    for(i=0;i<NUM_DRIVES;i++)
+        disk_writer_unlock();
+
+        init = true;
+    }
+    else
     {
-#ifdef HAVE_HOTSWAP
-        if (storage_present(i))
-#endif
-            mounted += disk_mount(i); 
+        DEBUGF("Bad boot sector signature\n");
     }
-#endif
 
-#ifdef HAVE_HOTSWAP
-    mutex_unlock(&disk_mutex);
-#endif
-    return mounted;
+    dc_release_buffer(sector);
+    return init;
 }
 
-static int get_free_volume(void)
+bool disk_partinfo(int partition, struct partinfo *info)
 {
-    int i;
-    for (i=0; i<NUM_VOLUMES; i++)
-    {
-        if (vol_drive[i] == -1) /* unassigned? */
-            return i;
-    }
+    if (partition < 0 || partition >= (int)ARRAYLEN(part) || !info)
+        return false;
 
-    return -1; /* none found */
+    disk_reader_lock();
+    *info = part[partition];
+    disk_reader_unlock();
+    return true;
 }
 
 int disk_mount(int drive)
 {
     int mounted = 0; /* reset partition-on-drive flag */
-    int volume;
-    struct partinfo* pinfo;
 
-#ifdef HAVE_HOTSWAP
-    mutex_lock(&disk_mutex);
-#endif
+    disk_writer_lock();
 
-    volume = get_free_volume();
-    pinfo = disk_init(IF_MD(drive));
-#ifdef MAX_LOG_SECTOR_SIZE
-    disk_sector_multiplier[drive] = 1;
-#endif
+    int volume = get_free_volume();
 
-    if (pinfo == NULL)
+    if (!disk_init(IF_MD(drive)))
     {
-#ifdef HAVE_HOTSWAP
-        mutex_unlock(&disk_mutex);
-#endif
+        disk_writer_unlock();
         return 0;
     }
-#if defined(TOSHIBA_GIGABEAT_S)
-    int i = 1;  /* For the Gigabeat S, we mount the second partition */
-#else
-    int i = 0;
+
+    struct partinfo *pinfo = &part[IF_MD_DRV(drive)*4];
+#ifdef MAX_LOG_SECTOR_SIZE
+    disk_sector_multiplier[IF_MD_DRV(drive)] = 1;
 #endif
-    for (; volume != -1 && i<4 && mounted<NUM_VOLUMES_PER_DRIVE; i++)
+
+    for (int i = CONFIG_DEFAULT_PARTNUM;
+         volume != -1 && i < 4 && mounted < NUM_VOLUMES_PER_DRIVE;
+         i++)
     {
         if (memchr(fat_partition_types, pinfo[i].type,
                    sizeof(fat_partition_types)) == NULL)
             continue;  /* not an accepted partition type */
 
-#ifdef MAX_LOG_SECTOR_SIZE
-        int j;
-        
-        for (j = 1; j <= (MAX_LOG_SECTOR_SIZE/SECTOR_SIZE); j <<= 1)
+        bool success = false;
+
+    #ifdef MAX_LOG_SECTOR_SIZE
+        for (int j = 1; j <= (MAX_LOG_SECTOR_SIZE/SECTOR_SIZE); j <<= 1)
         {
             if (!fat_mount(IF_MV(volume,) IF_MD(drive,) pinfo[i].start * j))
             {
@@ -218,93 +212,242 @@ int disk_mount(int drive)
                 vol_drive[volume] = drive; /* remember the drive for this volume */
                 volume = get_free_volume(); /* prepare next entry */
                 disk_sector_multiplier[drive] = j;
+                success = true;
                 break;
             }
         }
-#else
+    #else /* ndef MAX_LOG_SECTOR_SIZE */
         if (!fat_mount(IF_MV(volume,) IF_MD(drive,) pinfo[i].start))
         {
             mounted++;
             vol_drive[volume] = drive; /* remember the drive for this volume */
             volume = get_free_volume(); /* prepare next entry */
+            success = true;
         }
-#endif
+    #endif /* MAX_LOG_SECTOR_SIZE */
+
+        if (success)
+            volume_onmount_internal(IF_MV(volume));
     }
 
     if (mounted == 0 && volume != -1) /* none of the 4 entries worked? */
     {   /* try "superfloppy" mode */
         DEBUGF("No partition found, trying to mount sector 0.\n");
+
         if (!fat_mount(IF_MV(volume,) IF_MD(drive,) 0))
         {
-#ifdef MAX_LOG_SECTOR_SIZE
-            disk_sector_multiplier[drive] = fat_get_bytes_per_sector(IF_MV(volume))/SECTOR_SIZE;
-#endif
+        #ifdef MAX_LOG_SECTOR_SIZE
+            disk_sector_multiplier[drive] =
+                fat_get_bytes_per_sector(IF_MV(volume)) / SECTOR_SIZE;
+        #endif
             mounted = 1;
             vol_drive[volume] = drive; /* remember the drive for this volume */
+            volume_onmount_internal(IF_MV(volume));
         }
     }
-#ifdef HAVE_HOTSWAP
-    mutex_unlock(&disk_mutex);
-#endif
+
+    disk_writer_unlock();
+    return mounted;
+}
+
+int disk_mount_all(void)
+{
+    int mounted = 0;
+
+    disk_writer_lock();
+
+    /* reset all mounted partitions */
+    volume_onunmount_internal(IF_MV(-1));
+    fat_init();
+
+    for (int i = 0; i < NUM_VOLUMES; i++)
+        vol_drive[i] = -1; /* mark all as unassigned */
+
+    for (int i = 0; i < NUM_DRIVES; i++)
+    {
+    #ifdef HAVE_HOTSWAP
+        if (storage_present(i))
+    #endif
+            mounted += disk_mount(i);
+    }
+
+    disk_writer_unlock();
     return mounted;
 }
 
 int disk_unmount(int drive)
 {
+    if (!CHECK_DRV(drive))
+        return 0;
+
     int unmounted = 0;
-    int i;
-#ifdef HAVE_HOTSWAP
-    mutex_lock(&disk_mutex);
-#endif
-    for (i=0; i<NUM_VOLUMES; i++)
+
+    disk_writer_lock();
+
+    for (int i = 0; i < NUM_VOLUMES; i++)
     {
         if (vol_drive[i] == drive)
         {   /* force releasing resources */
             vol_drive[i] = -1; /* mark unused */
+
+            volume_onunmount_internal(IF_MV(i));
+            fat_unmount(IF_MV(i));
+
             unmounted++;
-            release_files(i);
-            release_dirs(i);
-            fat_unmount(i, false);
         }
     }
-#ifdef HAVE_HOTSWAP
-    mutex_unlock(&disk_mutex);
-#endif
 
+    disk_writer_unlock();
     return unmounted;
 }
 
 int disk_unmount_all(void)
 {
-#ifndef HAVE_MULTIDRIVE
-    return disk_unmount(0);
-#else  /* HAVE_MULTIDRIVE */
     int unmounted = 0;
-    int i;
-    for (i = 0; i < NUM_DRIVES; i++)
+
+    disk_writer_lock();
+
+    volume_onunmount_internal(IF_MV(-1));
+
+    for (int i = 0; i < NUM_DRIVES; i++)
     {
-#ifdef HAVE_HOTSWAP
+    #ifdef HAVE_HOTSWAP
         if (storage_present(i))
-#endif
+    #endif
             unmounted += disk_unmount(i);
     }
 
+    disk_writer_unlock();
     return unmounted;
-#endif  /* HAVE_MULTIDRIVE */
+}
+
+bool disk_present(IF_MD_NONVOID(int drive))
+{
+    int rc = -1;
+
+    if (CHECK_DRV(drive))
+    {
+        void *sector = dc_get_buffer();
+        if (sector)
+        {
+            rc = storage_read_sectors(IF_MD(drive,) 0, 1, sector);
+            dc_release_buffer(sector);
+        }
+    }
+
+    return rc == 0;
+}
+
+
+/** Volume-centric functions **/
+
+void volume_recalc_free(IF_MV_NONVOID(int volume))
+{
+    if (!CHECK_VOL(volume))
+        return;
+
+    /* FIXME: this is crummy but the only way to ensure a correct freecount
+       if other threads are writing and changing the fsinfo; it is possible
+       to get multiple threads calling here and also writing and get correct
+       freespace counts, however a bit complicated to do; if thou desireth I
+       shall implement the concurrent version -- jethead71 */
+    disk_writer_lock();
+    fat_recalc_free(IF_MV(volume));
+    disk_writer_unlock();
+}
+
+unsigned int volume_get_cluster_size(IF_MV_NONVOID(int volume))
+{
+    if (!CHECK_VOL(volume))
+        return 0;
+
+    disk_reader_lock();
+    unsigned int clustersize = fat_get_cluster_size(IF_MV(volume));
+    disk_reader_unlock();
+    return clustersize;
+}
+
+void volume_size(IF_MV(int volume,) unsigned long *sizep, unsigned long *freep)
+{
+    disk_reader_lock();
+
+    if (!CHECK_VOL(volume) || !fat_size(IF_MV(volume,) sizep, freep))
+    {
+        if (freep) *sizep = 0;
+        if (freep) *freep = 0;
+    }
+
+    disk_reader_unlock();
+}
+
+#if defined (HAVE_HOTSWAP) || defined (HAVE_MULTIDRIVE) \
+    || defined (HAVE_DIRCACHE)
+enum volume_info_type
+{
+#ifdef HAVE_HOTSWAP
+    VP_REMOVABLE,
+    VP_PRESENT,
+#endif
+#if defined (HAVE_MULTIDRIVE) || defined (HAVE_DIRCACHE)
+    VP_DRIVE,
+#endif
+};
+
+static int volume_properties(int volume, enum volume_info_type infotype)
+{
+    int res = -1;
+
+    disk_reader_lock();
+
+    if (CHECK_VOL(volume))
+    {
+        int vd = vol_drive[volume];
+        switch (infotype)
+        {
+    #ifdef HAVE_HOTSWAP
+        case VP_REMOVABLE:
+            res = storage_removable(vd) ? 1 : 0;
+            break;
+        case VP_PRESENT:
+            res = storage_present(vd) ? 1 : 0;
+            break;
+    #endif
+    #if defined(HAVE_MULTIDRIVE) || defined(HAVE_DIRCACHE)
+        case VP_DRIVE:
+            res = vd;
+            break;
+    #endif
+        }
+    }
+
+    disk_reader_unlock();
+    return res;
 }
 
 #ifdef HAVE_HOTSWAP
 bool volume_removable(int volume)
 {
-    if(vol_drive[volume] == -1)
-        return false;
-    return storage_removable(vol_drive[volume]);
+    return volume_properties(volume, VP_REMOVABLE) > 0;
 }
 
 bool volume_present(int volume)
 {
-    if(vol_drive[volume] == -1)
-        return false;
-    return storage_present(vol_drive[volume]);
+    return volume_properties(volume, VP_PRESENT) > 0;
 }
-#endif
+#endif /* HAVE_HOTSWAP */
+
+#ifdef HAVE_MULTIDRIVE
+int volume_drive(int volume)
+{
+    return volume_properties(volume, VP_DRIVE);
+}
+#endif /* HAVE_MULTIDRIVE */
+
+#ifdef HAVE_DIRCACHE
+bool volume_ismounted(IF_MV_NONVOID(int volume))
+{
+    return volume_properties(IF_MV_VOL(volume), VP_DRIVE) >= 0;
+}
+#endif /* HAVE_DIRCACHE */
+
+#endif /* HAVE_HOTSWAP || HAVE_MULTIDRIVE || HAVE_DIRCACHE */
diff --git a/firmware/common/disk_cache.c b/firmware/common/disk_cache.c
new file mode 100644
index 0000000000..0e842e7796
--- /dev/null
+++ b/firmware/common/disk_cache.c
@@ -0,0 +1,343 @@
+/***************************************************************************
+ *             __________               __   ___.
+ *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
+ *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
+ *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
+ *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
+ *                     \/            \/     \/    \/            \/
+ * $Id$
+ *
+ * Copyright (C) 2014 by Michael Sevakis
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ****************************************************************************/
+#include "config.h"
+#include "debug.h"
+#include "system.h"
+#include "linked_list.h"
+#include "disk_cache.h"
+#include "fat.h" /* for SECTOR_SIZE */
+#include "bitarray.h"
+
+/* Cache: LRU cache with separately-chained hashtable
+ *
+ * Each entry of the map is the mapped location of the hashed sector value
+ * where each bit in each map entry indicates which corresponding cache
+ * entries are occupied by sector values that collide in that map entry.
+ *
+ * Each volume is given its own bit map.
+ *
+ * To probe for a specific key, each bit in the map entry must be examined,
+ * its position used as an index into the cache_entry array and the actual
+ * sector information compared for that cache entry. If the search exhausts
+ * all bits, the sector is not cached.
+ *
+ * To avoid long chains, the map entry count should be much greater than the
+ * number of cache entries. Since the cache is an LRU design, no buffer entry
+ * in the array is intrinsically associated with any particular sector number
+ * or volume.
+ *
+ * Example 6-sector cache with 8-entry map:
+ * cache entry 543210
+ * cache map   100000 <- sector number hashes into map
+ *             000000
+ *             000100
+ *             000000
+ *             010000
+ *             000000
+ *             001001 <- collision
+ *             000000
+ * volume map  111101 <- entry usage by the volume (OR of all map entries)
+ */
+
+enum dce_flags /* flags for each cache entry */
+{
+    DCE_INUSE = 0x01, /* entry in use and valid */
+    DCE_DIRTY = 0x02, /* entry is dirty in need of writeback */
+    DCE_BUF   = 0x04, /* entry is being used as a general buffer */
+};
+
+struct disk_cache_entry
+{
+    struct lldc_node node;  /* LRU list links */
+    unsigned char flags;    /* entry flags */
+#ifdef HAVE_MULTIVOLUME
+    unsigned char volume;   /* volume of sector */
+#endif
+    unsigned long sector;   /* cached disk sector number */
+};
+
+BITARRAY_TYPE_DECLARE(cache_map_entry_t, cache_map, DC_NUM_ENTRIES)
+
+static inline unsigned int map_sector(unsigned long sector)
+{
+    /* keep sector hash simple for now */
+    return sector % DC_MAP_NUM_ENTRIES;
+}
+
+static struct lldc_head cache_lru; /* LRU cache list (head = LRU item) */
+static struct disk_cache_entry cache_entry[DC_NUM_ENTRIES];
+static cache_map_entry_t cache_map_entry[NUM_VOLUMES][DC_MAP_NUM_ENTRIES];
+static cache_map_entry_t cache_vol_map[NUM_VOLUMES] IBSS_ATTR;
+static uint8_t cache_buffer[DC_NUM_ENTRIES][DC_CACHE_BUFSIZE] CACHEALIGN_ATTR;
+struct mutex disk_cache_mutex SHAREDBSS_ATTR;
+
+#define CACHE_MAP_ENTRY(volume, mapnum) \
+    cache_map_entry[IF_MV_VOL(volume)][mapnum]
+#define CACHE_VOL_MAP(volume) \
+    cache_vol_map[IF_MV_VOL(volume)]
+
+#define DCE_LRU()      ((struct disk_cache_entry *)cache_lru.head)
+#define DCE_NEXT(fce)  ((struct disk_cache_entry *)(fce)->node.next)
+#define NODE_DCE(node) ((struct disk_cache_entry *)(node))
+
+/* get the cache index from a pointer to a buffer */
+#define DCIDX_FROM_BUF(buf) \
+    ((uint8_t (*)[DC_CACHE_BUFSIZE])(buf) - cache_buffer)
+
+#define DCIDX_FROM_DCE(dce) \
+    ((dce) - cache_entry)
+
+/* set the in-use bit in the map */
+static inline void cache_bitmap_set_bit(int volume, unsigned int mapnum,
+                                        unsigned int bitnum)
+{
+    cache_map_set_bit(&CACHE_MAP_ENTRY(volume, mapnum), bitnum);
+    cache_map_set_bit(&CACHE_VOL_MAP(volume), bitnum);
+    (void)volume;
+}
+
+/* clear the in-use bit in the map */
+static inline void cache_bitmap_clear_bit(int volume, unsigned int mapnum,
+                                          unsigned int bitnum)
+{
+    cache_map_clear_bit(&CACHE_MAP_ENTRY(volume, mapnum), bitnum);
+    cache_map_clear_bit(&CACHE_VOL_MAP(volume), bitnum);
+    (void)volume;
+}
+
+/* make entry MRU by moving it to the list tail */
+static inline void touch_cache_entry(struct disk_cache_entry *which)
+{
+    struct lldc_node *lru = cache_lru.head;
+    struct lldc_node *node = &which->node;
+
+    if (node == lru->prev)  /* already MRU */
+        ; /**/
+    else if (node == lru)   /* is the LRU? just rotate list */
+        cache_lru.head = lru->next;
+    else                    /* somewhere else; move it */
+    {
+        lldc_remove(&cache_lru, node);
+        lldc_insert_last(&cache_lru, node);
+    }
+}
+
+/* remove LRU entry from the cache list to use as a buffer */
+static struct disk_cache_entry * cache_remove_lru_entry(void)
+{
+    struct lldc_node *lru = cache_lru.head;
+
+    /* at least one is reserved for client */
+    if (lru == lru->next)
+        return NULL;
+
+    /* remove it; next-LRU becomes the LRU */
+    lldc_remove(&cache_lru, lru);
+    return NODE_DCE(lru);
+}
+
+/* return entry to the cache list and set it LRU */
+static void cache_return_lru_entry(struct disk_cache_entry *fce)
+{
+    lldc_insert_first(&cache_lru, &fce->node);
+}
+
+/* discard the entry's data and mark it unused */
+static inline void cache_discard_entry(struct disk_cache_entry *dce,
+                                       unsigned int index)
+{
+    cache_bitmap_clear_bit(IF_MV_VOL(dce->volume), map_sector(dce->sector),
+                           index);
+    dce->flags = 0;
+}
+
+/* search the cache for the specified sector, returning a buffer, either
+   to the specified sector, if it exists, or a new/evicted entry that must
+   be filled */
+void * dc_cache_probe(IF_MV(int volume,) unsigned long sector,
+                      unsigned int *flagsp)
+{
+    unsigned int mapnum = map_sector(sector);
+
+    FOR_EACH_BITARRAY_SET_BIT(&CACHE_MAP_ENTRY(volume, mapnum), index)
+    {
+        struct disk_cache_entry *dce = &cache_entry[index];
+
+        if (dce->sector == sector)
+        {
+            *flagsp = DCE_INUSE;
+            touch_cache_entry(dce);
+            return cache_buffer[index];
+        }
+    }
+
+    /* sector not found so the LRU is the victim */
+    struct disk_cache_entry *dce = DCE_LRU();
+    cache_lru.head = dce->node.next;
+
+    unsigned int index = DCIDX_FROM_DCE(dce);
+    void *buf = cache_buffer[index];
+    unsigned int old_flags = dce->flags;
+
+    if (old_flags)
+    {
+        int old_volume = IF_MV_VOL(dce->volume);
+        unsigned long sector = dce->sector;
+        unsigned int old_mapnum = map_sector(sector);
+
+        if (old_flags & DCE_DIRTY)
+            dc_writeback_callback(IF_MV(old_volume,) sector, buf);
+
+        if (mapnum == old_mapnum IF_MV( && volume == old_volume ))
+            goto finish_setup;
+
+        cache_bitmap_clear_bit(old_volume, old_mapnum, index);
+    }
+
+    cache_bitmap_set_bit(IF_MV_VOL(volume), mapnum, index);
+
+finish_setup:
+    dce->flags  = DCE_INUSE;
+#ifdef HAVE_MULTIVOLUME
+    dce->volume = volume;
+#endif
+    dce->sector = sector;
+
+    *flagsp = 0;
+    return buf;
+}
+
+/* mark in-use cache entry as dirty by buffer */
+void dc_dirty_buf(void *buf)
+{
+    unsigned int index = DCIDX_FROM_BUF(buf);
+
+    if (index >= DC_NUM_ENTRIES)
+        return;
+
+    /* dirt remains, sticky until flushed */
+    struct disk_cache_entry *fce = &cache_entry[index];
+    if (fce->flags & DCE_INUSE)
+        fce->flags |= DCE_DIRTY;
+}
+
+/* discard in-use cache entry by buffer */
+void dc_discard_buf(void *buf)
+{
+    unsigned int index = DCIDX_FROM_BUF(buf);
+
+    if (index >= DC_NUM_ENTRIES)
+        return;
+
+    struct disk_cache_entry *dce = &cache_entry[index];
+    if (dce->flags & DCE_INUSE)
+        cache_discard_entry(dce, index);
+}
+
+/* commit all dirty cache entries to storage for a specified volume */
+void dc_commit_all(IF_MV_NONVOID(int volume))
+{
+    DEBUGF("dc_commit_all()\n");
+
+    FOR_EACH_BITARRAY_SET_BIT(&CACHE_VOL_MAP(volume), index)
+    {
+        struct disk_cache_entry *dce = &cache_entry[index];
+        unsigned int flags = dce->flags;
+
+        if (flags & DCE_DIRTY)
+        {
+            dc_writeback_callback(IF_MV(volume,) dce->sector,
+                                  cache_buffer[index]);
+            dce->flags = flags & ~DCE_DIRTY;
+        }
+    }
+}
+
+/* discard all cache entries from the specified volume */
+void dc_discard_all(IF_MV_NONVOID(int volume))
+{
+    DEBUGF("dc_discard_all()\n");
+
+    FOR_EACH_BITARRAY_SET_BIT(&CACHE_VOL_MAP(volume), index)
+        cache_discard_entry(&cache_entry[index], index);
+}
+
+/* expropriate a buffer from the cache */
+void * dc_get_buffer(void)
+{
+    dc_lock_cache();
+
+    void *buf = NULL;
+    struct disk_cache_entry *dce = cache_remove_lru_entry();
+
+    if (dce)
+    {
+        unsigned int index = DCIDX_FROM_DCE(dce);
+        unsigned int flags = dce->flags;
+
+        buf = cache_buffer[index];
+
+        if (flags)
+        {
+            /* must first commit this sector if dirty */
+            if (flags & DCE_DIRTY)
+                dc_writeback_callback(IF_MV(dce->volume,) dce->sector, buf);
+
+            cache_discard_entry(dce, index);
+        }
+
+        dce->flags = DCE_BUF;
+    }
+    /* cache is out of buffers */
+
+    dc_unlock_cache();
+    return buf;
+}
+
+/* return buffer to the cache by buffer */
+void dc_release_buffer(void *buf)
+{
+    unsigned int index = DCIDX_FROM_BUF(buf);
+
+    if (index >= DC_NUM_ENTRIES)
+        return;
+
+    dc_lock_cache();
+
+    struct disk_cache_entry *dce = &cache_entry[index];
+
+    if (dce->flags & DCE_BUF)
+    {
+        dce->flags = 0;
+        cache_return_lru_entry(dce);
+    }
+
+    dc_unlock_cache();
+}
+
+/* one-time init at startup */
+void dc_init(void)
+{
+    mutex_init(&disk_cache_mutex);
+    lldc_init(&cache_lru);
+    for (unsigned int i = 0; i < DC_NUM_ENTRIES; i++)
+        lldc_insert_last(&cache_lru, &cache_entry[i].node);
+}
diff --git a/firmware/common/file.c b/firmware/common/file.c
index 920eada84e..7d3b5092ae 100644
--- a/firmware/common/file.c
+++ b/firmware/common/file.c
@@ -8,6 +8,7 @@
  * $Id$
  *
  * Copyright (C) 2002 by Björn Stenberg
+ * Copyright (C) 2014 by Michael Sevakis
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -18,819 +19,1201 @@
  * KIND, either express or implied.
  *
  ****************************************************************************/
+#define RB_FILESYSTEM_OS
+#include "config.h"
+#include "system.h"
 #include <string.h>
 #include <errno.h>
-#include <stdbool.h>
-#include "file.h"
-#include "fat.h"
-#include "dir_uncached.h"
 #include "debug.h"
-#include "dircache.h"
-#include "filefuncs.h"
-#include "system.h"
+#include "file.h"
+#include "fileobj_mgr.h"
+#include "disk_cache.h"
+#include "dircache_redirect.h"
+#include "string-extra.h"
+
+/**
+ * These functions provide a roughly POSIX-compatible file I/O API.
+ */
+
+/* structure used for open file descriptors */
+static struct filestr_desc
+{
+    struct filestr_base stream; /* basic stream info (first!) */
+    file_size_t         offset; /* current offset for stream */
+    file_size_t         *sizep; /* shortcut to file size in fileobj */
+} open_streams[MAX_OPEN_FILES];
 
-/*
-  These functions provide a roughly POSIX-compatible file IO API.
+/* check and return a struct filestr_desc* from a file descriptor number */
+static struct filestr_desc * get_filestr(int fildes)
+{
+    struct filestr_desc *file = &open_streams[fildes];
 
-  Since the fat32 driver only manages sectors, we maintain a one-sector
-  cache for each open file. This way we can provide byte access without
-  having to re-read the sector each time. 
-  The penalty is the RAM used for the cache and slightly more complex code.
-*/
+    if ((unsigned int)fildes >= MAX_OPEN_FILES)
+        file = NULL;
+    else if (file->stream.flags & FDO_BUSY)
+        return file;
 
-struct filedesc {
-    unsigned char cache[SECTOR_SIZE] CACHEALIGN_ATTR;
-    int cacheoffset; /* invariant: 0 <= cacheoffset <= SECTOR_SIZE */
-    long fileoffset;
-    long size;
-    int attr;
-    struct fat_file fatfile;
-    bool busy;
-    bool write;
-    bool dirty;
-    bool trunc;
-} CACHEALIGN_ATTR;
+    DEBUGF("fildes %d: bad file number\n", fildes);
+    errno = (file && file->stream.flags == FV_NONEXIST) ? ENXIO : EBADF;
+    return NULL;
+}
 
-static struct filedesc openfiles[MAX_OPEN_FILES] CACHEALIGN_ATTR;
+#define GET_FILESTR(type, fildes) \
+    ({                                                     \
+        file_internal_lock_##type();                       \
+        struct filestr_desc * _file = get_filestr(fildes); \
+        if (_file)                                         \
+            FILESTR_LOCK(type, &_file->stream);            \
+        else                                               \
+            file_internal_unlock_##type();                 \
+        _file;                                             \
+    })
+
+/* release the lock on the filestr_desc* */
+#define RELEASE_FILESTR(type, file) \
+    ({                                         \
+        FILESTR_UNLOCK(type, &(file)->stream); \
+        file_internal_unlock_##type();         \
+    })
+
+/* find a free file descriptor */
+static int alloc_filestr(struct filestr_desc **filep)
+{
+    for (int fildes = 0; fildes < MAX_OPEN_FILES; fildes++)
+    {
+        struct filestr_desc *file = &open_streams[fildes];
+        if (!file->stream.flags)
+        {
+            *filep = file;
+            return fildes;
+        }
+    }
 
-static int flush_cache(int fd);
+    DEBUGF("Too many files open\n");
+    return -1;
+}
 
-int file_creat(const char *pathname)
+/* return the file size in sectors */
+static inline unsigned long filesize_sectors(file_size_t size)
 {
-    return open(pathname, O_WRONLY|O_CREAT|O_TRUNC, 0666);
+    /* overflow proof whereas "(x + y - 1) / y" is not */
+    unsigned long numsectors = size / SECTOR_SIZE;
+
+    if (size % SECTOR_SIZE)
+        numsectors++;
+
+    return numsectors;
 }
 
-static int open_internal(const char* pathname, int flags, bool use_cache)
+/* flush a dirty cache buffer */
+static int flush_cache(struct filestr_desc *file)
 {
-    DIR_UNCACHED* dir;
-    struct dirent_uncached* entry;
-    int fd;
-    int pathnamesize = strlen(pathname) + 1;
-    char pathnamecopy[pathnamesize];
-    char* name;
-    struct filedesc* file = NULL;
     int rc;
-#ifndef HAVE_DIRCACHE
-    (void)use_cache;
-#endif
+    struct filestr_cache *cachep = file->stream.cachep;
 
-    LDEBUGF("open(\"%s\",%d)\n",pathname,flags);
+    DEBUGF("Flushing dirty sector cache (%lu)\n", cachep->sector);
 
-    if ( pathname[0] != '/' ) {
-        DEBUGF("'%s' is not an absolute path.\n",pathname);
-        DEBUGF("Only absolute pathnames supported at the moment\n");
-        errno = EINVAL;
-        return -1;
+    if (fat_query_sectornum(&file->stream.fatstr) != cachep->sector)
+    {
+        /* get on the correct sector */
+        rc = fat_seek(&file->stream.fatstr, cachep->sector);
+        if (rc < 0)
+            FILE_ERROR(EIO, rc * 10 - 1);
     }
 
-    /* find a free file descriptor */
-    for ( fd=0; fd<MAX_OPEN_FILES; fd++ )
-        if ( !openfiles[fd].busy )
-            break;
-
-    if ( fd == MAX_OPEN_FILES ) {
-        DEBUGF("Too many files open\n");
-        errno = EMFILE;
-        return -2;
+    rc = fat_readwrite(&file->stream.fatstr, 1, cachep->buffer, true);
+    if (rc < 0)
+    {
+        if (rc == FAT_RC_ENOSPC)
+            FILE_ERROR(ENOSPC, RC);
+        else
+            FILE_ERROR(EIO, rc * 10 - 2);
     }
 
-    file = &openfiles[fd];
-    memset(file, 0, sizeof(struct filedesc));
+    cachep->flags = 0;
+    return 1;
+file_error:
+    DEBUGF("Failed flushing cache: %d\n", rc);
+    return rc;
+}
+
+static void discard_cache(struct filestr_desc *file)
+{
+    struct filestr_cache *const cachep = file->stream.cachep;
+    cachep->flags = 0;
+}
 
-    if (flags & (O_RDWR | O_WRONLY)) {
-        file->write = true;
+/* set the file pointer */
+static off_t lseek_internal(struct filestr_desc *file, off_t offset,
+                            int whence)
+{
+    off_t rc;
+    file_size_t pos;
 
-        if (flags & O_TRUNC)
-            file->trunc = true;
-    }
-    file->busy = true;
+    file_size_t size = MIN(*file->sizep, FILE_SIZE_MAX);
 
-#ifdef HAVE_DIRCACHE
-    if (dircache_is_enabled() && !file->write && use_cache)
+    switch (whence)
     {
-# ifdef HAVE_MULTIVOLUME
-        int volume = strip_volume(pathname, pathnamecopy);
-# endif
+    case SEEK_SET:
+        if (offset < 0 || (file_size_t)offset > size)
+            FILE_ERROR(EINVAL, -1);
 
-        int ce = dircache_get_entry_id(pathname);
-        if (ce < 0)
-        {
-            errno = ENOENT;
-            file->busy = false;
-            return -7;
-        }
+        pos = offset;
+        break;
 
-        long startcluster = _dircache_get_entry_startcluster(ce);
-        fat_open(IF_MV(volume,)
-                 startcluster,
-                 &(file->fatfile),
-                 NULL);
-        struct dirinfo *info = _dircache_get_entry_dirinfo(ce);
-        file->size = info->size;
-        file->attr = info->attribute;
-        file->cacheoffset = -1;
-        file->fileoffset = 0;
+    case SEEK_CUR:
+        if ((offset < 0 && (file_size_t)-offset > file->offset) ||
+            (offset > 0 && (file_size_t)offset > size - file->offset))
+            FILE_ERROR(EINVAL, -1);
 
-        return fd;
-    }
-#endif
+        pos = file->offset + offset;
+        break;
 
-    strlcpy(pathnamecopy, pathname, pathnamesize);
+    case SEEK_END:
+        if (offset > 0 || (file_size_t)-offset > size)
+            FILE_ERROR(EINVAL, -1);
 
-    /* locate filename */
-    name=strrchr(pathnamecopy+1,'/');
-    if ( name ) {
-        *name = 0; 
-        dir = opendir_uncached(pathnamecopy);
-        *name = '/';
-        name++;
-    }
-    else {
-        dir = opendir_uncached("/");
-        name = pathnamecopy+1;
-    }
-    if (!dir) {
-        DEBUGF("Failed opening dir\n");
-        errno = EIO;
-        file->busy = false;
-        return -4;
-    }
-
-    if(name[0] == 0) {
-        DEBUGF("Empty file name\n");
-        errno = EINVAL;
-        file->busy = false;
-        closedir_uncached(dir);
-        return -5;
-    }
-
-    /* scan dir for name */
-    while ((entry = readdir_uncached(dir))) {
-        if ( !strcasecmp(name, entry->d_name) ) {
-            fat_open(IF_MV(dir->fatdir.file.volume,)
-                     entry->startcluster,
-                     &(file->fatfile),
-                     &(dir->fatdir));
-            file->size = file->trunc ? 0 : entry->info.size;
-            file->attr = entry->info.attribute;
-            break;
-        }
+        pos = size + offset;
+        break;
+
+    default:
+        FILE_ERROR(EINVAL, -1);
     }
 
-    if ( !entry ) {
-        LDEBUGF("Didn't find file %s\n",name);
-        if ( file->write && (flags & O_CREAT) ) {
-            rc = fat_create_file(name,
-                                 &(file->fatfile),
-                                 &(dir->fatdir));
-            if (rc < 0) {
-                DEBUGF("Couldn't create %s in %s\n",name,pathnamecopy);
-                errno = EIO;
-                file->busy = false;
-                closedir_uncached(dir);
-                return rc * 10 - 6;
+    file->offset = pos;
+
+    return pos;
+file_error:
+    return rc;
+}
+
+/* callback for each file stream to make sure all data is in sync with new
+   size */
+void ftruncate_internal_callback(struct filestr_base *stream,
+                                 struct filestr_base *s)
+{
+    struct filestr_desc *file = (struct filestr_desc *)s;
+    file_size_t size = *file->sizep;
+
+    /* caches with data beyond new extents are invalid */
+    unsigned long sector = file->stream.cachep->sector;
+    if (sector != INVALID_SECNUM && sector >= filesize_sectors(size))
+        filestr_discard_cache(&file->stream);
+
+    /* keep all positions within bounds */
+    if (file->offset > size)
+        file->offset = size;
+
+    (void)stream;
+}
+
+/* truncate the file to the specified length */
+static int ftruncate_internal(struct filestr_desc *file, file_size_t size,
+                              bool write_now)
+{
+    int rc = 0, rc2 = 1;
+
+    file_size_t cursize = *file->sizep;
+    file_size_t truncsize = MIN(size, cursize);
+
+    if (write_now)
+    {
+        unsigned long sector = filesize_sectors(truncsize);
+        struct filestr_cache *const cachep = file->stream.cachep;
+
+        if (cachep->flags == (FSC_NEW|FSC_DIRTY) &&
+            cachep->sector + 1 == sector)
+        {
+            /* sector created but may have never been added to the cluster
+               chain; flush it now or the subsequent may fail */
+            rc2 = flush_cache(file);
+            if (rc2 == FAT_RC_ENOSPC)
+            {
+                /* no space left on device; further truncation needed */
+                discard_cache(file);
+                truncsize = ALIGN_DOWN(truncsize - 1, SECTOR_SIZE);
+                sector--;
+                rc = rc2;
             }
-#ifdef HAVE_DIRCACHE
-            dircache_add_file(pathname, file->fatfile.firstcluster);
-#endif
-            file->size = 0;
-            file->attr = 0;
-        }
-        else {
-            DEBUGF("Couldn't find %s in %s\n",name,pathnamecopy);
-            errno = ENOENT;
-            file->busy = false;
-            closedir_uncached(dir);
-            return -7;
-        }
-    } else {
-        if(file->write && (file->attr & FAT_ATTR_DIRECTORY)) {
-            errno = EISDIR;
-            file->busy = false;
-            closedir_uncached(dir);
-            return -8;
+            else if (rc2 < 0)
+                FILE_ERROR(ERRNO, rc2 * 10 - 1);
         }
-    }
-    closedir_uncached(dir);
 
-    file->cacheoffset = -1;
-    file->fileoffset = 0;
+        rc2 = fat_seek(&file->stream.fatstr, sector);
+        if (rc2 < 0)
+            FILE_ERROR(EIO, rc2 * 10 - 2);
 
-    if (file->write && (flags & O_APPEND)) {
-        rc = lseek(fd,0,SEEK_END);
-        if (rc < 0 )
-            return rc * 10 - 9;
+        rc2 = fat_truncate(&file->stream.fatstr);
+        if (rc2 < 0)
+            FILE_ERROR(EIO, rc2 * 10 - 3);
     }
+    /* else just change the cached file size */
 
-#ifdef HAVE_DIRCACHE
-    if (file->write)
-        dircache_bind(fd, pathname);
-#endif
+    if (truncsize < cursize)
+    {
+        *file->sizep = truncsize;
+        fileop_ontruncate_internal(&file->stream);
+    }
 
-    return fd;
-}
+    /* if truncation was partially successful, it effectively destroyed
+       everything after the truncation point; still, indicate failure
+       after adjusting size */
+    if (rc2 == 0)
+        FILE_ERROR(EIO, -4);
+    else if (rc2 < 0)
+        FILE_ERROR(ERRNO, rc2);
 
-int file_open(const char* pathname, int flags)
-{
-    /* By default, use the dircache if available. */
-    return open_internal(pathname, flags, true);
+file_error:
+    return rc;
 }
 
-int close(int fd)
+/* flush back all outstanding writes to the file */
+static int fsync_internal(struct filestr_desc *file)
 {
-    struct filedesc* file = &openfiles[fd];
+    /* call only when holding WRITER lock (updates directory entries) */
     int rc = 0;
 
-    LDEBUGF("close(%d)\n", fd);
+    file_size_t size = *file->sizep;
+    unsigned int foflags = fileobj_get_flags(&file->stream);
+
+    /* flush sector cache? */
+    struct filestr_cache *const cachep = file->stream.cachep;
+    if (cachep->flags & FSC_DIRTY)
+    {
+        int rc2 = flush_cache(file);
+        if (rc2 == FAT_RC_ENOSPC && (cachep->flags & FSC_NEW))
+        {
+            /* no space left on device so this must be dropped */
+            discard_cache(file);
+            size = ALIGN_DOWN(size - 1, SECTOR_SIZE);
+            foflags |= FO_TRUNC;
+            rc = rc2;
+        }
+        else if (rc2 < 0)
+            FILE_ERROR(ERRNO, rc2 * 10 - 1);
+    }
+
+    /* truncate? */
+    if (foflags & FO_TRUNC)
+    {
+        int rc2 = ftruncate_internal(file, size, rc == 0);
+        if (rc2 < 0)
+            FILE_ERROR(ERRNO, rc2 * 10 - 2);
 
-    if (fd < 0 || fd > MAX_OPEN_FILES-1) {
-        errno = EINVAL;
-        return -1;
+        /* never needs to be done this way again since any data beyond the
+           cached size is now gone */
+        fileobj_change_flags(&file->stream, 0, FO_TRUNC);
     }
-    if (!file->busy) {
-        errno = EBADF;
-        return -2;
+
+file_error:;
+    /* tie up all loose ends (try to close the file even if failing) */
+    int rc2 = fat_closewrite(&file->stream.fatstr, size,
+                             get_dir_fatent_dircache());
+    if (rc2 >= 0)
+        fileop_onsync_internal(&file->stream); /* dir_fatent is implicit arg */
+
+    if (rc2 < 0 && rc >= 0)
+    {
+        errno = EIO;
+        rc = rc2 * 10 - 3;
     }
-    if (file->write) {
-        rc = fsync(fd);
+
+    return rc;
+}
+
+/* finish with the file and free resources */
+static int close_internal(struct filestr_desc *file)
+{
+    /* call only when holding WRITER lock (updates directory entries) */
+    int rc;
+
+    if ((file->stream.flags & FD_WRITE) &&
+        !(fileobj_get_flags(&file->stream) & FO_REMOVED))
+    {
+        rc = fsync_internal(file);
         if (rc < 0)
-            return rc * 10 - 3;
-#ifdef HAVE_DIRCACHE
-        dircache_update_filesize(fd, file->size, file->fatfile.firstcluster);
-        dircache_update_filetime(fd);
-#endif
+            FILE_ERROR(ERRNO, rc * 10 - 1);
     }
 
-    file->busy = false;
-    return 0;
+    rc = 0;
+file_error:;
+    int rc2 = close_stream_internal(&file->stream);
+    if (rc2 < 0 && rc >= 0)
+        rc = rc2 * 10 - 2;
+    return rc;
 }
 
-int fsync(int fd)
+/* actually do the open gruntwork */
+static int open_internal_inner2(const char *path,
+                                struct filestr_desc *file,
+                                unsigned int callflags)
 {
-    struct filedesc* file = &openfiles[fd];
-    int rc = 0;
-
-    LDEBUGF("fsync(%d)\n", fd);
+    int rc;
 
-    if (fd < 0 || fd > MAX_OPEN_FILES-1) {
-        errno = EINVAL;
-        return -1;
-    }
-    if (!file->busy) {
-        errno = EBADF;
-        return -2;
+    struct path_component_info compinfo;
+    rc = open_stream_internal(path, callflags, &file->stream, &compinfo);
+    if (rc < 0)
+    {
+        DEBUGF("Open failed: %d\n", rc);
+        FILE_ERROR_RETURN(ERRNO, rc * 10 - 1);
     }
-    if (file->write) {
-        /* flush sector cache */
-        if ( file->dirty ) {
-            rc = flush_cache(fd);
-            if (rc < 0)
-            {
-                /* when failing, try to close the file anyway */
-                fat_closewrite(&(file->fatfile), file->size, file->attr);
-                return rc * 10 - 3;
-            }
+
+    bool created = false;
+
+    if (rc > 0)
+    {
+        if (callflags & FF_EXCL)
+        {
+            DEBUGF("File exists\n");
+            FILE_ERROR(EEXIST, -2);
         }
 
-        /* truncate? */
-        if (file->trunc) {
-            rc = ftruncate(fd, file->size);
-            if (rc < 0)
+        if (compinfo.attr & ATTR_DIRECTORY)
+        {
+            if ((callflags & FD_WRITE) || !(callflags & FF_ANYTYPE))
             {
-                /* when failing, try to close the file anyway */
-                fat_closewrite(&(file->fatfile), file->size, file->attr);
-                return rc * 10 - 4;
+                DEBUGF("File is a directory\n");
+                FILE_ERROR(EISDIR, -3);
             }
+
+            compinfo.filesize = MAX_DIRECTORY_SIZE; /* allow file ops */
+        }
+    }
+    else if (callflags & FF_CREAT)
+    {
+        if (compinfo.attr & ATTR_DIRECTORY)
+        {
+            DEBUGF("File is a directory\n");
+            FILE_ERROR(EISDIR, -5);
         }
 
-        /* tie up all loose ends */
-        rc = fat_closewrite(&(file->fatfile), file->size, file->attr);
+        /* not found; try to create it */
+
+        callflags &= ~FO_TRUNC;
+        rc = create_stream_internal(&compinfo.parentinfo, compinfo.name, 
+                                    compinfo.length, ATTR_NEW_FILE, callflags,
+                                    &file->stream);
         if (rc < 0)
-            return rc * 10 - 5;
-    }
-    return 0;
-}
+            FILE_ERROR(ERRNO, rc * 10 - 6);
 
-int remove(const char* name)
-{
-    int rc;
-    struct filedesc* file;
-    /* Can't use dircache now, because we need to access the fat structures. */
-    int fd = open_internal(name, O_WRONLY, false);
-    if ( fd < 0 )
-        return fd * 10 - 1;
-
-    file = &openfiles[fd];
-#ifdef HAVE_DIRCACHE
-    dircache_remove(name);
-#endif
-    rc = fat_remove(&(file->fatfile));
-    if ( rc < 0 ) {
-        DEBUGF("Failed removing file: %d\n", rc);
-        errno = EIO;
-        return rc * 10 - 3;
+        created = true;
     }
+    else
+    {
+        DEBUGF("File not found\n");
+        FILE_ERROR(ENOENT, -7);
+    }
+
+    fat_rewind(&file->stream.fatstr);
+    file->sizep = fileobj_get_sizep(&file->stream);
+    file->offset = 0;
+
+    if (!created)
+    {
+        /* size from storage applies to first stream only otherwise it's
+           already up to date */
+        const bool first = fileobj_get_flags(&file->stream) & FO_SINGLE;
+        if (first)
+            *file->sizep = compinfo.filesize;
 
-    file->size = 0;
+        if (callflags & FO_TRUNC)
+        {
+            /* if the file is kind of "big" then free some space now */
+            rc = ftruncate_internal(file, 0, *file->sizep >= O_TRUNC_THRESH);
+            if (rc < 0)
+            {
+                DEBUGF("O_TRUNC failed: %d\n", rc);
+                FILE_ERROR(ERRNO, rc * 10 - 4);
+            }
+        }
+    }
 
-    rc = close(fd);
-    if (rc<0)
-        return rc * 10 - 4;
+    rc = 0;
+file_error:
+    if (rc < 0)
+        close_stream_internal(&file->stream);
 
-    return 0;
+    return rc;
 }
 
-int rename(const char* path, const char* newpath)
+/* allocate a file descriptor, if needed, assemble stream flags and open
+   a new stream */
+static int open_internal_inner1(const char *path, int oflag,
+                                unsigned int callflags)
 {
-    int rc, fd;
-    DIR_UNCACHED* dir;
-    char* nameptr;
-    char* dirptr;
-    struct filedesc* file;
-    char newpath2[MAX_PATH];
+    DEBUGF("%s(path=\"%s\",oflag=%X,callflags=%X)\n", __func__,
+           path, oflag, callflags);
 
-    /* verify new path does not already exist */
-    /* If it is a directory, errno == EISDIR if the name exists */
-    fd = open(newpath, O_RDONLY);
-    if ( fd >= 0 || errno == EISDIR) {
-        close(fd);
-        errno = EBUSY;
-        return -1;
-    }
-    close(fd);
+    int rc;
 
-    fd = open_internal(path, O_RDONLY, false);
-    if ( fd < 0 ) {
-        errno = EIO;
-        return fd * 10 - 2;
-    }
+    struct filestr_desc *file;
+    int fildes = alloc_filestr(&file);
+    if (fildes < 0)
+        FILE_ERROR(EMFILE, -1);
 
-    /* extract new file name */
-    nameptr = strrchr(newpath,'/');
-    if (nameptr)
-        nameptr++;
-    else {
-        close(fd);
-        return - 3;
-    }
+    callflags &= ~FDO_MASK;
 
-    /* Extract new path */
-    strcpy(newpath2, newpath);
+    if (oflag & O_ACCMODE)
+    {
+        callflags |= FD_WRITE;
 
-    dirptr = strrchr(newpath2,'/');
-    if(dirptr)
-        *dirptr = 0;
-    else {
-        close(fd);
-        return - 4;
-    }
+        if ((oflag & O_ACCMODE) == O_WRONLY)
+            callflags |= FD_WRONLY;
 
-    dirptr = newpath2;
+        if (oflag & O_APPEND)
+            callflags |= FD_APPEND;
 
-    if(strlen(dirptr) == 0) {
-        dirptr = "/";
+        if (oflag & O_TRUNC)
+            callflags |= FO_TRUNC;
     }
-
-    dir = opendir_uncached(dirptr);
-    if(!dir) {
-        close(fd);
-        return - 5;
+    else if (oflag & O_TRUNC)
+    {
+        /* O_TRUNC requires write mode */
+        DEBUGF("No write mode but have O_TRUNC\n");
+        FILE_ERROR(EINVAL, -2);
     }
 
-    file = &openfiles[fd];
+    /* O_CREAT and O_APPEND are fine without write mode
+     * for the former, an empty file is created but no data may be written
+     * for the latter, no append will be allowed anyway */
+    if (oflag & O_CREAT)
+    {
+        callflags |= FF_CREAT;
 
-    rc = fat_rename(&file->fatfile, &dir->fatdir, nameptr,
-                    file->size, file->attr);
-#ifdef HAVE_MULTIVOLUME
-    if ( rc == -1) {
-        close(fd);
-        closedir_uncached(dir);
-        DEBUGF("Failed renaming file across volumnes: %d\n", rc);
-        errno = EXDEV;
-        return -6;
-    }
-#endif
-    if ( rc < 0 ) {
-        close(fd);
-        closedir_uncached(dir);
-        DEBUGF("Failed renaming file: %d\n", rc);
-        errno = EIO;
-        return rc * 10 - 7;
+        if (oflag & O_EXCL)
+            callflags |= FF_EXCL;
     }
 
-#ifdef HAVE_DIRCACHE
-    dircache_rename(path, newpath);
-#endif
+    rc = open_internal_inner2(path, file, callflags);
+    if (rc < 0)
+        FILE_ERROR(ERRNO, rc * 10 - 3);
 
-    rc = close(fd);
-    if (rc<0) {
-        closedir_uncached(dir);
-        errno = EIO;
-        return rc * 10 - 8;
-    }
+    return fildes;
 
-    rc = closedir_uncached(dir);
-    if (rc<0) {
-        errno = EIO;
-        return rc * 10 - 9;
-    }
+file_error:
+    return rc;
+}
 
-    return 0;
+static int open_internal_locked(const char *path, int oflag,
+                                unsigned int callflags)
+{
+    file_internal_lock_WRITER();
+    int rc = open_internal_inner1(path, oflag, callflags);
+    file_internal_unlock_WRITER();
+    return rc;
 }
 
-int ftruncate(int fd, off_t size)
+/* fill a cache buffer with a new sector */
+static int readwrite_fill_cache(struct filestr_desc *file, unsigned long sector,
+                                unsigned long filesectors, bool write)
 {
-    int rc, sector;
-    struct filedesc* file = &openfiles[fd];
+    /* sector != cachep->sector should have been checked by now */
 
-    sector = size / SECTOR_SIZE;
-    if (size % SECTOR_SIZE)
-        sector++;
+    int rc;
+    struct filestr_cache *cachep = filestr_get_cache(&file->stream);
 
-    rc = fat_seek(&(file->fatfile), sector);
-    if (rc < 0) {
-        errno = EIO;
-        return rc * 10 - 1;
+    if (cachep->flags & FSC_DIRTY)
+    {
+        rc = flush_cache(file);
+        if (rc < 0)
+            FILE_ERROR(ERRNO, rc * 10 - 1);
     }
 
-    rc = fat_truncate(&(file->fatfile));
-    if (rc < 0) {
-        errno = EIO;
-        return rc * 10 - 2;
+    if (fat_query_sectornum(&file->stream.fatstr) != sector)
+    {
+        /* get on the correct sector */
+        rc = fat_seek(&file->stream.fatstr, sector);
+        if (rc < 0)
+            FILE_ERROR(EIO, rc * 10 - 2);
     }
 
-    file->size = size;
-#ifdef HAVE_DIRCACHE
-    dircache_update_filesize(fd, size, file->fatfile.firstcluster);
-#endif
+    if (!write || sector < filesectors)
+    {
+        /* only reading or this sector would have been flushed if the cache
+           was previously needed for a different sector */
+        rc = fat_readwrite(&file->stream.fatstr, 1, cachep->buffer, false);
+        if (rc < 0)
+            FILE_ERROR(rc == FAT_RC_ENOSPC ? ENOSPC : EIO, rc * 10 - 3);
+    }
+    else
+    {
+        /* create a fresh, shiny, new sector with that new sector smell */
+        cachep->flags = FSC_NEW;
+    }
 
-    return 0;
+    cachep->sector = sector;
+    return 1;
+file_error:
+    DEBUGF("Failed caching sector: %d\n", rc);
+    return rc;
 }
 
-static int flush_cache(int fd)
+/* read or write to part or all of the cache buffer */
+static inline void readwrite_cache(struct filestr_cache *cachep, void *buf,
+                                   unsigned long secoffset, size_t nbyte,
+                                   bool write)
 {
-    int rc;
-    struct filedesc* file = &openfiles[fd];
-    long sector = file->fileoffset / SECTOR_SIZE;
-
-    DEBUGF("Flushing dirty sector cache\n");
+    void *dst, *cbufp = cachep->buffer + secoffset;
 
-    /* make sure we are on correct sector */
-    rc = fat_seek(&(file->fatfile), sector);
-    if ( rc < 0 )
-        return rc * 10 - 3;
-
-    rc = fat_readwrite(&(file->fatfile), 1, file->cache, true );
+    if (write)
+    {
+        dst = cbufp;
+        cachep->flags |= FSC_DIRTY;
+    }
+    else
+    {
+        dst = buf;
+        buf = cbufp;
+    }
 
-    if ( rc < 0 ) {
-        if(file->fatfile.eof)
-            errno = ENOSPC;
+    memcpy(dst, buf, nbyte);
+}
 
-        return rc * 10 - 2;
+/* read or write a partial sector using the file's cache */
+static inline ssize_t readwrite_partial(struct filestr_desc *file,
+                                        struct filestr_cache *cachep,
+                                        unsigned long sector,
+                                        unsigned long secoffset,
+                                        void *buf,
+                                        size_t nbyte,
+                                        unsigned long filesectors,
+                                        unsigned int flags)
+{
+    if (sector != cachep->sector)
+    {
+        /* wrong sector in buffer */
+        int rc = readwrite_fill_cache(file, sector, filesectors, flags);
+        if (rc <= 0)
+            return rc;
     }
 
-    file->dirty = false;
-
-    return 0;
+    readwrite_cache(cachep, buf, secoffset, nbyte, flags);
+    return nbyte;
 }
 
-static int readwrite(int fd, void* buf, long count, bool write)
+/* read from or write to the file; back end to read() and write() */
+static ssize_t readwrite(struct filestr_desc *file, void *buf, size_t nbyte,
+                         bool write)
 {
-    long sectors;
-    long nread=0;
-    struct filedesc* file;
-    int rc;
-#ifdef STORAGE_NEEDS_ALIGN
-    long i;
-    int rc2;
-#endif
+    DEBUGF("readwrite(%p,%lx,%ld,%s)\n",
+           file, (long)buf, nbyte, write ? "write" : "read");
 
-    if (fd < 0 || fd > MAX_OPEN_FILES-1) {
-        errno = EINVAL;
-        return -1;
-    }
+    const file_size_t size = *file->sizep;
+    file_size_t filerem;
 
-    file = &openfiles[fd];
+    if (write)
+    {
+        /* if opened in append mode, move pointer to end */
+        if (file->stream.flags & FD_APPEND)
+            file->offset = MIN(size, FILE_SIZE_MAX);
 
-    if ( !file->busy ) {
-        errno = EBADF;
-        return -1;
+        filerem = FILE_SIZE_MAX - file->offset;
+    }
+    else
+    {
+        /* limit to maximum possible offset (EOF or FILE_SIZE_MAX) */
+        filerem = MIN(size, FILE_SIZE_MAX) - file->offset;
     }
 
-    if(file->attr & FAT_ATTR_DIRECTORY) {
-        errno = EISDIR;
-        return -1;
+    if (nbyte > filerem)
+    {
+        nbyte = filerem;
+        if (nbyte > 0)
+            {}
+        else if (write)
+            FILE_ERROR_RETURN(EFBIG, -1);     /* would get too large */
+        else if (file->offset >= FILE_SIZE_MAX)
+            FILE_ERROR_RETURN(EOVERFLOW, -2); /* can't read here */
     }
 
-    LDEBUGF( "readwrite(%d,%lx,%ld,%s)\n",
-             fd,(long)buf,count,write?"write":"read");
+    if (nbyte == 0)
+        return 0;
 
-    /* attempt to read past EOF? */
-    if (!write && count > file->size - file->fileoffset)
-        count = file->size - file->fileoffset;
+    int rc = 0;
+
+    struct filestr_cache * const cachep = file->stream.cachep;
+    void * const bufstart = buf;
+
+    const unsigned long filesectors = filesize_sectors(size);
+    unsigned long sector = file->offset / SECTOR_SIZE;
+    unsigned long sectoroffs = file->offset % SECTOR_SIZE;
 
     /* any head bytes? */
-    if ( file->cacheoffset != -1 ) {
-        int offs = file->cacheoffset;
-        int headbytes = MIN(count, SECTOR_SIZE - offs);
+    if (sectoroffs)
+    {
+        size_t headbytes = MIN(nbyte, SECTOR_SIZE - sectoroffs);
+        rc = readwrite_partial(file, cachep, sector, sectoroffs, buf, headbytes,
+                               filesectors, write);
+        if (rc <= 0)
+        {
+            if (rc < 0)
+                FILE_ERROR(ERRNO, rc * 10 - 3);
 
-        if (write) {
-            memcpy( file->cache + offs, buf, headbytes );
-            file->dirty = true;
+            nbyte = 0; /* eof, skip the rest */
         }
-        else {
-            memcpy( buf, file->cache + offs, headbytes );
+        else
+        {
+            buf += rc;
+            nbyte -= rc;
+            sector++;  /* if nbyte goes to 0, the rest is skipped anyway */
         }
+    }
 
-        if (offs + headbytes == SECTOR_SIZE) {
-            if (file->dirty) {
-                rc = flush_cache(fd);
-                if ( rc < 0 ) {
-                    errno = EIO;
-                    return rc * 10 - 2;
-                }
-            }
-            file->cacheoffset = -1;
-        }
-        else {
-            file->cacheoffset += headbytes;
-        }
+    /* read/write whole sectors right into/from the supplied buffer */
+    unsigned long sectorcount = nbyte / SECTOR_SIZE;
 
-        nread = headbytes;
-        count -= headbytes;
-    }
+    while (sectorcount)
+    {
+        unsigned long runlen = sectorcount;
 
-    /* If the buffer has been modified, either it has been flushed already
-     * (if (offs+headbytes == SECTOR_SIZE)...) or does not need to be (no
-     * more data to follow in this call). Do NOT flush here. */
+        /* if a cached sector is inside the transfer range, split the transfer
+           into two parts and use the cache for that sector to keep it coherent
+           without writeback */
+        if (UNLIKELY(cachep->sector >= sector &&
+                     cachep->sector < sector + sectorcount))
+        {
+            runlen = cachep->sector - sector;
+        }
 
-    /* read/write whole sectors right into/from the supplied buffer */
-    sectors = count / SECTOR_SIZE;
-    rc = 0;
-    if ( sectors ) {
-#ifdef STORAGE_NEEDS_ALIGN
-        if (((uint32_t)buf + nread) & (CACHEALIGN_SIZE - 1))
-            for (i = 0; i < sectors; i++)
+        if (runlen)
+        {
+            if (fat_query_sectornum(&file->stream.fatstr) != sector)
             {
-                if (write) memcpy(file->cache, buf+nread+i*SECTOR_SIZE, SECTOR_SIZE);
-                rc2 = fat_readwrite(&(file->fatfile), 1, file->cache, write );
-                if (rc2 < 0)
+                /* get on the correct sector */
+                rc = 0;
+
+                /* If the dirty bit isn't set, we're somehow beyond the file
+                   size and you can't explain _that_ */
+                if (sector >= filesectors && cachep->flags == (FSC_NEW|FSC_DIRTY))
                 {
-                    rc = rc2;
-                    break;
+                    rc = flush_cache(file);
+                    if (rc < 0)
+                        FILE_ERROR(ERRNO, rc * 10 - 4);
+
+                    if (cachep->sector + 1 == sector)
+                        rc = 1; /* if now ok, don't seek */
+                }
+
+                if (rc == 0)
+                {
+                    rc = fat_seek(&file->stream.fatstr, sector);
+                    if (rc < 0)
+                        FILE_ERROR(EIO, rc * 10 - 5);
                 }
-                else rc += rc2;
-                if (!write) memcpy(buf+nread+i*SECTOR_SIZE, file->cache, SECTOR_SIZE);
-            }
-        else
-#endif
-            rc = fat_readwrite(&(file->fatfile), sectors, (unsigned char*)buf+nread, write );
-        if ( rc < 0 ) {
-            DEBUGF("Failed read/writing %ld sectors\n",sectors);
-            errno = EIO;
-            if(write && file->fatfile.eof) {
-                DEBUGF("No space left on device\n");
-                errno = ENOSPC;
-            } else {
-                file->fileoffset += nread;
             }
-            file->cacheoffset = -1;
-            /* adjust file size to length written */
-            if ( write && file->fileoffset > file->size )
+
+            rc = fat_readwrite(&file->stream.fatstr, runlen, buf, write);
+            if (rc < 0)
             {
-                file->size = file->fileoffset;
-#ifdef HAVE_DIRCACHE
-                dircache_update_filesize(fd, file->size, file->fatfile.firstcluster);
-#endif
+                DEBUGF("I/O error %sing %ld sectors\n", sectors,
+                       write ? "writ" : "read");
+                FILE_ERROR(rc == FAT_RC_ENOSPC ? ENOSPC : EIO,
+                           rc * 10 - 6);
             }
-            return nread ? nread : rc * 10 - 4;
-        }
-        else {
-            if ( rc > 0 ) {
-                nread += rc * SECTOR_SIZE;
-                count -= sectors * SECTOR_SIZE;
+            else
+            {
+                buf += rc * SECTOR_SIZE;
+                nbyte -= rc * SECTOR_SIZE;
+                sector += rc;
+                sectorcount -= rc;
 
                 /* if eof, skip tail bytes */
-                if ( rc < sectors )
-                    count = 0;
-            }
-            else {
-                /* eof */
-                count=0;
+                if ((unsigned long)rc < runlen)
+                    nbyte = 0;
+
+                if (!nbyte)
+                    break;
             }
+        }
 
-            file->cacheoffset = -1;
+        if (UNLIKELY(sectorcount && sector == cachep->sector))
+        {
+            /* do this one sector with the cache */
+            readwrite_cache(cachep, buf, 0, SECTOR_SIZE, write);
+            buf += SECTOR_SIZE;
+            nbyte -= SECTOR_SIZE;
+            sector++;
+            sectorcount--;
         }
     }
 
     /* any tail bytes? */
-    if ( count ) {
-        if (write) {
-            if ( file->fileoffset + nread < file->size ) {
-                /* sector is only partially filled. copy-back from disk */
-                LDEBUGF("Copy-back tail cache\n");
-                rc = fat_readwrite(&(file->fatfile), 1, file->cache, false );
-                if ( rc < 0 ) {
-                    DEBUGF("Failed writing\n");
-                    errno = EIO;
-                    file->fileoffset += nread;
-                    file->cacheoffset = -1;
-                    /* adjust file size to length written */
-                    if ( file->fileoffset > file->size )
-                    {
-                        file->size = file->fileoffset;
-#ifdef HAVE_DIRCACHE
-                        dircache_update_filesize(fd, file->size, file->fatfile.firstcluster);
-#endif
-                    }
-                    return nread ? nread : rc * 10 - 5;
-                }
-                /* seek back one sector to put file position right */
-                rc = fat_seek(&(file->fatfile), 
-                              (file->fileoffset + nread) /
-                              SECTOR_SIZE);
-                if ( rc < 0 ) {
-                    DEBUGF("fat_seek() failed\n");
-                    errno = EIO;
-                    file->fileoffset += nread;
-                    file->cacheoffset = -1;
-                    /* adjust file size to length written */
-                    if ( file->fileoffset > file->size )
-                    {
-                        file->size = file->fileoffset;
-#ifdef HAVE_DIRCACHE
-                        dircache_update_filesize(fd, file->size, file->fatfile.firstcluster);
-#endif
-                    }
-                    return nread ? nread : rc * 10 - 6;
-                }
-            }
-            memcpy( file->cache, (unsigned char*)buf + nread, count );
-            file->dirty = true;
-        }
-        else {
-            rc = fat_readwrite(&(file->fatfile), 1, file->cache,false);
-            if (rc < 1 ) {
-                DEBUGF("Failed caching sector\n");
-                errno = EIO;
-                file->fileoffset += nread;
-                file->cacheoffset = -1;
-                return nread ? nread : rc * 10 - 7;
-            }
-            memcpy( (unsigned char*)buf + nread, file->cache, count );
-        }
+    if (nbyte)
+    {
+        /* tail bytes always start at sector offset 0 */
+        rc = readwrite_partial(file, cachep, sector, 0, buf, nbyte,
+                               filesectors, write);
+        if (rc < 0)
+            FILE_ERROR(ERRNO, rc * 10 - 7);
 
-        nread += count;
-        file->cacheoffset = count;
+        buf += rc;
     }
 
-    file->fileoffset += nread;
-    LDEBUGF("fileoffset: %ld\n", file->fileoffset);
+file_error:;
+#ifdef DEBUG
+    if (errno == ENOSPC)
+        DEBUGF("No space left on device\n");
+#endif
 
-    /* adjust file size to length written */
-    if ( write && file->fileoffset > file->size )
+    size_t done = buf - bufstart;
+    if (done)
     {
-        file->size = file->fileoffset;
-#ifdef HAVE_DIRCACHE
-        dircache_update_filesize(fd, file->size, file->fatfile.firstcluster);
-#endif
+        /* error or not, update the file offset and size if anything was
+           transferred */
+        file->offset += done;
+        DEBUGF("file offset: %ld\n", file->offset);
+
+        /* adjust file size to length written */
+        if (write && file->offset > size)
+            *file->sizep = file->offset;
+
+        if (rc > 0)
+            return done;
     }
 
-    return nread;
+    return rc;
 }
 
-ssize_t write(int fd, const void* buf, size_t count)
+
+/** Internal interface **/
+
+/* open a file without codepage conversion during the directory search;
+   required to avoid any reentrancy when opening codepages and when scanning
+   directories internally, which could infinitely recurse and would corrupt
+   the static data */
+int open_noiso_internal(const char *path, int oflag)
 {
-    if (!openfiles[fd].write) {
-        errno = EACCES;
-        return -1;
-    }
-    return readwrite(fd, (void *)buf, count, true);
+    return open_internal_locked(path, oflag, FF_ANYTYPE | FF_NOISO);
 }
 
-ssize_t read(int fd, void* buf, size_t count)
+
+/** POSIX **/
+
+/* open a file */
+int open(const char *path, int oflag)
 {
-    return readwrite(fd, buf, count, false);
+    DEBUGF("open(path=\"%s\",oflag=%X)\n", path, (unsigned)oflag);
+    return open_internal_locked(path, oflag, FF_ANYTYPE);
 }
 
+/* create a new file or rewrite an existing one */
+int creat(const char *path)
+{
+    DEBUGF("creat(path=\"%s\")\n", path);
+    return open_internal_locked(path, O_WRONLY|O_CREAT|O_TRUNC, FF_ANYTYPE);
+}
 
-off_t lseek(int fd, off_t offset, int whence)
+/* close a file descriptor */
+int close(int fildes)
 {
-    off_t pos;
-    long newsector;
-    long oldsector;
-    int sectoroffset;
+    DEBUGF("close(fd=%d)\n", fildes);
+
     int rc;
-    struct filedesc* file = &openfiles[fd];
 
-    LDEBUGF("lseek(%d,%ld,%d)\n",fd,offset,whence);
+    file_internal_lock_WRITER();
 
-    if (fd < 0 || fd > MAX_OPEN_FILES-1) {
-        errno = EINVAL;
-        return -1;
-    }
-    if ( !file->busy ) {
-        errno = EBADF;
-        return -1;
+    /* needs to work even if marked "nonexistant" */
+    struct filestr_desc *file = &open_streams[fildes];
+    if ((unsigned int)fildes >= MAX_OPEN_FILES || !file->stream.flags)
+    {
+        DEBUGF("filedes %d not open\n", fildes);
+        FILE_ERROR(EBADF, -2);
     }
 
-    switch ( whence ) {
-        case SEEK_SET:
-            pos = offset;
-            break;
+    rc = close_internal(file);
+    if (rc < 0)
+        FILE_ERROR(ERRNO, rc * 10 - 3);
 
-        case SEEK_CUR:
-            pos = file->fileoffset + offset;
-            break;
+file_error:
+    file_internal_unlock_WRITER();
+    return rc;
+}
+
+/* truncate a file to a specified length */
+int ftruncate(int fildes, off_t length)
+{
+    DEBUGF("ftruncate(fd=%d,len=%ld)\n", fildes, (long)length);
 
-        case SEEK_END:
-            pos = file->size + offset;
-            break;
+    struct filestr_desc * const file = GET_FILESTR(READER, fildes);
+    if (!file)
+        FILE_ERROR_RETURN(ERRNO, -1);
 
-        default:
-            errno = EINVAL;
-            return -2;
+    int rc;
+
+    if (!(file->stream.flags & FD_WRITE))
+    {
+        DEBUGF("Descriptor is read-only mode\n");
+        FILE_ERROR(EBADF, -2);
     }
-    if ((pos < 0) || (pos > file->size)) {
-        errno = EINVAL;
-        return -3;
+
+    if (length < 0)
+    {
+        DEBUGF("Length %ld is invalid\n", (long)length);
+        FILE_ERROR(EINVAL, -3);
     }
 
-    /* new sector? */
-    newsector = pos / SECTOR_SIZE;
-    oldsector = file->fileoffset / SECTOR_SIZE;
-    sectoroffset = pos % SECTOR_SIZE;
+    rc = ftruncate_internal(file, length, true);
+    if (rc < 0)
+        FILE_ERROR(ERRNO, rc * 10 - 4);
 
-    if ( (newsector != oldsector) ||
-         ((file->cacheoffset==-1) && sectoroffset) ) {
+file_error:
+    RELEASE_FILESTR(READER, file);
+    return rc;
+}
 
-        if ( newsector != oldsector ) {
-            if (file->dirty) {
-                rc = flush_cache(fd);
-                if (rc < 0)
-                    return rc * 10 - 5;
-            }
+/* synchronize changes to a file */
+int fsync(int fildes)
+{
+    DEBUGF("fsync(fd=%d)\n", fildes);
 
-            rc = fat_seek(&(file->fatfile), newsector);
-            if ( rc < 0 ) {
-                errno = EIO;
-                return rc * 10 - 4;
-            }
-        }
-        if ( sectoroffset ) {
-            rc = fat_readwrite(&(file->fatfile), 1, file->cache ,false);
-            if ( rc < 0 ) {
-                errno = EIO;
-                return rc * 10 - 6;
-            }
-            file->cacheoffset = sectoroffset;
-        }
-        else
-            file->cacheoffset = -1;
+    struct filestr_desc * const file = GET_FILESTR(WRITER, fildes);
+    if (!file)
+        FILE_ERROR_RETURN(ERRNO, -1);
+
+    int rc;
+
+    if (!(file->stream.flags & FD_WRITE))
+    {
+        DEBUGF("Descriptor is read-only mode\n", fd);
+        FILE_ERROR(EINVAL, -2);
     }
-    else
-        if ( file->cacheoffset != -1 )
-            file->cacheoffset = sectoroffset;
 
-    file->fileoffset = pos;
+    rc = fsync_internal(file);
+    if (rc < 0)
+        FILE_ERROR(ERRNO, rc * 10 - 3);
 
-    return pos;
+file_error:
+    RELEASE_FILESTR(WRITER, file);
+    return rc;
 }
 
-off_t filesize(int fd)
+/* move the read/write file offset */
+off_t lseek(int fildes, off_t offset, int whence)
 {
-    struct filedesc* file = &openfiles[fd];
+    DEBUGF("lseek(fd=%d,ofs=%ld,wh=%d)\n", fildes, (long)offset, whence);
 
-    if (fd < 0 || fd > MAX_OPEN_FILES-1) {
-        errno = EINVAL;
-        return -1;
+    struct filestr_desc * const file = GET_FILESTR(READER, fildes);
+    if (!file)
+        FILE_ERROR_RETURN(ERRNO, -1);
+
+    off_t rc = lseek_internal(file, offset, whence);
+    if (rc < 0)
+        FILE_ERROR(ERRNO, rc * 10 - 2);
+
+file_error:
+    RELEASE_FILESTR(READER, file);
+    return rc;
+}
+
+/* read from a file */
+ssize_t read(int fildes, void *buf, size_t nbyte)
+{
+    struct filestr_desc * const file = GET_FILESTR(READER, fildes);
+    if (!file)
+        FILE_ERROR_RETURN(ERRNO, -1);
+
+    ssize_t rc;
+
+    if (file->stream.flags & FD_WRONLY)
+    {
+        DEBUGF("read(fd=%d,buf=%p,nb=%lu) - "
+               "descriptor is write-only mode\n", fildes, buf, nbyte);
+        FILE_ERROR(EBADF, -2);
     }
-    if ( !file->busy ) {
-        errno = EBADF;
-        return -1;
+
+    rc = readwrite(file, buf, nbyte, false);
+    if (rc < 0)
+        FILE_ERROR(ERRNO, rc * 10 - 3);
+
+file_error:
+    RELEASE_FILESTR(READER, file);
+    return rc;
+}
+
+/* write on a file */
+ssize_t write(int fildes, const void *buf, size_t nbyte)
+{
+    struct filestr_desc * const file = GET_FILESTR(READER, fildes);
+    if (!file)
+        FILE_ERROR_RETURN(ERRNO, -1);
+
+    ssize_t rc;
+
+    if (!(file->stream.flags & FD_WRITE))
+    {
+        DEBUGF("write(fd=%d,buf=%p,nb=%lu) - "
+               "descriptor is read-only mode\n", fildes, buf, nbyte);
+        FILE_ERROR(EBADF, -2);
     }
 
-    return file->size;
+    rc = readwrite(file, (void *)buf, nbyte, true);
+    if (rc < 0)
+        FILE_ERROR(ERRNO, rc * 10 - 3);
+
+file_error:
+    RELEASE_FILESTR(READER, file);
+    return rc;
 }
 
+/* remove a file */
+int remove(const char *path)
+{
+    DEBUGF("remove(path=\"%s\")\n", path);
+
+    file_internal_lock_WRITER();
+    int rc = remove_stream_internal(path, NULL, FF_FILE);
+    file_internal_unlock_WRITER();
+    return rc;
+}
 
-/* release all file handles on a given volume "by force", to avoid leaks */
-int release_files(int volume)
+/* rename a file */
+int rename(const char *old, const char *new)
 {
-    struct filedesc* pfile = openfiles;
-    int fd;
-    int closed = 0;
-    for ( fd=0; fd<MAX_OPEN_FILES; fd++, pfile++)
+    DEBUGF("rename(old=\"%s\",new=\"%s\")\n", old, new);
+
+    int rc, open1rc = -1, open2rc = -1;
+    struct filestr_base oldstr, newstr;
+    struct path_component_info oldinfo, newinfo;
+
+    file_internal_lock_WRITER();
+
+    /* open 'old'; it must exist */
+    open1rc = open_stream_internal(old, FF_ANYTYPE, &oldstr, &oldinfo);
+    if (open1rc <= 0)
     {
+        DEBUGF("Failed opening old: %d\n", rc);
+        if (open1rc == 0)
+            FILE_ERROR(ENOENT, -1);
+        else
+            FILE_ERROR(ERRNO, open1rc * 10 - 1);
+    }
+
+    /* if 'old' is a directory then 'new' is also required to be one if 'new'
+       is to be overwritten */
+    const bool are_dirs = oldinfo.attr & ATTR_DIRECTORY;
+
+    /* open new (may or may not exist) */
+    unsigned int callflags = FF_FILE;
+    if (are_dirs)
+    {
+        /* if 'old' is found while parsing the new directory components then
+           'new' contains path prefix that names 'old'; if new and old are in
+           the same directory, this tests positive but that is checked later */
+        callflags = FF_DIR | FF_CHECKPREFIX;
+        newinfo.prefixp = oldstr.infop;
+    }
+
+    open2rc = open_stream_internal(new, callflags, &newstr, &newinfo);
+    if (open2rc < 0)
+    {
+        DEBUGF("Failed opening new file: %d\n", rc);
+        FILE_ERROR(ERRNO, open2rc * 10 - 2);
+    }
+
 #ifdef HAVE_MULTIVOLUME
-        if (pfile->fatfile.volume == volume)
-#else
-        (void)volume;
-#endif
+    if (oldinfo.parentinfo.volume != newinfo.parentinfo.volume)
+    {
+        DEBUGF("Cross-device link\n");
+        FILE_ERROR(EXDEV, -3);
+    }
+#endif /* HAVE_MULTIVOLUME */
+
+    /* if the parent is changing then this is a move, not a simple rename */
+    const bool is_move = !fat_file_is_same(&oldinfo.parentinfo.fatfile,
+                                           &newinfo.parentinfo.fatfile);
+    /* prefix found and moving? */
+    if (is_move && (newinfo.attr & ATTR_PREFIX))
+    {
+        DEBUGF("New contains prefix that names old\n");
+        FILE_ERROR(EINVAL, -4);
+    }
+
+    const char * const oldname = strmemdupa(oldinfo.name, oldinfo.length);
+    const char * const newname = strmemdupa(newinfo.name, newinfo.length);
+    bool is_overwrite = false;
+
+    if (open2rc > 0)
+    {
+        /* new name exists in parent; check if 'old' is overwriting 'new';
+           if it's the very same file, then it's just a rename */
+        is_overwrite = oldstr.bindp != newstr.bindp;
+
+        if (is_overwrite)
+        {
+            if (are_dirs)
+            {
+                /* the directory to be overwritten must be empty */
+                rc = test_dir_empty_internal(&newstr);
+                if (rc < 0)
+                    FILE_ERROR(ERRNO, rc * 10 - 5);
+            }
+        }
+        else if (!strcmp(newname, oldname)) /* case-only is ok */
+        {
+            DEBUGF("No name change (success)\n");
+            rc = 0;
+            FILE_ERROR(ERRNO, RC);
+        }
+    }
+    else if (!are_dirs && (newinfo.attr & ATTR_DIRECTORY))
+    {
+        /* even if new doesn't exist, canonical path type must match
+           (ie. a directory path such as "/foo/bar/" when old names a file) */
+        DEBUGF("New path is a directory\n");
+        FILE_ERROR(EISDIR, -6);
+    }
+
+    /* first, create the new entry so that there's never a time that the
+       victim's data has no reference in the directory tree, that is, until
+       everything else first succeeds */
+    struct file_base_info old_fileinfo = *oldstr.infop;
+    rc = fat_rename(&newinfo.parentinfo.fatfile, &oldstr.infop->fatfile,
+                    newname);
+    if (rc < 0)
+    {
+        DEBUGF("I/O error renaming file: %d\n", rc);
+        FILE_ERROR(rc == FAT_RC_ENOSPC ? ENOSPC : EIO, rc * 10 - 7);
+    }
+
+    if (is_overwrite)
+    {
+        /* 'new' would have been assigned its own directory entry and
+           succeeded so at this point it is treated like a remove() call
+           on the victim which preserves data until the last reference is
+           closed */
+        rc = remove_stream_internal(NULL, &newstr, callflags);
+        if (rc < 0)
+            FILE_ERROR(ERRNO, rc * 10 - 8);
+    }
+
+    fileop_onrename_internal(&oldstr, is_move ? &old_fileinfo : NULL,
+                             &newinfo.parentinfo, newname);
+
+file_error:
+    /* for now, there is nothing to fail upon closing the old stream */
+    if (open1rc >= 0)
+        close_stream_internal(&oldstr);
+
+    /* the 'new' stream could fail to close cleanly because it became
+       impossible to remove its data if this was an overwrite operation */
+    if (open2rc >= 0)
+    {
+        int rc2 = close_stream_internal(&newstr);
+        if (rc2 < 0 && rc >= 0)
         {
-            pfile->busy = false; /* mark as available, no further action */
-            closed++;
+            DEBUGF("Success but failed closing new: %d\n", rc2);
+            rc = rc2 * 10 - 9;
         }
     }
-    return closed; /* return how many we did */
+
+    file_internal_unlock_WRITER();
+    return rc;
+}
+
+
+/** Extensions **/
+
+/* get the binary size of a file (in bytes) */
+off_t filesize(int fildes)
+{
+    struct filestr_desc * const file = GET_FILESTR(READER, fildes);
+    if (!file)
+        FILE_ERROR_RETURN(ERRNO, -1);
+
+    off_t rc;
+    file_size_t size = *file->sizep;
+
+    if (size > FILE_SIZE_MAX)
+        FILE_ERROR(EOVERFLOW, -2);
+
+    rc = (off_t)size;
+file_error:
+    RELEASE_FILESTR(READER, file);
+    return rc;
+}
+
+/* test if two file descriptors refer to the same file */
+int fsamefile(int fildes1, int fildes2)
+{
+    struct filestr_desc * const file1 = GET_FILESTR(WRITER, fildes1);
+    if (!file1)
+        FILE_ERROR_RETURN(ERRNO, -1);
+
+    int rc = -2;
+
+    struct filestr_desc * const file2 = get_filestr(fildes2);
+    if (file2)
+        rc = file1->stream.bindp == file2->stream.bindp ? 1 : 0;
+
+    RELEASE_FILESTR(WRITER, file1);
+    return rc;
+}
+
+/* tell the relationship of path1 to path2 */
+int relate(const char *path1, const char *path2)
+{
+    /* this is basically what rename() does but reduced to the relationship
+       determination */
+    DEBUGF("relate(path1=\"%s\",path2=\"%s\")\n", path1, path2);
+
+    int rc, open1rc = -1, open2rc = -1;
+    struct filestr_base str1, str2;
+    struct path_component_info info1, info2;
+
+    file_internal_lock_WRITER();
+
+    open1rc = open_stream_internal(path1, FF_ANYTYPE, &str1, &info1);
+    if (open1rc <= 0)
+    {
+        DEBUGF("Failed opening path1: %d\n", rc);
+        if (open1rc < 0)
+            FILE_ERROR(ERRNO, open1rc * 10 - 1);
+        else
+            FILE_ERROR(ENOENT, -1);
+    }
+
+    info2.prefixp = str1.infop;
+    open2rc = open_stream_internal(path2, FF_ANYTYPE | FF_CHECKPREFIX,
+                                   &str2, &info2);
+    if (open2rc < 0)
+    {
+        DEBUGF("Failed opening path2: %d\n", rc);
+        FILE_ERROR(ERRNO, open2rc * 10 - 2);
+    }
+
+    rc = RELATE_DIFFERENT;
+
+    if (open2rc > 0)
+    {
+        if (str1.bindp == str2.bindp)
+            rc = RELATE_SAME;
+        else if (info2.attr & ATTR_PREFIX)
+            rc = RELATE_PREFIX;
+    }
+    else /* open2rc == 0 */
+    {
+        /* path1 existing and path2's final part not can only be a prefix or
+           different */
+        if (info2.attr & ATTR_PREFIX)
+            rc = RELATE_PREFIX;
+    }
+
+file_error:
+    if (open1rc >= 0)
+        close_stream_internal(&str1);
+
+    if (open2rc >= 0)
+        close_stream_internal(&str2);
+
+    file_internal_unlock_WRITER();
+    return rc;
+}
+
+/* test file or directory existence */
+bool file_exists(const char *path)
+{
+    file_internal_lock_WRITER();
+    bool rc = test_stream_exists_internal(path, FF_ANYTYPE) > 0;
+    file_internal_unlock_WRITER();
+    return rc;
 }
diff --git a/firmware/common/file_internal.c b/firmware/common/file_internal.c
new file mode 100644
index 0000000000..ebe77f0c9f
--- /dev/null
+++ b/firmware/common/file_internal.c
@@ -0,0 +1,776 @@
+/***************************************************************************
+ *             __________               __   ___.
+ *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
+ *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
+ *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
+ *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
+ *                     \/            \/     \/    \/            \/
+ * $Id$
+ *
+ * Copyright (C) 2014 by Michael Sevakis
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ****************************************************************************/
+#include "config.h"
+#include <errno.h>
+#include "system.h"
+#include "debug.h"
+#include "panic.h"
+#include "pathfuncs.h"
+#include "disk_cache.h"
+#include "fileobj_mgr.h"
+#include "dir.h"
+#include "dircache_redirect.h"
+#include "dircache.h"
+#include "string-extra.h"
+#include "rbunicode.h"
+
+/** Internal common filesystem service functions  **/
+
+/* for internal functions' scanning use to save quite a bit of stack space -
+   access must be serialized by the writer lock */
+#if defined(CPU_SH) || defined(IAUDIO_M5)
+/* otherwise, out of IRAM */
+struct fat_direntry dir_fatent;
+#else
+struct fat_direntry dir_fatent IBSS_ATTR;
+#endif
+
+struct mrsw_lock file_internal_mrsw SHAREDBSS_ATTR;
+
+
+/** File stream sector caching **/
+
+/* initialize a new cache structure */
+void file_cache_init(struct filestr_cache *cachep)
+{
+    cachep->buffer = NULL;
+    cachep->sector = INVALID_SECNUM;
+    cachep->flags  = 0;
+}
+
+/* discard and mark the cache buffer as unused */
+void file_cache_reset(struct filestr_cache *cachep)
+{
+    cachep->sector = INVALID_SECNUM;
+    cachep->flags  = 0;
+}
+
+/* allocate resources attached to the cache */
+void file_cache_alloc(struct filestr_cache *cachep)
+{
+    /* if this fails, it is a bug; check for leaks and that the cache has
+       enough buffers for the worst case */
+    if (!cachep->buffer && !(cachep->buffer = dc_get_buffer()))
+        panicf("file_cache_alloc - OOM");
+}
+
+/* free resources attached to the cache */
+void file_cache_free(struct filestr_cache *cachep)
+{
+    if (cachep && cachep->buffer)
+    {
+        dc_release_buffer(cachep->buffer);
+        cachep->buffer = NULL;
+    }
+
+    file_cache_reset(cachep);
+}
+
+
+/** Stream base APIs **/
+
+static inline void filestr_clear(struct filestr_base *stream)
+{
+    stream->flags = 0;
+    stream->bindp = NULL;
+#if 0
+    stream->mtx   = NULL;
+#endif
+}
+
+/* actually late-allocate the assigned cache */
+void filestr_alloc_cache(struct filestr_base *stream)
+{
+    file_cache_alloc(stream->cachep);
+}
+
+/* free the stream's cache buffer if it's its own */
+void filestr_free_cache(struct filestr_base *stream)
+{
+    if (stream->cachep == &stream->cache)
+        file_cache_free(stream->cachep);
+}
+
+/* assign a cache to the stream */
+void filestr_assign_cache(struct filestr_base *stream,
+                          struct filestr_cache *cachep)
+{
+    if (cachep)
+    {
+        filestr_free_cache(stream);
+        stream->cachep = cachep;
+    }
+    else /* assign own cache */
+    {
+        file_cache_reset(&stream->cache);
+        stream->cachep = &stream->cache;
+    }
+}
+
+/* duplicate a cache into a stream's local cache */
+void filestr_copy_cache(struct filestr_base *stream,
+                        struct filestr_cache *cachep)
+{
+    stream->cachep = &stream->cache;
+    stream->cache.sector = cachep->sector;
+    stream->cache.flags  = cachep->flags;
+
+    if (cachep->buffer)
+    {
+        file_cache_alloc(&stream->cache);
+        memcpy(stream->cache.buffer, cachep->buffer, DC_CACHE_BUFSIZE);
+    }
+    else
+    {
+        file_cache_free(&stream->cache);
+    }
+}
+
+/* discard cache contents and invalidate it */
+void filestr_discard_cache(struct filestr_base *stream)
+{
+    file_cache_reset(stream->cachep);
+}
+
+/* Initialize the base descriptor */
+void filestr_base_init(struct filestr_base *stream)
+{
+    filestr_clear(stream);
+    file_cache_init(&stream->cache);
+    stream->cachep = &stream->cache;
+}
+
+/* free base descriptor resources */
+void filestr_base_destroy(struct filestr_base *stream)
+{
+    filestr_clear(stream);
+    filestr_free_cache(stream);
+}
+
+
+/** Internal directory service functions **/
+
+/* read the next directory entry and return its FS info */
+int uncached_readdir_internal(struct filestr_base *stream,
+                              struct file_base_info *infop,
+                              struct fat_direntry *fatent)
+{
+    return fat_readdir(&stream->fatstr, &infop->fatfile.e,
+                       filestr_get_cache(stream), fatent);
+}
+
+/* rewind the FS directory to the beginning */
+void uncached_rewinddir_internal(struct file_base_info *infop)
+{
+    fat_rewinddir(&infop->fatfile.e);
+}
+
+/* check if the directory is empty (ie. only "." and/or ".." entries
+   exist at most) */
+int test_dir_empty_internal(struct filestr_base *stream)
+{
+    int rc;
+
+    struct file_base_info info;
+    fat_rewind(&stream->fatstr);
+    rewinddir_internal(&info);
+
+    while ((rc = readdir_internal(stream, &info, &dir_fatent)) > 0)
+    {
+        /* no OEM decoding is recessary for this simple check */
+        if (!is_dotdir_name(dir_fatent.name))
+        {
+            DEBUGF("Directory not empty\n");
+            FILE_ERROR_RETURN(ENOTEMPTY, -1);
+        }
+    }
+
+    if (rc < 0)
+    {
+        DEBUGF("I/O error checking directory: %d\n", rc);
+        FILE_ERROR_RETURN(EIO, rc * 10 - 2);
+    }
+
+    return 0;
+}
+
+/* iso decode the name to UTF-8 */
+void iso_decode_d_name(char *d_name)
+{
+    if (is_dotdir_name(d_name))
+        return;
+
+    char shortname[13];
+    size_t len = strlcpy(shortname, d_name, sizeof (shortname));
+    /* This MUST be the default codepage thus not something that could be
+       loaded on call */
+    iso_decode(shortname, d_name, -1, len + 1);
+}
+
+#ifdef HAVE_DIRCACHE
+/* nullify all the fields of the struct dirent */
+void empty_dirent(struct dirent *entry)
+{
+    entry->d_name[0] = '\0';
+    entry->info.attr    = 0;
+    entry->info.size    = 0;
+    entry->info.wrtdate = 0;
+    entry->info.wrttime = 0;
+}
+
+/* fill the native dirinfo from the static dir_fatent */
+void fill_dirinfo_native(struct dirinfo_native *dinp)
+{
+    struct fat_direntry *fatent = get_dir_fatent();
+    dinp->attr    = fatent->attr;
+    dinp->size    = fatent->filesize;
+    dinp->wrtdate = fatent->wrtdate;
+    dinp->wrttime = fatent->wrttime;
+}
+#endif /* HAVE_DIRCACHE */
+
+int uncached_readdir_dirent(struct filestr_base *stream,
+                            struct dirscan_info *scanp,
+                            struct dirent *entry)
+{
+    struct fat_direntry fatent;
+    int rc = fat_readdir(&stream->fatstr, &scanp->fatscan,
+                         filestr_get_cache(stream), &fatent);
+
+    /* FAT driver clears the struct fat_dirent if nothing is returned */
+    strcpy(entry->d_name, fatent.name);
+    entry->info.attr    = fatent.attr;
+    entry->info.size    = fatent.filesize;
+    entry->info.wrtdate = fatent.wrtdate;
+    entry->info.wrttime = fatent.wrttime;
+
+    return rc;
+}
+
+/* rewind the FS directory pointer */
+void uncached_rewinddir_dirent(struct dirscan_info *scanp)
+{
+    fat_rewinddir(&scanp->fatscan);
+}
+
+
+/** open_stream_internal() helpers and types **/
+
+struct pathwalk
+{
+    const char                 *path;     /* current location in input path */
+    unsigned int               callflags; /* callflags parameter */
+    struct path_component_info *compinfo; /* compinfo parameter */
+    file_size_t                filesize;  /* size of the file */
+};
+
+struct pathwalk_component
+{
+    struct file_base_info     info;       /* basic file information */
+    const char                *name;      /* component name location in path */
+    uint16_t                  length;     /* length of name of component */
+    uint16_t                  attr;       /* attributes of this component */
+    struct pathwalk_component *nextp;     /* parent if in use else next free */
+};
+
+#define WALK_RC_NOT_FOUND    0   /* successfully not found */
+#define WALK_RC_FOUND        1   /* found and opened */
+#define WALK_RC_FOUND_ROOT   2   /* found and opened sys/volume root */
+#define WALK_RC_CONT_AT_ROOT 3   /* continue at root level */
+
+/* return another struct pathwalk_component from the pool, or NULL if the
+   pool is completely used */
+static void * pathwalk_comp_alloc_(struct pathwalk_component *parentp)
+{
+    /* static pool that goes to a depth of STATIC_COMP_NUM before allocating
+       elements from the stack */
+    static struct pathwalk_component aux_pathwalk[STATIC_PATHCOMP_NUM];
+    struct pathwalk_component *compp = NULL;
+
+    if (!parentp)
+        compp = &aux_pathwalk[0]; /* root */
+    else if (PTR_IN_ARRAY(aux_pathwalk, parentp, STATIC_PATHCOMP_NUM-1))
+        compp = parentp + 1;
+
+    return compp;
+}
+
+/* allocates components from the pool or stack depending upon the depth */
+#define pathwalk_comp_alloc(parentp) \
+    ({                                                        \
+        void *__c = pathwalk_comp_alloc_(parentp);            \
+        if (!__builtin_constant_p(parentp) && !__c)           \
+            __c = alloca(sizeof (struct pathwalk_component)); \
+        (struct pathwalk_component *)__c;                     \
+    })
+
+/* fill in the details of the struct path_component_info for caller */
+static int fill_path_compinfo(struct pathwalk *walkp,
+                              struct pathwalk_component *compp,
+                              int rc)
+{
+    if (rc == -ENOENT)
+    {
+        /* this component wasn't found; see if more of them exist or path
+           has trailing separators; if it does, this component should be
+           interpreted as a directory even if it doesn't exist and it's the
+           final one; also, this has to be the last part or it's an error*/
+        const char *p = GOBBLE_PATH_SEPCH(walkp->path);
+        if (!*p)
+        {
+            if (p > walkp->path)
+                compp->attr |= ATTR_DIRECTORY;
+
+            rc = WALK_RC_NOT_FOUND; /* successfully not found */
+        }
+    }
+
+    if (rc >= 0)
+    {
+        struct path_component_info *compinfo = walkp->compinfo;
+        compinfo->name       = compp->name;
+        compinfo->length     = compp->length;
+        compinfo->attr       = compp->attr;
+        compinfo->filesize   = walkp->filesize;
+        compinfo->parentinfo = (compp->nextp ?: compp)->info;
+    }
+
+    return rc;
+}
+
+/* open the final stream itself, if found */
+static int walk_open_info(struct pathwalk *walkp,
+                          struct pathwalk_component *compp,
+                          int rc,
+                          struct filestr_base *stream)
+{
+    /* this may make adjustments to things; do it first */
+    if (walkp->compinfo)
+        rc = fill_path_compinfo(walkp, compp, rc);
+
+    if (rc < 0 || rc == WALK_RC_NOT_FOUND)
+        return rc;
+
+    unsigned int callflags = walkp->callflags;
+    bool isdir = compp->attr & ATTR_DIRECTORY;
+
+    /* type must match what is called for */
+    switch (callflags & FF_TYPEMASK)
+    {
+    case FF_FILE:
+        if (!isdir) break;
+        DEBUGF("File is a directory\n");
+        return -EISDIR;
+    case FF_DIR:
+        if (isdir) break;
+        DEBUGF("File is not a directory\n");
+        return -ENOTDIR;
+    /* FF_ANYTYPE: basically, ignore FF_FILE/FF_DIR */
+    }
+
+    /* FO_DIRECTORY must match type */
+    if (isdir)
+        callflags |= FO_DIRECTORY;
+    else
+        callflags &= ~FO_DIRECTORY;
+
+    fileop_onopen_internal(stream, &compp->info, callflags);
+    return compp->nextp ? WALK_RC_FOUND : WALK_RC_FOUND_ROOT;
+}
+
+/* check the component against the prefix test info */
+static void walk_check_prefix(struct pathwalk *walkp,
+                              struct pathwalk_component *compp)
+{
+    if (compp->attr & ATTR_PREFIX)
+        return;
+
+    if (!fat_file_is_same(&compp->info.fatfile,
+                          &walkp->compinfo->prefixp->fatfile))
+        return;
+
+    compp->attr |= ATTR_PREFIX;
+}
+
+/* opens the component named by 'comp' in the directory 'parent' */
+static NO_INLINE int open_path_component(struct pathwalk *walkp,
+                                         struct pathwalk_component *compp,
+                                         struct filestr_base *stream)
+{
+    int rc;
+
+    /* create a null-terminated copy of the component name */
+    char *compname = strmemdupa(compp->name, compp->length);
+
+    unsigned int callflags = walkp->callflags;
+    struct pathwalk_component *parentp = compp->nextp;
+
+    /* children inherit the prefix coloring from the parent */
+    compp->attr = parentp->attr & ATTR_PREFIX;
+
+    /* most of the next would be abstracted elsewhere if doing other
+       filesystems */
+
+    /* scan parent for name; stream is converted to this parent */
+    file_cache_reset(stream->cachep);
+    stream->infop = &parentp->info;
+    fat_filestr_init(&stream->fatstr, &parentp->info.fatfile);
+    rewinddir_internal(&compp->info);
+
+    while ((rc = readdir_internal(stream, &compp->info, &dir_fatent)) > 0)
+    {
+        if (rc > 1 && !(callflags & FF_NOISO))
+            iso_decode_d_name(dir_fatent.name);
+
+        if (!strcasecmp(compname, dir_fatent.name))
+            break;
+    }
+
+    if (rc == 0)
+    {
+        DEBUGF("File/directory not found\n");
+        return -ENOENT;
+    }
+    else if (rc < 0)
+    {
+        DEBUGF("I/O error reading directory %d\n", rc);
+        return -EIO;
+    }
+
+    rc = fat_open(stream->fatstr.fatfilep, dir_fatent.firstcluster,
+                  &compp->info.fatfile);
+    if (rc < 0)
+    {
+        DEBUGF("I/O error opening file/directory %s (%d)\n",
+               compname, rc);
+        return -EIO;
+    }
+
+    walkp->filesize = dir_fatent.filesize;
+    compp->attr    |= dir_fatent.attr;
+
+    if (callflags & FF_CHECKPREFIX)
+        walk_check_prefix(walkp, compp);
+
+    return WALK_RC_FOUND;
+}
+
+/* parse a path component, open it and process the next */
+static NO_INLINE int
+walk_path(struct pathwalk *walkp, struct pathwalk_component *compp,
+          struct filestr_base *stream)
+{
+    int rc = WALK_RC_FOUND;
+
+    if (walkp->callflags & FF_CHECKPREFIX)
+        walk_check_prefix(walkp, compp);
+
+    /* alloca is used in a loop, but we reuse any blocks previously allocated
+       if we went up then back down; if the path takes us back to the root, then
+       everything is cleaned automatically */
+    struct pathwalk_component *freep = NULL;
+
+    const char *name;
+    ssize_t len;
+
+    while ((len = parse_path_component(&walkp->path, &name)))
+    {
+        /* whatever is to be a parent must be a directory */
+        if (!(compp->attr & ATTR_DIRECTORY))
+            return -ENOTDIR;
+
+        switch (len)
+        {
+        case 1:
+        case 2:
+            /* check for "." and ".." */
+            if (name[0] == '.')
+            {
+                if (len == 2 && name[1] == '.')
+                {
+                    struct pathwalk_component *parentp = compp->nextp;
+                    if (!parentp)
+                        return WALK_RC_CONT_AT_ROOT;
+
+                    compp->nextp = freep;
+                    freep = compp;
+                    compp = parentp;
+                }
+
+                break;
+            }
+
+        /* fallthrough */
+        default:
+            if (len >= MAX_NAME)
+                return -ENAMETOOLONG;
+
+            struct pathwalk_component *newp = freep;
+            if (!newp)
+                newp = pathwalk_comp_alloc(compp);
+            else
+                freep = freep->nextp;
+
+            newp->nextp = compp;
+            compp = newp;
+
+            compp->name = name;
+            compp->length = len;
+
+            rc = open_path_component(walkp, compp, stream);
+            if (rc < 0)
+                break;
+        }
+    }
+
+    return walk_open_info(walkp, compp, rc, stream);
+}
+
+/* open a stream given a path to the resource */
+int open_stream_internal(const char *path, unsigned int callflags,
+                         struct filestr_base *stream,
+                         struct path_component_info *compinfo)
+{
+    DEBUGF("%s(path=\"%s\",flg=%X,str=%p,compinfo=%p)\n", path, callflags,
+           stream, compinfo);
+    int rc;
+
+    filestr_base_init(stream);
+
+    if (!path_is_absolute(path))
+    {
+        /* while this supports relative components, there is currently no
+           current working directory concept at this level by which to
+           fully qualify the path (though that would not be excessively
+           difficult to add) */
+        DEBUGF("\"%s\" is not an absolute path\n"
+               "Only absolute paths currently supported.\n", path);
+        FILE_ERROR(path ? ENOENT : EFAULT, -1);
+    }
+
+    /* if !compinfo, then the result of this check is not visible anyway */
+    if (!compinfo)
+        callflags &= ~FF_CHECKPREFIX;
+
+    struct pathwalk walk;
+    walk.path      = path;
+    walk.callflags = callflags;
+    walk.compinfo  = compinfo;
+    walk.filesize  = 0;
+
+    struct pathwalk_component *rootp = pathwalk_comp_alloc(NULL);
+    rootp->nextp = NULL;
+    rootp->attr  = ATTR_DIRECTORY;
+
+#ifdef HAVE_MULTIVOLUME
+    int volume = 0, rootrc = WALK_RC_FOUND;
+#endif /* HAVE_MULTIVOLUME */
+
+    while (1)
+    {
+        const char *pathptr = walk.path;
+
+    #ifdef HAVE_MULTIVOLUME
+        /* this seamlessly integrates secondary filesystems into the
+           root namespace (e.g. "/<0>/../../<1>/../foo/." :<=> "/foo") */
+        const char *p;
+        volume = path_strip_volume(pathptr, &p, false);
+        if (!CHECK_VOL(volume))
+        {
+            DEBUGF("No such device or address: %d\n", volume);
+            FILE_ERROR(ENXIO, -2);
+        }
+
+        /* the root of this subpath is the system root? */
+        rootrc = p == pathptr ? WALK_RC_FOUND_ROOT : WALK_RC_FOUND;
+        walk.path = p;
+    #endif /* HAVE_MULTIVOLUME */
+
+        /* set name to start at last leading separator; names of volume
+           specifiers will be returned as "/<fooN>" */
+        rootp->name   = GOBBLE_PATH_SEPCH(pathptr) - 1;
+        rootp->length =
+            IF_MV( rootrc == WALK_RC_FOUND ? p - rootp->name : ) 1;
+
+        rc = fat_open_rootdir(IF_MV(volume,) &rootp->info.fatfile);
+        if (rc < 0)
+        {
+            /* not mounted */
+            DEBUGF("No such device or address: %d\n", IF_MV_VOL(volume));
+            rc = -ENXIO;
+            break;
+        }
+
+        get_rootinfo_internal(&rootp->info);
+        rc = walk_path(&walk, rootp, stream);
+        if (rc != WALK_RC_CONT_AT_ROOT)
+            break;
+    }
+
+    switch (rc)
+    {
+    case WALK_RC_FOUND_ROOT:
+        IF_MV( rc = rootrc; )
+    case WALK_RC_NOT_FOUND:
+    case WALK_RC_FOUND:
+        break;
+
+    default: /* utter, abject failure :`( */
+        DEBUGF("Open failed: rc=%d, errno=%d\n", rc, errno);
+        filestr_base_destroy(stream);
+        FILE_ERROR(-rc, -2);
+    }
+
+    file_cache_reset(stream->cachep);
+
+file_error:
+    return rc;
+}
+
+/* close the stream referenced by 'stream' */
+int close_stream_internal(struct filestr_base *stream)
+{
+    int rc;
+    unsigned int foflags = fileobj_get_flags(stream);
+
+    if ((foflags & (FO_SINGLE|FO_REMOVED)) == (FO_SINGLE|FO_REMOVED))
+    {
+        /* nothing is referencing it so now remove the file's data */
+        rc = fat_remove(&stream->infop->fatfile, FAT_RM_DATA);
+        if (rc < 0)
+        {
+            DEBUGF("I/O error removing file data: %d\n", rc);
+            FILE_ERROR(EIO, rc * 10 - 1);
+        }
+    }
+
+    rc = 0;
+file_error:
+    /* close no matter what */
+    fileop_onclose_internal(stream);
+    return rc;
+}
+
+/* create a new stream in the parent directory */
+int create_stream_internal(struct file_base_info *parentinfop,
+                           const char *basename, size_t length,
+                           unsigned int attr, unsigned int callflags,
+                           struct filestr_base *stream)
+{
+    /* assumes an attempt was made beforehand to open *stream with
+       open_stream_internal() which returned zero (successfully not found),
+       so does not initialize it here */
+    const char * const name = strmemdupa(basename, length);
+    DEBUGF("Creating \"%s\"\n", name);
+
+    struct file_base_info info;
+    int rc = fat_create_file(&parentinfop->fatfile, name, attr,
+                             &info.fatfile, get_dir_fatent_dircache());
+    if (rc < 0)
+    {
+        DEBUGF("Create failed: %d\n", rc);
+        FILE_ERROR(rc == FAT_RC_ENOSPC ? ENOSPC : EIO, rc * 10 - 1);
+    }
+
+    /* dir_fatent is implicit arg */
+    fileop_oncreate_internal(stream, &info, callflags, parentinfop, name);
+    rc = 0;
+file_error:
+    return rc;
+}
+
+/* removes files and directories - back-end to remove() and rmdir() */
+int remove_stream_internal(const char *path, struct filestr_base *stream,
+                           unsigned int callflags)
+{
+    /* Only FF_* flags should be in callflags */
+    int rc;
+
+    struct filestr_base opened_stream;
+    if (!stream)
+        stream = &opened_stream;
+
+    if (stream == &opened_stream)
+    {
+        /* no stream provided so open local one */
+        rc = open_stream_internal(path, callflags, stream, NULL);
+        if (rc < 0)
+        {
+            DEBUGF("Failed opening path: %d\n", rc);
+            FILE_ERROR(ERRNO, rc * 10 - 1);
+        }
+    }
+    /* else ignore the 'path' argument */
+
+    if (callflags & FF_DIR)
+    {
+        /* directory to be removed must be empty */
+        rc = test_dir_empty_internal(stream);
+        if (rc < 0)
+            FILE_ERROR(ERRNO, rc * 10 - 2);
+    }
+
+    /* save old info since fat_remove() will destroy the dir info */
+    struct file_base_info oldinfo = *stream->infop;
+    rc = fat_remove(&stream->infop->fatfile, FAT_RM_DIRENTRIES);
+    if (rc < 0)
+    {
+        DEBUGF("I/O error removing dir entries: %d\n", rc);
+        FILE_ERROR(EIO, rc * 10 - 3);
+    }
+
+    fileop_onremove_internal(stream, &oldinfo);
+
+    rc = 0;
+file_error:
+    if (stream == &opened_stream)
+    {
+        /* will do removal of data below if this is the only reference */
+        int rc2 = close_stream_internal(stream);
+        if (rc2 < 0 && rc >= 0)
+        {
+            rc = rc2 * 10 - 4;
+            DEBUGF("Success but failed closing stream: %d\n", rc);
+        }
+    }
+
+    return rc;
+}
+
+/* test file/directory existence with constraints */
+int test_stream_exists_internal(const char *path, unsigned int callflags)
+{
+    /* only FF_* flags should be in callflags */
+    struct filestr_base stream;
+    int rc = open_stream_internal(path, callflags, &stream, NULL);
+    if (rc > 0)
+        close_stream_internal(&stream);
+
+    return rc;
+}
+
+/* one-time init at startup */
+void filesystem_init(void)
+{
+    mrsw_init(&file_internal_mrsw);
+    dc_init();
+    fileobj_mgr_init();
+}
diff --git a/firmware/common/filefuncs.c b/firmware/common/filefuncs.c
deleted file mode 100644
index 16f8d88684..0000000000
--- a/firmware/common/filefuncs.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/***************************************************************************
- *             __________               __   ___.
- *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
- *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
- *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
- *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
- *                     \/            \/     \/    \/            \/
- * $Id$
- *
- * Copyright (C) 2002 by Björn Stenberg
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
- * KIND, either express or implied.
- *
- ****************************************************************************/
-#include "config.h"
-#include "dir.h"
-#include "stdlib.h"
-#include "string.h"
-#include "debug.h"
-#include "file.h"
-#include "filefuncs.h"
-#include "string-extra.h"
-
-#ifndef __PCTOOL__
-#ifdef HAVE_MULTIVOLUME
-
-/* returns on which volume this is, and copies the reduced name
-   (sortof a preprocessor for volume-decorated pathnames) */
-int strip_volume(const char* name, char* namecopy)
-{
-    int volume = 0;
-    const char *temp = name;
-
-    while (*temp == '/')          /* skip all leading slashes */
-        ++temp;
-
-    if (*temp && !strncmp(temp, VOL_NAMES, VOL_ENUM_POS))
-    {
-        temp += VOL_ENUM_POS;     /* behind special name */
-        volume = atoi(temp);      /* number is following */
-        temp = strchr(temp, '/'); /* search for slash behind */
-        if (temp != NULL)
-            name = temp;          /* use the part behind the volume */
-        else
-            name = "/";           /* else this must be the root dir */
-    }
-
-    strlcpy(namecopy, name, MAX_PATH);
-
-    return volume;
-}
-#endif /* #ifdef HAVE_MULTIVOLUME */
-
-#endif /* __PCTOOL__ */
-/* Test file existence, using dircache of possible */
-bool file_exists(const char *file)
-{
-    int fd;
-
-#ifdef DEBUG
-    if (!file || !*file)
-    {
-        DEBUGF("%s(%p): Invalid parameter!\n", __func__, file);
-        return false;
-    }
-#endif
-
-#ifdef HAVE_DIRCACHE
-    if (dircache_is_enabled())
-        return (dircache_get_entry_id(file) >= 0);
-#endif
-
-    fd = open(file, O_RDONLY);
-    if (fd < 0)
-        return false;
-    close(fd);
-    return true;
-}
-
-bool dir_exists(const char *path)
-{
-    DIR* d = opendir(path);
-    if (!d)
-        return false;
-    closedir(d);
-    return true;
-}
-
-
-#if (CONFIG_PLATFORM & PLATFORM_NATIVE) || defined(SIMULATOR)
-struct dirinfo dir_get_info(DIR* parent, struct dirent *entry)
-{
-    (void)parent;
-    return entry->info;
-}
-#endif
diff --git a/firmware/common/fileobj_mgr.c b/firmware/common/fileobj_mgr.c
new file mode 100644
index 0000000000..8e7831d36c
--- /dev/null
+++ b/firmware/common/fileobj_mgr.c
@@ -0,0 +1,396 @@
+/***************************************************************************
+ *             __________               __   ___.
+ *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
+ *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
+ *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
+ *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
+ *                     \/            \/     \/    \/            \/
+ * $Id$
+ *
+ * Copyright (C) 2014 by Michael Sevakis
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ****************************************************************************/
+#include "config.h"
+#include "system.h"
+#include "debug.h"
+#include "file.h"
+#include "dir.h"
+#include "disk_cache.h"
+#include "fileobj_mgr.h"
+#include "dircache_redirect.h"
+
+/**
+ * Manages file and directory streams on all volumes
+ *
+ * Intended for internal use by disk, file and directory code
+ */
+
+
+/* there will always be enough of these for all user handles, thus these
+   functions don't return failure codes */
+#define MAX_FILEOBJS (MAX_OPEN_HANDLES + AUX_FILEOBJS)
+
+/* describes the file as an image on the storage medium */
+static struct fileobj_binding
+{
+    struct file_base_binding bind;    /* base info list item (first!) */
+    uint16_t                 flags;   /* F(D)(O)_* bits of this file/dir */
+    uint16_t                 writers; /* number of writer streams */
+    struct filestr_cache     cache;   /* write mode shared cache */
+    file_size_t              size;    /* size of this file */
+    struct ll_head           list;    /* open streams for this file/dir */
+} fobindings[MAX_FILEOBJS];
+static struct mutex stream_mutexes[MAX_FILEOBJS] SHAREDBSS_ATTR;
+static struct ll_head free_bindings;
+static struct ll_head busy_bindings[NUM_VOLUMES];
+
+#define BUSY_BINDINGS(volume) \
+    (&busy_bindings[IF_MV_VOL(volume)])
+
+#define BASEBINDING_LIST(bindp) \
+    (BUSY_BINDINGS(BASEBINDING_VOL(bindp)))
+
+#define FREE_BINDINGS() \
+    (&free_bindings)
+
+#define BINDING_FIRST(type, volume...) \
+    ((struct fileobj_binding *)type##_BINDINGS(volume)->head)
+
+#define BINDING_NEXT(fobp) \
+    ((struct fileobj_binding *)(fobp)->bind.node.next)
+
+#define FOR_EACH_BINDING(volume, fobp) \
+    for (struct fileobj_binding *fobp = BINDING_FIRST(BUSY, volume); \
+         fobp; fobp = BINDING_NEXT(fobp))
+
+#define STREAM_FIRST(fobp) \
+    ((struct filestr_base *)(fobp)->list.head)
+
+#define STREAM_NEXT(s) \
+    ((struct filestr_base *)(s)->node.next)
+
+#define STREAM_THIS(s) \
+    (s)
+
+#define FOR_EACH_STREAM(what, start, s) \
+    for (struct filestr_base *s = STREAM_##what(start); \
+         s; s = STREAM_NEXT(s))
+
+
+/* syncs information for the stream's old and new parent directory if any are
+   currently opened */
+static void fileobj_sync_parent(const struct file_base_info *infop[],
+                                int count)
+{
+    FOR_EACH_BINDING(infop[0]->volume, fobp)
+    {
+        if ((fobp->flags & (FO_DIRECTORY|FO_REMOVED)) != FO_DIRECTORY)
+            continue; /* not directory or removed can't be parent of anything */
+
+        struct filestr_base *parentstrp = STREAM_FIRST(fobp);
+        struct fat_file *parentfilep = &parentstrp->infop->fatfile;
+
+        for (int i = 0; i < count; i++)
+        {
+            if (!fat_dir_is_parent(parentfilep, &infop[i]->fatfile))
+                continue;
+
+            /* discard scan/read caches' parent dir info */
+            FOR_EACH_STREAM(THIS, parentstrp, s)
+                filestr_discard_cache(s);
+        }
+    }
+}
+
+/* see if this file has open streams and return that fileobj_binding if so,
+   else grab a new one from the free list; returns true if this stream is
+   the only open one */
+static bool binding_assign(const struct file_base_info *srcinfop,
+                           struct fileobj_binding **fobpp)
+{
+    FOR_EACH_BINDING(srcinfop->fatfile.volume, fobp)
+    {
+        if (fobp->flags & FO_REMOVED)
+            continue;
+
+        if (fat_file_is_same(&srcinfop->fatfile, &fobp->bind.info.fatfile))
+        {
+            /* already has open streams */
+            *fobpp = fobp;
+            return false;
+        }
+    }
+
+    /* not found - allocate anew */
+    *fobpp = BINDING_FIRST(FREE);
+    ll_remove_first(FREE_BINDINGS());
+    ll_init(&(*fobpp)->list);
+    return true;
+}
+
+/* mark descriptor as unused and return to the free list */
+static void binding_add_to_free_list(struct fileobj_binding *fobp)
+{
+    fobp->flags = 0;
+    ll_insert_last(FREE_BINDINGS(), &fobp->bind.node);
+}
+
+/** File and directory internal interface **/
+
+void file_binding_insert_last(struct file_base_binding *bindp)
+{
+    ll_insert_last(BASEBINDING_LIST(bindp), &bindp->node);
+}
+
+void file_binding_remove(struct file_base_binding *bindp)
+{
+    ll_remove(BASEBINDING_LIST(bindp), &bindp->node);
+}
+
+#ifdef HAVE_DIRCACHE
+void file_binding_insert_first(struct file_base_binding *bindp)
+{
+    ll_insert_first(BASEBINDING_LIST(bindp), &bindp->node);
+}
+
+void file_binding_remove_next(struct file_base_binding *prevp,
+                              struct file_base_binding *bindp)
+{
+    ll_remove_next(BASEBINDING_LIST(bindp), &prevp->node);
+    (void)bindp;
+}
+#endif /* HAVE_DIRCACHE */
+
+/* opens the file object for a new stream and sets up the caches;
+ * the stream must already be opened at the FS driver level and *stream
+ * initialized.
+ *
+ * NOTE: switches stream->infop to the one kept in common for all streams of
+ *       the same file, making a copy for only the first stream
+ */
+void fileobj_fileop_open(struct filestr_base *stream,
+                         const struct file_base_info *srcinfop,
+                         unsigned int callflags)
+{
+    struct fileobj_binding *fobp;
+    bool first = binding_assign(srcinfop, &fobp);
+
+    /* add stream to this file's list */
+    ll_insert_last(&fobp->list, &stream->node);
+
+    /* initiate the new stream into the enclave */
+    stream->flags = FDO_BUSY | (callflags & (FD_WRITE|FD_WRONLY|FD_APPEND));
+    stream->infop = &fobp->bind.info;
+    stream->fatstr.fatfilep = &fobp->bind.info.fatfile;
+    stream->bindp = &fobp->bind;
+    stream->mtx   = &stream_mutexes[fobp - fobindings];
+
+    if (first)
+    {
+        /* first stream for file */
+        fobp->bind.info = *srcinfop;
+        fobp->flags     = FDO_BUSY | FO_SINGLE |
+                          (callflags & (FO_DIRECTORY|FO_TRUNC));
+        fobp->writers   = 0;
+        fobp->size      = 0;
+
+        if (callflags & FD_WRITE)
+        {
+            /* first one is a writer */
+            fobp->writers = 1;
+            file_cache_init(&fobp->cache);
+            filestr_assign_cache(stream, &fobp->cache);
+        }
+
+        fileobj_bind_file(&fobp->bind);
+    }
+    else
+    {
+        /* additional stream for file */
+        fobp->flags &= ~FO_SINGLE;
+        fobp->flags |= callflags & FO_TRUNC;
+
+        /* once a file/directory, always a file/directory; such a change
+           is a bug */
+        if ((callflags ^ fobp->flags) & FO_DIRECTORY)
+        {
+            DEBUGF("%s - FO_DIRECTORY flag does not match: %p %u\n",
+                   __func__, stream, callflags);
+        }
+
+        if (fobp->writers)
+        {
+            /* already writers present */
+            fobp->writers++;
+            filestr_assign_cache(stream, &fobp->cache);
+        }
+        else if (callflags & FD_WRITE)
+        {
+            /* first writer */
+            fobp->writers = 1;
+            file_cache_init(&fobp->cache);
+            FOR_EACH_STREAM(FIRST, fobp, s)
+                filestr_assign_cache(s, &fobp->cache);
+        }
+        /* else another reader */
+    }
+}
+
+/* close the stream and free associated resources */
+void fileobj_fileop_close(struct filestr_base *stream)
+{
+    switch (stream->flags)
+    {
+    case 0:              /* not added to manager */
+    case FV_NONEXIST:    /* forced-closed by unmounting */
+        filestr_base_destroy(stream);
+        return;
+    }
+
+    struct fileobj_binding *fobp = (struct fileobj_binding *)stream->bindp;
+    unsigned int foflags = fobp->flags;
+
+    ll_remove(&fobp->list, &stream->node);
+
+    if ((foflags & FO_SINGLE) || fobp->writers == 0)
+    {
+        if (foflags & FO_SINGLE)
+        {
+            /* last stream for file; close everything */
+            fileobj_unbind_file(&fobp->bind);
+
+            if (fobp->writers)
+                file_cache_free(&fobp->cache);
+
+            binding_add_to_free_list(fobp);
+        }
+    }
+    else if ((stream->flags & FD_WRITE) && --fobp->writers == 0)
+    {
+        /* only readers remain; switch back to stream-local caching */
+        FOR_EACH_STREAM(FIRST, fobp, s)
+            filestr_copy_cache(s, &fobp->cache);
+
+        file_cache_free(&fobp->cache);
+    }
+
+    if (!(foflags & FO_SINGLE) && fobp->list.head == fobp->list.tail)
+        fobp->flags |= FO_SINGLE; /* only one open stream remaining */
+
+    filestr_base_destroy(stream);
+}
+
+/* informs manager that file has been created */
+void fileobj_fileop_create(struct filestr_base *stream,
+                           const struct file_base_info *srcinfop,
+                           unsigned int callflags)
+{
+    fileobj_fileop_open(stream, srcinfop, callflags);
+    fileobj_sync_parent((const struct file_base_info *[]){ stream->infop }, 1);
+}
+
+/* informs manager that file has been removed */
+void fileobj_fileop_remove(struct filestr_base *stream,
+                           const struct file_base_info *oldinfop)
+{
+    ((struct fileobj_binding *)stream->bindp)->flags |= FO_REMOVED;
+    fileobj_sync_parent((const struct file_base_info *[]){ oldinfop }, 1);
+}
+
+/* informs manager that file has been renamed */
+void fileobj_fileop_rename(struct filestr_base *stream,
+                           const struct file_base_info *oldinfop)
+{
+    /* if there is old info then this was a move and the old parent has to be
+       informed */
+    int count = oldinfop ? 2 : 1;
+    fileobj_sync_parent(&(const struct file_base_info *[])
+                            { oldinfop, stream->infop }[2 - count],
+                        count);
+}
+
+/* informs manager than directory entries have been updated */
+void fileobj_fileop_sync(struct filestr_base *stream)
+{
+    fileobj_sync_parent((const struct file_base_info *[]){ stream->infop }, 1);
+}
+
+/* inform manager that file has been truncated */
+void fileobj_fileop_truncate(struct filestr_base *stream)
+{
+    /* let caller update internal info */
+    FOR_EACH_STREAM(FIRST, (struct fileobj_binding *)stream->bindp, s)
+        ftruncate_internal_callback(stream, s);
+}
+
+/* query for the pointer to the size storage for the file object */
+file_size_t * fileobj_get_sizep(const struct filestr_base *stream)
+{
+    if (!stream->bindp)
+        return NULL;
+
+    return &((struct fileobj_binding *)stream->bindp)->size;
+}
+
+/* query manager bitflags for the file object */
+unsigned int fileobj_get_flags(const struct filestr_base *stream)
+{
+    if (!stream->bindp)
+        return 0;
+
+    return ((struct fileobj_binding *)stream->bindp)->flags;
+}
+
+/* change manager bitflags for the file object */
+void fileobj_change_flags(struct filestr_base *stream,
+                          unsigned int flags, unsigned int mask)
+{
+    struct fileobj_binding *fobp = (struct fileobj_binding *)stream->bindp;
+    if (fobp)
+        fobp->flags = (fobp->flags & ~mask) | (flags & mask);
+}
+
+/* mark all open streams on a device as "nonexistant" */
+void fileobj_mgr_unmount(IF_MV_NONVOID(int volume))
+{
+    /* right now, there is nothing else to be freed when marking a descriptor
+       as "nonexistant" but a callback could be added if that changes */
+    FOR_EACH_VOLUME(volume, v)
+    {
+        struct fileobj_binding *fobp;
+        while ((fobp = BINDING_FIRST(BUSY, v)))
+        {
+            struct filestr_base *s;
+            while ((s = STREAM_FIRST(fobp)))
+            {
+                /* keep it "busy" to avoid races; any valid file/directory
+                   descriptor returned by an open call should always be
+                   closed by whomever opened it (of course!) */
+                fileop_onclose_internal(s);
+                s->flags = FV_NONEXIST;
+            }
+        }
+    }
+}
+
+/* one-time init at startup */
+void fileobj_mgr_init(void)
+{
+    for (unsigned int i = 0; i < NUM_VOLUMES; i++)
+        ll_init(BUSY_BINDINGS(i));
+
+    ll_init(FREE_BINDINGS());
+    for (unsigned int i = 0; i < MAX_FILEOBJS; i++)
+    {
+        mutex_init(&stream_mutexes[i]);
+        binding_add_to_free_list(&fobindings[i]);
+    }
+}
diff --git a/firmware/common/pathfuncs.c b/firmware/common/pathfuncs.c
new file mode 100644
index 0000000000..4410275adb
--- /dev/null
+++ b/firmware/common/pathfuncs.c
@@ -0,0 +1,421 @@
+/***************************************************************************
+ *             __________               __   ___.
+ *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
+ *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
+ *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
+ *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
+ *                     \/            \/     \/    \/            \/
+ * $Id$
+ *
+ * Copyright (C) 2014 by Michael Sevakis
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ****************************************************************************/
+#include <string.h>
+#include <ctype.h>
+#include "system.h"
+#include "pathfuncs.h"
+#include "string-extra.h"
+
+#ifdef HAVE_MULTIVOLUME
+#include <stdio.h>
+#include "storage.h"
+
+enum storage_name_dec_indexes
+{
+#if (CONFIG_STORAGE & STORAGE_ATA)
+    STORAGE_DEC_IDX_ATA,
+#endif
+#if (CONFIG_STORAGE & STORAGE_MMC)
+    STORAGE_DEC_IDX_MMC,
+#endif
+#if (CONFIG_STORAGE & STORAGE_SD)
+    STORAGE_DEC_IDX_SD,
+#endif
+#if (CONFIG_STORAGE & STORAGE_NAND)
+    STORAGE_DEC_IDX_NAND,
+#endif
+#if (CONFIG_STORAGE & STORAGE_RAMDISK)
+    STORAGE_DEC_IDX_RAMDISK,
+#endif
+#if (CONFIG_STORAGE & STORAGE_HOSTFS)
+    STORAGE_DEC_IDX_HOSTFS,
+#endif
+    STORAGE_NUM_DEC_IDX,
+};
+
+static const char * const storage_dec_names[STORAGE_NUM_DEC_IDX+1] =
+{
+#if (CONFIG_STORAGE & STORAGE_ATA)
+    [STORAGE_DEC_IDX_ATA]     = ATA_VOL_DEC,
+#endif
+#if (CONFIG_STORAGE & STORAGE_MMC)
+    [STORAGE_DEC_IDX_MMC]     = MMC_VOL_DEC,
+#endif
+#if (CONFIG_STORAGE & STORAGE_SD)
+    [STORAGE_DEC_IDX_SD]      = SD_VOL_DEC,
+#endif
+#if (CONFIG_STORAGE & STORAGE_NAND)
+    [STORAGE_DEC_IDX_NAND]    = NAND_VOL_DEC,
+#endif
+#if (CONFIG_STORAGE & STORAGE_RAMDISK)
+    [STORAGE_DEC_IDX_RAMDISK] = RAMDISK_VOL_DEC,
+#endif
+#if (CONFIG_STORAGE & STORAGE_HOSTFS)
+    [STORAGE_DEC_IDX_HOSTFS]  = HOSTFS_VOL_DEC,
+#endif
+    [STORAGE_NUM_DEC_IDX]     = DEFAULT_VOL_DEC,
+};
+
+static const unsigned char storage_dec_indexes[STORAGE_NUM_TYPES+1] =
+{
+    [0 ... STORAGE_NUM_TYPES] = STORAGE_NUM_DEC_IDX,
+#if (CONFIG_STORAGE & STORAGE_ATA)
+    [STORAGE_ATA_NUM]     = STORAGE_DEC_IDX_ATA,
+#endif
+#if (CONFIG_STORAGE & STORAGE_MMC)
+    [STORAGE_MMC_NUM]     = STORAGE_DEC_IDX_MMC,
+#endif
+#if (CONFIG_STORAGE & STORAGE_SD)
+    [STORAGE_SD_NUM]      = STORAGE_DEC_IDX_SD,
+#endif
+#if (CONFIG_STORAGE & STORAGE_NAND)
+    [STORAGE_NAND_NUM]    = STORAGE_DEC_IDX_NAND,
+#endif
+#if (CONFIG_STORAGE & STORAGE_RAMDISK)
+    [STORAGE_RAMDISK_NUM] = STORAGE_DEC_IDX_RAMDISK,
+#endif
+#if (CONFIG_STORAGE & STORAGE_HOSTFS)
+    [STORAGE_HOSTFS_NUM]  = STORAGE_DEC_IDX_HOSTFS,
+#endif
+};
+
+/* Returns on which volume this is and sets *nameptr to the portion of the
+ * path after the volume specifier, which could be the null if the path is
+ * just a volume root. If *nameptr > name, then a volume specifier was
+ * found. If 'greedy' is 'true', then it all separators after the volume
+ * specifier are consumed, if one was found.
+ */
+int path_strip_volume(const char *name, const char **nameptr, bool greedy)
+{
+    int volume = 0;
+    const char *t = name;
+    int c, v = 0;
+
+    /* format: "/<xxx##>/foo/bar"
+     * the "xxx" is pure decoration; only an unbroken trailing string of
+     * digits within the brackets is parsed as the volume number and of
+     * those, only the last ones VOL_MUM_MAX allows.
+     */
+    c = *(t = GOBBLE_PATH_SEPCH(t)); /* skip all leading slashes */
+    if (c != VOL_START_TOK)     /* missing start token? no volume */
+        goto volume0;
+
+    do
+    {
+        switch (c)
+        {
+        case '0' ... '9':       /* digit; parse volume number */
+            v = (v * 10 + c - '0') % VOL_NUM_MAX;
+            break;
+        case '\0':
+        case PATH_SEPCH:        /* no closing bracket; no volume */
+            goto volume0;
+        default:                /* something else; reset volume */
+            v = 0;
+        }
+    }
+    while ((c = *++t) != VOL_END_TOK);  /* found end token? */
+
+    if (!(c = *++t))            /* no more path and no '/' is ok */
+        ;
+    else if (c != PATH_SEPCH)   /* more path and no separator after end */
+        goto volume0;
+    else if (greedy)
+        t = GOBBLE_PATH_SEPCH(++t); /* strip remaining separators */
+
+    /* if 'greedy' is true and **nameptr == '\0' then it's only a volume
+       root whether or not it has trailing separators */
+
+    volume = v;
+    name = t;
+volume0:
+    if (nameptr)
+        *nameptr = name;
+    return volume;
+}
+
+/* Returns the volume specifier decorated with the storage type name.
+ * Assumes the supplied buffer size is at least {VOL_MAX_LEN}+1.
+ */
+int get_volume_name(int volume, char *buffer)
+{
+    if (volume < 0)
+    {
+        *buffer = '\0';
+        return 0;
+    }
+
+    volume %= VOL_NUM_MAX; /* as path parser would have it */
+
+    int type = storage_driver_type(volume_drive(volume));
+    if (type < 0 || type > STORAGE_NUM_TYPES)
+        type = STORAGE_NUM_TYPES;
+
+    const char *voldec = storage_dec_names[storage_dec_indexes[type]];
+    return snprintf(buffer, VOL_MAX_LEN + 1, "%c%s%d%c",
+                    VOL_START_TOK, voldec, volume, VOL_END_TOK);
+}
+#endif /* HAVE_MULTIVOLUME */
+
+/* Just like path_strip_volume() but strips a leading drive specifier and
+ * returns the drive number (A=0, B=1, etc.). -1 means no drive was found.
+ * If 'greedy' is 'true', all separators after the volume are consumed.
+ */
+int path_strip_drive(const char *name, const char **nameptr, bool greedy)
+{
+    int c = toupper(*name);
+
+    if (c >= 'A' && c <= 'Z' && name[1] == PATH_DRVSEPCH)
+    {
+        name = &name[2];
+        if (greedy)
+            name = GOBBLE_PATH_SEPCH(name);
+
+        *nameptr = name;
+        return c - 'A';
+    }
+
+    *nameptr = name;
+    return -1;
+}
+
+/* Strips leading and trailing whitespace from a path
+ * "  a/b \txyz"  *nameptr->a, len=3: "a/b"
+ */
+size_t path_trim_whitespace(const char *name, const char **nameptr)
+{
+    int c;
+    while ((c = *name) <= ' ' && c)
+        ++name;
+
+    const char *first = name;
+    const char *last = name;
+
+    while (1)
+    {
+        if (c < ' ')
+        {
+            *nameptr = first;
+            return last - first;
+        }
+
+        while ((c = *++name) > ' ');
+        last = name;
+        while (c == ' ') c = *++name;
+    }
+}
+
+/* Strips directory components from the path
+ * ""      *nameptr->NUL,   len=0: ""
+ * "/"     *nameptr->/,     len=1: "/"
+ * "//"    *nameptr->2nd /, len=1: "/"
+ * "/a"    *nameptr->a,     len=1: "a"
+ * "/a/bc" *nameptr->b,     len=2: "bc"
+ * "d"     *nameptr->d,     len=1: "d"
+ * "ef/gh" *nameptr->g,     len=2: "gh"
+ */
+size_t path_basename(const char *name, const char **nameptr)
+{
+    const char *p = name;
+    const char *q = p;
+    const char *r = q;
+
+    while (*(p = GOBBLE_PATH_SEPCH(p)))
+    {
+        q = p;
+        p = GOBBLE_PATH_COMP(++p);
+        r = p;
+    }
+
+    if (r == name && p > name)
+        q = p, r = q--; /* root - return last slash */
+    /* else path is an empty string */
+
+    *nameptr = q;
+    return r - q;
+}
+
+/* Strips the trailing component from the path
+ * ""      *nameptr->NUL,   len=0: ""
+ * "/"     *nameptr->/,     len=1: "/"
+ * "//"    *nameptr->2nd /, len=1: "/"
+ * "/a"    *nameptr->/,     len=1: "/"
+ * "/a/bc" *nameptr->/,     len=2: "/a"
+ * "d"     *nameptr->d,     len=0: ""
+ * "ef/gh" *nameptr->e,     len=2: "ef"
+ */
+size_t path_dirname(const char *name, const char **nameptr)
+{
+    const char *p = GOBBLE_PATH_SEPCH(name);
+    const char *q = name;
+    const char *r = p;
+
+    while (*(p = GOBBLE_PATH_COMP(p)))
+    {
+        const char *s = p;
+
+        if (!*(p = GOBBLE_PATH_SEPCH(p)))
+            break;
+
+        q = s;
+    }
+
+    if (q == name && r > name)
+        name = r, q = name--; /* root - return last slash */
+
+    *nameptr = name;
+    return q - name;
+}
+
+/* Removes trailing separators from a path
+ * ""     *nameptr->NUL,   len=0: ""
+ * "/"    *nameptr->/,     len=1: "/"
+ * "//"   *nameptr->2nd /, len=1: "/"
+ * "/a/"  *nameptr->/,     len=2: "/a"
+ * "//b/" *nameptr->1st /, len=3: "//b"
+ * "/c/"  *nameptr->/,     len=2: "/c"
+ */
+size_t path_strip_trailing_separators(const char *name, const char **nameptr)
+{
+    const char *p;
+    size_t len = path_basename(name, &p);
+
+    if (len == 1 && *p == '/' && p > name)
+    {
+        *nameptr = p;
+        name = p - 1; /* root with multiple separators */
+    }
+    else
+    {
+        *nameptr = name;
+        p += len; /* length to end of basename */
+    }
+
+    return p - name;
+}
+
+/* Transforms "wrong" separators into the correct ones
+ * "c:\windows\system32" -> "c:/windows/system32"
+ *
+ * 'path' and 'dstpath' may either be the same buffer or non-overlapping
+ */
+void path_correct_separators(char *dstpath, const char *path)
+{
+    char *dstp = dstpath;
+    const char *p = path;
+
+    while (1)
+    {
+        const char *next = strchr(p, PATH_BADSEPCH);
+        if (!next)
+            break;
+
+        size_t size = next - p;
+
+        if (dstpath != path)
+            memcpy(dstp, p, size); /* not in-place */
+
+        dstp += size;
+        *dstp++ = PATH_SEPCH;
+        p = next + 1;
+    }
+
+    if (dstpath != path)
+        strcpy(dstp, p);
+}
+
+/* Appends one path to another, adding separators between components if needed.
+ * Return value and behavior is otherwise as strlcpy so that truncation may be
+ * detected.
+ *
+ * For basepath and component:
+ * PA_SEP_HARD adds a separator even if the base path is empty
+ * PA_SEP_SOFT adds a separator only if the base path is not empty
+ */
+size_t path_append(char *buf, const char *basepath,
+                   const char *component, size_t bufsize)
+{
+    const char *base = basepath && basepath[0] ? basepath : buf;
+    if (!base)
+        return bufsize; /* won't work to get lengths from buf */
+
+    if (!buf)
+        bufsize = 0;
+
+    if (path_is_absolute(component))
+    {
+        /* 'component' is absolute; replace all */
+        basepath = component;
+        component = "";
+    }
+
+    /* if basepath is not null or empty, buffer contents are replaced,
+       otherwise buf contains the base path */
+    size_t len = base == buf ? strlen(buf) : strlcpy(buf, basepath, bufsize);
+
+    bool separate = false;
+
+    if (!basepath || !component)
+        separate = !len || base[len-1] != PATH_SEPCH;
+    else if (component[0])
+        separate = len && base[len-1] != PATH_SEPCH;
+
+    /* caller might lie about size of buf yet use buf as the base */
+    if (base == buf && bufsize && len >= bufsize)
+        buf[bufsize - 1] = '\0';
+
+    buf += len;
+    bufsize -= MIN(len, bufsize);
+
+    if (separate && (len++, bufsize > 0) && --bufsize > 0)
+        *buf++ = PATH_SEPCH;
+
+    return len + strlcpy(buf, component ?: "", bufsize);
+}
+
+/* Returns the location and length of the next path component, consuming the
+ * input in the process.
+ *
+ * "/a/bc/d" breaks into:
+ *   start:  *namep->1st /
+ *   call 1: *namep->a,   *pathp->2nd / len=1: "a"
+ *   call 2: *namep->b,   *pathp->3rd / len=2: "bc"
+ *   call 3: *namep->d,   *pathp->NUL,  len=1: "d"
+ *   call 4: *namep->NUL, *pathp->NUL,  len=0: ""
+ *
+ * Returns: 0 if the input has been consumed
+ *          The length of the component otherwise
+ */
+ssize_t parse_path_component(const char **pathp, const char **namep)
+{
+    /* a component starts at a non-separator and continues until the next
+       separator or null */
+    const char *p = GOBBLE_PATH_SEPCH(*pathp);
+    const char *name = p;
+
+    if (*p)
+        p = GOBBLE_PATH_COMP(++p);
+
+    *pathp = p;
+    *namep = name;
+
+    return p - name;
+}
diff --git a/firmware/common/rbpaths.c b/firmware/common/rbpaths.c
deleted file mode 100644
index 69543bc3a7..0000000000
--- a/firmware/common/rbpaths.c
+++ /dev/null
@@ -1,432 +0,0 @@
-/***************************************************************************
- *             __________               __   ___.
- *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
- *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
- *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
- *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
- *                     \/            \/     \/    \/            \/
- * $Id$
- *
- * Copyright (C) 2010 Thomas Martitz
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
- * KIND, either express or implied.
- *
- ****************************************************************************/
-
-
-#include <stdio.h> /* snprintf */
-#include <stdlib.h>
-#include <stdarg.h>
-#include <sys/stat.h>
-#include <time.h>
-#include <unistd.h>
-#include "config.h"
-#include "rbpaths.h"
-#include "crc32.h"
-#include "file.h" /* MAX_PATH */
-#include "logf.h"
-#include "gcc_extensions.h"
-#include "string-extra.h"
-#include "filefuncs.h"
-
-/* In this file we need the actual OS library functions, not the shadowed
- * wrapper used within Rockbox' application code (except SDL adds
- * another layer) */
-#undef open
-#undef creat
-#undef remove
-#undef rename
-#undef opendir
-#undef closedir
-#undef readdir
-#undef mkdir
-#undef rmdir
-#undef dirent
-#undef DIR
-#undef readlink
-
-#if (CONFIG_PLATFORM & PLATFORM_ANDROID)
-static const char rbhome[] = "/sdcard";
-#elif (CONFIG_PLATFORM & (PLATFORM_SDL|PLATFORM_MAEMO|PLATFORM_PANDORA)) && !defined(__PCTOOL__)
-const char *rbhome;
-#else
-/* YPR0, YPR1 */
-static const char rbhome[] = HOME_DIR;
-#endif
-
-#if !(defined(SAMSUNG_YPR0) || defined(SAMSUNG_YPR1)) && !defined(__PCTOOL__)
-/* Special dirs are user-accessible (and user-writable) dirs which take priority
- * over the ones where Rockbox is installed to. Classic example would be
- * $HOME/.config/rockbox.org vs /usr/share/rockbox */
-#define HAVE_SPECIAL_DIRS
-#endif
-
-/* flags for get_user_file_path() */
-/* whether you need write access to that file/dir, especially true
- * for runtime generated files (config.cfg) */
-#define NEED_WRITE          (1<<0)
-/* file or directory? */
-#define IS_FILE             (1<<1)
-
-#ifdef HAVE_MULTIDRIVE
-static uint32_t rbhome_hash;
-/* A special link is created under e.g. HOME_DIR/<microSD1>, e.g. to access
- * external storage in a convinient location, much similar to the mount
- * point on our native targets. Here they are treated as symlink (one which
- * doesn't actually exist in the filesystem and therefore we have to override
- * readlink() */
-static const char *handle_special_links(const char* link, unsigned flags,
-                                char *buf, const size_t bufsize)
-{
-    (void) flags;
-    char vol_string[VOL_ENUM_POS + 8];
-    int len = sprintf(vol_string, VOL_NAMES, 1);
-
-    /* link might be passed with or without HOME_DIR expanded. To handle
-     * both perform substring matching (VOL_NAMES is unique enough) */
-    const char *begin = strstr(link, vol_string);
-    if (begin)
-    {
-        /* begin now points to the start of vol_string within link,
-         * we want to copy the remainder of the paths, prefixed by
-         * the actual mount point (the remainder might be "") */
-        snprintf(buf, bufsize, MULTIDRIVE_DIR"%s", begin + len);
-        return buf;
-    }
-
-    return link;
-}
-#endif
-
-void paths_init(void)
-{
-#ifdef HAVE_SPECIAL_DIRS
-    /* make sure $HOME/.config/rockbox.org exists, it's needed for config.cfg */
-#if (CONFIG_PLATFORM & PLATFORM_ANDROID)
-    mkdir("/sdcard/rockbox", 0777);
-    mkdir("/sdcard/rockbox/rocks.data", 0777);
-#else
-    char config_dir[MAX_PATH];
-
-    const char *home = getenv("RBROOT");
-    if (!home)
-    {
-        home = getenv("HOME");
-    }
-    if (!home)
-    {
-      logf("HOME environment var not set. Can't write config");
-      return;
-    }
-
-    rbhome = home;
-    snprintf(config_dir, sizeof(config_dir), "%s/.config", home);
-    mkdir(config_dir, 0777);
-    snprintf(config_dir, sizeof(config_dir), "%s/.config/rockbox.org", home);
-    mkdir(config_dir, 0777);
-    /* Plugin data directory */
-    snprintf(config_dir, sizeof(config_dir), "%s/.config/rockbox.org/rocks.data", home);
-    mkdir(config_dir, 0777);
-#endif
-#endif /* HAVE_SPECIAL_DIRS */
-
-#ifdef HAVE_MULTIDRIVE
-    rbhome_hash = crc_32((const void *) rbhome, strlen(rbhome), 0xffffffff);
-#endif /* HAVE_MULTIDRIVE */
-}
-
-#ifdef HAVE_SPECIAL_DIRS
-static bool try_path(const char* filename, unsigned flags)
-{
-    if (flags & IS_FILE)
-    {
-        if (file_exists(filename))
-            return true;
-    }
-    else
-    {
-        if (dir_exists(filename))
-            return true;
-    }
-    return false;
-}
-
-static const char* _get_user_file_path(const char *path,
-                                       unsigned flags,
-                                       char* buf,
-                                       const size_t bufsize)
-{
-    const char *ret = path;
-    const char *pos = path;
-    /* replace ROCKBOX_DIR in path with $HOME/.config/rockbox.org */
-    pos += ROCKBOX_DIR_LEN;
-    if (*pos == '/') pos += 1;
-
-#if (CONFIG_PLATFORM & PLATFORM_ANDROID)
-    if (snprintf(buf, bufsize, "/sdcard/rockbox/%s", pos)
-#else
-    if (snprintf(buf, bufsize, "%s/.config/rockbox.org/%s", rbhome, pos)
-#endif
-            >= (int)bufsize)
-        return NULL;
-
-    /* always return the replacement buffer (pointing to $HOME) if
-     * write access is needed */
-    if (flags & NEED_WRITE)
-        ret = buf;
-    else if (try_path(buf, flags))
-        ret = buf;
-        
-    if (ret != buf) /* not found in $HOME, try ROCKBOX_BASE_DIR, !NEED_WRITE only */
-    {
-        if (snprintf(buf, bufsize, ROCKBOX_SHARE_PATH "/%s", pos) >= (int)bufsize)
-            return NULL;
-
-        if (try_path(buf, flags))
-            ret = buf;
-    }
-
-    return ret;
-}
-
-#endif
-
-static const char* handle_special_dirs(const char* dir, unsigned flags,
-                                char *buf, const size_t bufsize)
-{
-    (void) flags; (void) buf; (void) bufsize;
-#ifdef HAVE_SPECIAL_DIRS
-    if (!strncmp(HOME_DIR, dir, HOME_DIR_LEN))
-    {
-        const char *p = dir + HOME_DIR_LEN;
-        while (*p == '/') p++;
-        snprintf(buf, bufsize, "%s/%s", rbhome, p);
-        dir = buf;
-    }
-    else if (!strncmp(ROCKBOX_DIR, dir, ROCKBOX_DIR_LEN))
-        dir = _get_user_file_path(dir, flags, buf, bufsize);
-#endif
-#ifdef HAVE_MULTIDRIVE
-    dir = handle_special_links(dir, flags, buf, bufsize);
-#endif
-    return dir;
-}
-
-int app_open(const char *name, int o, ...)
-{
-    char realpath[MAX_PATH];
-    va_list ap;
-    int fd;
-    int flags = IS_FILE;
-    if (o & (O_CREAT|O_RDWR|O_TRUNC|O_WRONLY))
-        flags |= NEED_WRITE;
-    
-    name = handle_special_dirs(name, flags, realpath, sizeof(realpath));
-
-    va_start(ap, o);
-    fd = open(name, o, va_arg(ap, unsigned int));
-    va_end(ap);
-    
-    return fd;
-}
-
-int app_creat(const char* name, mode_t mode)
-{
-    return app_open(name, O_CREAT|O_WRONLY|O_TRUNC, mode);
-}
-
-int app_remove(const char *name)
-{
-    char realpath[MAX_PATH];
-    const char *fname = handle_special_dirs(name, NEED_WRITE, realpath, sizeof(realpath));
-
-    return remove(fname);
-}
-
-int app_rename(const char *old, const char *new)
-{
-    char realpath_old[MAX_PATH], realpath_new[MAX_PATH];
-    const char *final_old, *final_new;
-
-    final_old = handle_special_dirs(old, NEED_WRITE, realpath_old, sizeof(realpath_old));
-    final_new = handle_special_dirs(new, NEED_WRITE, realpath_new, sizeof(realpath_new));
-
-    return rename(final_old, final_new);
-}
-
-/* need to wrap around DIR* because we need to save the parent's
- * directory path in order to determine dirinfo, required to implement
- * get_dir_info() */
-struct __dir {
-    DIR *dir;
-#ifdef HAVE_MULTIDRIVE
-    int volumes_returned;
-    /* A crc of rbhome is used to speed op the common case where
-     * readdir()/get_dir_info() is called on non-rbhome paths, because
-     * each call needs to check against rbhome for the virtual
-     * mount point of the external storage */
-    uint32_t path_hash;
-#endif
-    char path[];
-};
-
-struct dirinfo dir_get_info(DIR* _parent, struct dirent *dir)
-{
-    struct __dir *parent = (struct __dir*)_parent;
-    struct stat s;
-    struct tm *tm = NULL;
-    struct dirinfo ret;
-    char path[MAX_PATH];
-
-    memset(&ret, 0, sizeof(ret));
-
-#ifdef HAVE_MULTIDRIVE
-    char vol_string[VOL_ENUM_POS + 8];
-    sprintf(vol_string, VOL_NAMES, 1);
-    if (UNLIKELY(rbhome_hash == parent->path_hash) &&
-            /* compare path anyway because of possible hash collision */
-            !strcmp(vol_string, dir->d_name))
-    {
-        ret.attribute = ATTR_LINK;
-        strcpy(path, MULTIDRIVE_DIR);
-    }
-    else
-#endif
-        snprintf(path, sizeof(path), "%s/%s", parent->path, dir->d_name);
-
-
-    if (!lstat(path, &s))
-    {
-        int err = 0;
-        if (S_ISLNK(s.st_mode))
-        {
-            ret.attribute |= ATTR_LINK;
-            err = stat(path, &s);
-        }
-        if (!err)
-        {
-            if (S_ISDIR(s.st_mode))
-                ret.attribute |= ATTR_DIRECTORY;
-
-            ret.size = s.st_size;
-            if ((tm = localtime(&(s.st_mtime))))
-            {
-                ret.wrtdate = ((tm->tm_year - 80) << 9) |
-                                  ((tm->tm_mon + 1) << 5) |
-                                    tm->tm_mday;
-                ret.wrttime = (tm->tm_hour << 11) |
-                                  (tm->tm_min << 5) |
-                                  (tm->tm_sec >> 1);
-            }
-        }
-    }
-
-    return ret;
-}
-   
-DIR* app_opendir(const char *_name)
-{
-    size_t name_len;
-    char realpath[MAX_PATH];
-    const char *name = handle_special_dirs(_name, 0, realpath, sizeof(realpath));
-    name_len = strlen(name);
-    char *buf = malloc(sizeof(struct __dir) + name_len+1);
-    if (!buf)
-        return NULL;
-
-    struct __dir *this = (struct __dir*)buf;
-    /* carefully remove any trailing slash from the input, so that
-     * hash/path matching in readdir() works properly */
-    while (name[name_len-1] == '/' && name_len > 1)
-        name_len -= 1;
-    /* strcpy cannot be used because of trailing slashes */
-    memcpy(this->path, name, name_len);
-    this->path[name_len] = 0;
-    this->dir = opendir(this->path);
-
-    if (!this->dir)
-    {
-        free(buf);
-        return NULL;
-    }
-#ifdef HAVE_MULTIDRIVE
-    this->volumes_returned = 0;
-    this->path_hash = crc_32((const void *)this->path, name_len, 0xffffffff);
-#endif
-
-    return (DIR*)this;
-}
-
-int app_closedir(DIR *dir)
-{
-    struct __dir *this = (struct __dir*)dir;
-    int ret = closedir(this->dir);
-    free(this);
-    return ret;
-}
-
-
-struct dirent* app_readdir(DIR* dir)
-{
-    struct __dir *d = (struct __dir*)dir;
-#ifdef HAVE_MULTIDRIVE
-    /* this is not MT-safe but OK according to man readdir */
-    static struct dirent voldir;
-    if (UNLIKELY(rbhome_hash == d->path_hash)
-            && d->volumes_returned < (NUM_VOLUMES-1)
-            && volume_present(d->volumes_returned+1)
-            /* compare path anyway because of possible hash collision */
-            && !strcmp(d->path, rbhome))
-    {
-        d->volumes_returned += 1;
-        sprintf(voldir.d_name, VOL_NAMES, d->volumes_returned);
-        return &voldir;
-    }
-#endif
-    return readdir(d->dir);
-}
-
-
-int app_mkdir(const char* name)
-{
-    char realpath[MAX_PATH];
-    const char *fname = handle_special_dirs(name, NEED_WRITE, realpath, sizeof(realpath));
-    return mkdir(fname, 0777);
-}
-
-
-int app_rmdir(const char* name)
-{
-    char realpath[MAX_PATH];
-    const char *fname = handle_special_dirs(name, NEED_WRITE, realpath, sizeof(realpath));
-    return rmdir(fname);
-}
-
-
-/* On MD we create a virtual symlink for the external drive,
- * for this we need to override readlink(). */
-ssize_t app_readlink(const char *path, char *buf, size_t bufsiz)
-{
-    char _buf[MAX_PATH];
-    (void) path; (void) buf; (void) bufsiz;
-    path = handle_special_dirs(path, 0, _buf, sizeof(_buf));
-#ifdef HAVE_MULTIDRIVE
-    /* if path == _buf then we can be sure handle_special_dir() did something
-     * and path is not an ordinary directory */
-    if (path == _buf && !strncmp(path, MULTIDRIVE_DIR, sizeof(MULTIDRIVE_DIR)-1))
-    {
-        /* copying NUL is not required as per readlink specification */
-        ssize_t len = strlen(path);
-        memcpy(buf, path, len);
-        return len;
-    }
-#endif
-    /* does not append NUL !! */
-    return readlink(path, buf, bufsiz);
-}
diff --git a/firmware/common/unicode.c b/firmware/common/unicode.c
index 3ff1814c4b..954ad47e1d 100644
--- a/firmware/common/unicode.c
+++ b/firmware/common/unicode.c
@@ -28,161 +28,227 @@
 
 #include <stdio.h>
 #include "config.h"
+#include "system.h"
+#include "thread.h"
 #include "file.h"
 #include "debug.h"
 #include "rbunicode.h"
 #include "rbpaths.h"
+#include "pathfuncs.h"
+#include "core_alloc.h"
 
 #ifndef O_BINARY
 #define O_BINARY 0
 #endif
+#ifndef O_NOISODECODE
+#define O_NOISODECODE 0
+#endif
 
-static int default_codepage = 0;
-static int loaded_cp_table = 0;
-
-#ifdef HAVE_LCD_BITMAP
+#define getle16(p) (p[0] | (p[1] >> 8))
+#define getbe16(p) ((p[1] << 8) | p[0])
 
-#define MAX_CP_TABLE_SIZE  32768
-#define NUM_TABLES             5
+#if !defined (__PCTOOL__) && (CONFIG_PLATFORM & PLATFORM_NATIVE)
+/* Because file scanning uses the default CP table when matching entries,
+   on-demand loading is not feasible; we also must use the filesystem lock */
+#include "file_internal.h"
+#else /* APPLICATION */
+#ifdef __PCTOOL__
+#define yield()
+#endif
+#define open_noiso_internal open
+#endif /* !APPLICATION */
+
+#if 0 /* not needed just now (will probably end up a spinlock) */
+#include "mutex.h"
+static struct mutex cp_mutex SHAREDBSS_ATTR;
+#define cp_lock_init()   mutex_init(&cp_mutex)
+#define cp_lock_enter()  mutex_lock(&cp_mutex)
+#define cp_lock_leave()  mutex_unlock(&cp_mutex)
+#else
+#define cp_lock_init()   do {} while (0)
+#define cp_lock_enter()  asm volatile ("")
+#define cp_lock_leave()  asm volatile ("")
+#endif
 
-static const char * const filename[NUM_TABLES] =
+enum cp_tid
 {
-    CODEPAGE_DIR"/iso.cp",
-    CODEPAGE_DIR"/932.cp",  /* SJIS    */
-    CODEPAGE_DIR"/936.cp",  /* GB2312  */
-    CODEPAGE_DIR"/949.cp",  /* KSX1001 */
-    CODEPAGE_DIR"/950.cp"   /* BIG5    */
+    CP_TID_NONE = -1,
+    CP_TID_ISO,
+    CP_TID_932,
+    CP_TID_936,
+    CP_TID_949,
+    CP_TID_950,
 };
 
-static const char cp_2_table[NUM_CODEPAGES] =
+struct cp_info
 {
-    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 0
+    int8_t      tid;
+    const char  *filename;
+    const char  *name;
 };
 
-static const char * const name_codepages[NUM_CODEPAGES+1] =
-{
-    "ISO-8859-1",
-    "ISO-8859-7",
-    "ISO-8859-8",
-    "CP1251",
-    "ISO-8859-11",
-    "CP1256",
-    "ISO-8859-9",
-    "ISO-8859-2",
-    "CP1250",
-    "CP1252",
-    "SJIS",
-    "GB-2312",
-    "KSX-1001",
-    "BIG5",
-    "UTF-8",
-    "unknown"
-};
+#ifdef HAVE_LCD_BITMAP
 
-#if defined(APPLICATION) && defined(__linux__)
-static const char * const name_codepages_linux[NUM_CODEPAGES+1] =
-{
-    /* "ISO-8859-1" */ "iso8859-1",
-    /* "ISO-8859-7" */ "iso8859-7",
-    /* "ISO-8859-8" */ "iso8859-8",
-    /* "CP1251"     */ "cp1251",
-    /* "ISO-8859-11"*/ "iso8859-11",
-    /* "CP1256"     */ "cp1256",
-    /* "ISO-8859-9" */ "iso8859-9",
-    /* "ISO-8859-2" */ "iso8859-2",
-    /* "CP1250"     */ "cp1250",
-    /* "CP1252"     */ "iso8859-15", /* closest, linux doesnt have a codepage named cp1252 */
-    /* "SJIS"       */ "cp932",
-    /* "GB-2312"    */ "cp936",
-    /* "KSX-1001"   */ "cp949",
-    /* "BIG5"       */ "cp950",
-    /* "UTF-8"      */ "utf8",
-    /* "unknown"    */ "cp437"
-};
+#define MAX_CP_TABLE_SIZE  32768
 
-const char *get_current_codepage_name_linux(void)
+#define CPF_ISO "iso.cp"
+#define CPF_932 "932.cp"  /* SJIS    */
+#define CPF_936 "936.cp"  /* GB2312  */
+#define CPF_949 "949.cp"  /* KSX1001 */
+#define CPF_950 "950.cp"  /* BIG5    */
+
+static const struct cp_info cp_info[NUM_CODEPAGES+1] =
 {
-    if (default_codepage < 0 || default_codepage >= NUM_CODEPAGES)
-        return name_codepages_linux[NUM_CODEPAGES];
-    return name_codepages_linux[default_codepage];
-}
-#endif
+    [0 ... NUM_CODEPAGES] = { CP_TID_NONE, NULL   , "unknown"     },
+    [ISO_8859_1]          = { CP_TID_NONE, NULL   , "ISO-8859-1"  },
+    [ISO_8859_7]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-7"  },
+    [ISO_8859_8]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-8"  },
+    [WIN_1251]            = { CP_TID_ISO , CPF_ISO, "CP1251"      },
+    [ISO_8859_11]         = { CP_TID_ISO , CPF_ISO, "ISO-8859-11" },
+    [WIN_1256]            = { CP_TID_ISO , CPF_ISO, "CP1256"      },
+    [ISO_8859_9]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-9"  },
+    [ISO_8859_2]          = { CP_TID_ISO , CPF_ISO, "ISO-8859-2"  },
+    [WIN_1250]            = { CP_TID_ISO , CPF_ISO, "CP1250"      },
+    [WIN_1252]            = { CP_TID_ISO , CPF_ISO, "CP1252"      },
+    [SJIS]                = { CP_TID_932 , CPF_932, "SJIS"        },
+    [GB_2312]             = { CP_TID_936 , CPF_936, "GB-2312"     },
+    [KSX_1001]            = { CP_TID_949 , CPF_949, "KSX-1001"    },
+    [BIG_5]               = { CP_TID_950 , CPF_950, "BIG5"        },
+    [UTF_8]               = { CP_TID_NONE, NULL   , "UTF-8"       },
+};
 
 #else /* !HAVE_LCD_BITMAP, reduced support */
 
 #define MAX_CP_TABLE_SIZE  768
-#define NUM_TABLES           1
 
-static const char * const filename[NUM_TABLES] = {
-    CODEPAGE_DIR"/isomini.cp"
-};
+#define CPF_ISOMINI "isomini.cp"
 
-static const char cp_2_table[NUM_CODEPAGES] =
+static const struct cp_info cp_info[NUM_CODEPAGES+1] =
 {
-    0, 1, 1, 1, 1, 1, 1, 0
+    [0 ... NUM_CODEPAGES] = { CP_TID_NONE, NULL       , "unknown"    },
+    [ISO_8859_1]          = { CP_TID_NONE, NULL       , "ISO-8859-1" },
+    [ISO_8859_7]          = { CP_TID_ISO , CPF_ISOMINI, "ISO-8859-7" },
+    [WIN_1251]            = { CP_TID_ISO , CPF_ISOMINI, "CP1251"     },
+    [ISO_8859_9]          = { CP_TID_ISO , CPF_ISOMINI, "ISO-8859-9" },
+    [ISO_8859_2]          = { CP_TID_ISO , CPF_ISOMINI, "ISO-8859-2" },
+    [WIN_1250]            = { CP_TID_ISO , CPF_ISOMINI, "CP1250"     },
+    [WIN_1252]            = { CP_TID_ISO , CPF_ISOMINI, "CP1252"     },
+    [UTF_8]               = { CP_TID_ISO , NULL       , "UTF-8"      },
 };
 
-static const char * const name_codepages[NUM_CODEPAGES+1] =
+#endif /* HAVE_LCD_BITMAP */
+
+static int default_cp = INIT_CODEPAGE;
+static int default_cp_tid = CP_TID_NONE;
+static int default_cp_handle = 0;
+static int volatile default_cp_table_ref = 0;
+
+static int loaded_cp_tid = CP_TID_NONE;
+static int volatile cp_table_ref = 0;
+#define CP_LOADING BIT_N(sizeof(int)*8-1) /* guard against multi loaders */
+
+/* non-default codepage table buffer (cannot be bufalloced! playback itself
+   may be making the load request) */
+static unsigned short codepage_table[MAX_CP_TABLE_SIZE+1];
+
+#if defined(APPLICATION) && defined(__linux__)
+static const char * const name_codepages_linux[NUM_CODEPAGES+1] =
 {
-    "ISO-8859-1",
-    "ISO-8859-7",
-    "CP1251",
-    "ISO-8859-9",
-    "ISO-8859-2",
-    "CP1250",
-    "CP1252",
-    "UTF-8",
-    "unknown"
+    [0 ... NUM_CODEPAGES] = "unknown",
+    [ISO_8859_1]          = "iso8859-1",
+    [ISO_8859_7]          = "iso8859-7",
+    [ISO_8859_8]          = "iso8859-8",
+    [WIN_1251]            = "cp1251",
+    [ISO_8859_11]         = "iso8859-11",
+    [WIN_1256]            = "cp1256",
+    [ISO_8859_9]          = "iso8859-9",
+    [ISO_8859_2]          = "iso8859-2",
+    [WIN_1250]            = "cp1250",
+    /* iso8859-15 is closest, linux doesnt have a codepage named cp1252 */
+    [WIN_1252]            = "iso8859-15",
+    [SJIS]                = "cp932",
+    [GB_2312]             = "cp936",
+    [KSX_1001]            = "cp949",
+    [BIG_5]               = "cp950",
+    [UTF_8]               = "utf8",
 };
 
-#endif
-
-static unsigned short codepage_table[MAX_CP_TABLE_SIZE];
+const char *get_current_codepage_name_linux(void)
+{
+    int cp = default_cp;
+    if (cp < 0 || cp>= NUM_CODEPAGES)
+        cp = NUM_CODEPAGES;
+    return name_codepages_linux[cp];
+}
+#endif /* defined(APPLICATION) && defined(__linux__) */
 
 static const unsigned char utf8comp[6] =
 {
     0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
 };
 
-/* Load codepage file into memory */
-static int load_cp_table(int cp)
+static inline void cptable_tohw16(uint16_t *buf, unsigned int count)
 {
-    int i = 0;
-    int table = cp_2_table[cp];
-    int file, tablesize;
-    unsigned char tmp[2];
+#ifdef ROCKBOX_BIG_ENDIAN
+    for (unsigned int i = 0; i < count; i++)
+        buf[i] = letoh16(buf[i]);
+#endif
+    (void)buf; (void)count;
+}
 
-    if (table == 0 || table == loaded_cp_table)
-        return 1;
+static int move_callback(int handle, void *current, void *new)
+{
+    /* we don't keep a pointer but we have to stop it if this applies to a
+       buffer not yet swapped-in since it will likely be in use in an I/O
+       call */
+    return (handle != default_cp_handle || default_cp_table_ref != 0) ?
+                BUFLIB_CB_CANNOT_MOVE : BUFLIB_CB_OK;
+    (void)current; (void)new;
+}
 
-    file = open(filename[table-1], O_RDONLY|O_BINARY);
+static int alloc_and_load_cp_table(int cp, void *buf)
+{
+    static struct buflib_callbacks ops =
+        { .move_callback = move_callback };
 
-    if (file < 0) {
-        DEBUGF("Can't open codepage file: %s.cp\n", filename[table-1]);
+    /* alloc and read only if there is an associated file */
+    const char *filename = cp_info[cp].filename;
+    if (!filename)
         return 0;
+
+    char path[MAX_PATH];
+    if (path_append(path, CODEPAGE_DIR, filename, sizeof (path))
+        >= sizeof (path)) {
+        return -1;
     }
 
-    tablesize = filesize(file) / 2;
+    /* must be opened without a chance of reentering from FS code */
+    int fd = open_noiso_internal(path, O_RDONLY);
+    if (fd < 0)
+        return -1;
 
-    if (tablesize > MAX_CP_TABLE_SIZE) {
-        DEBUGF("Invalid codepage file: %s.cp\n", filename[table-1]);
-        close(file);
-        return 0;
-    }
+    off_t size = filesize(fd);
 
-    while (i < tablesize) {
-        if (!read(file, tmp, 2)) {
-            DEBUGF("Can't read from codepage file: %s.cp\n", 
-                    filename[table-1]);
-            loaded_cp_table = 0;
-            return 0;
+    if (size > 0 && size <= MAX_CP_TABLE_SIZE*2 &&
+        !(size % (off_t)sizeof (uint16_t))) {
+
+        /* if the buffer is provided, use that but don't alloc */
+        int handle = buf ? 0 : core_alloc_ex(filename, size, &ops);
+        if (handle > 0)
+            buf = core_get_data(handle);
+
+        if (buf && read(fd, buf, size) == size) {
+            close(fd);
+            cptable_tohw16(buf, size / sizeof (uint16_t));
+            return handle;
         }
-        codepage_table[i++] = (tmp[1] << 8) | tmp[0];
+
+        if (handle > 0)
+            core_free(handle);
     }
 
-    loaded_cp_table = table;
-    close(file);
-    return 1;
+    close(fd);
+    return -1;
 }
 
 /* Encode a UCS value as UTF-8 and return a pointer after this UTF-8 char. */
@@ -205,47 +271,96 @@ unsigned char* utf8encode(unsigned long ucs, unsigned char *utf8)
 unsigned char* iso_decode(const unsigned char *iso, unsigned char *utf8,
                           int cp, int count)
 {
-    unsigned short ucs, tmp;
+    uint16_t *table = NULL;
+
+    cp_lock_enter();
+
+    if (cp < 0 || cp >= NUM_CODEPAGES)
+        cp = default_cp;
 
-    if (cp == -1) /* use default codepage */
-        cp = default_codepage;
+    int tid = cp_info[cp].tid;
 
-    if (!load_cp_table(cp)) cp = 0;
+    while (1) {
+        if (tid == default_cp_tid) {
+            /* use default table */
+            if (default_cp_handle > 0) {
+                table = core_get_data(default_cp_handle);
+                default_cp_table_ref++;
+            }
+
+            break;
+        }
+
+        bool load = false;
+
+        if (tid == loaded_cp_tid) {
+            /* use loaded table */
+            if (!(cp_table_ref & CP_LOADING)) {
+                if (tid != CP_TID_NONE) {
+                    table = codepage_table;
+                    cp_table_ref++;
+                }
+
+                break;
+            }
+        } else if (cp_table_ref == 0) {
+            load = true;
+            cp_table_ref |= CP_LOADING;
+        }
+
+        /* alloc and load must be done outside the lock */
+        cp_lock_leave();
+
+        if (!load) {
+            yield();
+        } else if (alloc_and_load_cp_table(cp, codepage_table) < 0) {
+            cp = INIT_CODEPAGE; /* table may be clobbered now */
+            tid = cp_info[cp].tid;
+        }
+
+        cp_lock_enter();
+
+        if (load) {
+            loaded_cp_tid = tid;
+            cp_table_ref &= ~CP_LOADING;
+        }
+    }
+
+    cp_lock_leave();
 
     while (count--) {
+        unsigned short ucs, tmp;
+
         if (*iso < 128 || cp == UTF_8) /* Already UTF-8 */
             *utf8++ = *iso++;
 
         else {
-
-            /* cp tells us which codepage to convert from */
-            switch (cp) {
-                case ISO_8859_7:  /* Greek */
-                case WIN_1252:    /* Western European */
-                case WIN_1251:    /* Cyrillic */
-                case ISO_8859_9:  /* Turkish */
-                case ISO_8859_2:  /* Latin Extended */
-                case WIN_1250:    /* Central European */
-#ifdef HAVE_LCD_BITMAP
-                case ISO_8859_8:  /* Hebrew */
-                case ISO_8859_11: /* Thai */
-                case WIN_1256:    /* Arabic */
-#endif
+            /* tid tells us which table to use and how */
+            switch (tid) {
+                case CP_TID_ISO: /* Greek */
+                                 /* Hebrew */
+                                 /* Cyrillic */
+                                 /* Thai */
+                                 /* Arabic */
+                                 /* Turkish */
+                                 /* Latin Extended */
+                                 /* Central European */
+                                 /* Western European */
                     tmp = ((cp-1)*128) + (*iso++ - 128);
-                    ucs = codepage_table[tmp];
+                    ucs = table[tmp];
                     break;
 
 #ifdef HAVE_LCD_BITMAP
-                case SJIS: /* Japanese */
+                case CP_TID_932: /* Japanese */
                     if (*iso > 0xA0 && *iso < 0xE0) {
                         tmp = *iso++ | (0xA100 - 0x8000);
-                        ucs = codepage_table[tmp];
+                        ucs = table[tmp];
                         break;
                     }
 
-                case GB_2312:  /* Simplified Chinese */
-                case KSX_1001: /* Korean */
-                case BIG_5:    /* Traditional Chinese */
+                case CP_TID_936: /* Simplified Chinese */
+                case CP_TID_949: /* Korean */
+                case CP_TID_950: /* Traditional Chinese */
                     if (count < 1 || !iso[1]) {
                         ucs = *iso++;
                         break;
@@ -256,7 +371,7 @@ unsigned char* iso_decode(const unsigned char *iso, unsigned char *utf8,
                     tmp = *iso++ << 8;
                     tmp |= *iso++;
                     tmp -= 0x8000;
-                    ucs = codepage_table[tmp];
+                    ucs = table[tmp];
                     count--;
                     break;
 #endif /* HAVE_LCD_BITMAP */
@@ -271,6 +386,17 @@ unsigned char* iso_decode(const unsigned char *iso, unsigned char *utf8,
             utf8 = utf8encode(ucs, utf8);
         }
     }
+
+    if (table) {
+        cp_lock_enter();
+        if (table == codepage_table) {
+            cp_table_ref--;
+        } else {
+            default_cp_table_ref--;
+        }
+        cp_lock_leave();
+    }
+
     return utf8;
 }
 
@@ -288,7 +414,7 @@ unsigned char* utf16LEdecode(const unsigned char *utf16, unsigned char *utf8,
             utf16 += 4;
             count -= 2;
         } else {
-            ucs = (utf16[0] | (utf16[1] << 8));
+            ucs = getle16(utf16);
             utf16 += 2;
             count -= 1;
         }
@@ -310,7 +436,7 @@ unsigned char* utf16BEdecode(const unsigned char *utf16, unsigned char *utf8,
             utf16 += 4;
             count -= 2;
         } else {
-            ucs = (utf16[0] << 8) | utf16[1];
+            ucs = getbe16(utf16);
             utf16 += 2;
             count -= 1;
         }
@@ -400,8 +526,50 @@ const unsigned char* utf8decode(const unsigned char *utf8, unsigned short *ucs)
 
 void set_codepage(int cp)
 {
-    default_codepage = cp;
-    return;
+    if (cp < 0 || cp >= NUM_CODEPAGES)
+        cp = NUM_CODEPAGES;
+
+    /* load first then swap if load is successful, else just leave it; if
+       handle is 0 then we just free the current one; this won't happen often
+       thus we don't worry about reusing it and consequently avoid possible
+       clobbering of the existing one */
+
+    int handle = -1;
+    int tid = cp_info[cp].tid;
+
+    while (1) {
+        cp_lock_enter();
+
+        if (default_cp_tid == tid)
+            break;
+
+        if (handle >= 0 && default_cp_table_ref == 0) {
+            int hold = default_cp_handle;
+            default_cp_handle = handle;
+            handle = hold;
+            default_cp_tid = tid;
+            break;
+        }
+
+        /* alloc and load must be done outside the lock */
+        cp_lock_leave();
+
+        if (handle < 0 && (handle = alloc_and_load_cp_table(cp, NULL)) < 0)
+            return; /* OOM; change nothing */
+
+        yield();
+    }
+
+    default_cp = cp;
+    cp_lock_leave();
+
+    if (handle > 0)
+        core_free(handle);
+}
+
+int get_codepage(void)
+{
+    return default_cp;
 }
 
 /* seek to a given char in a utf8 string and
@@ -418,9 +586,16 @@ int utf8seek(const unsigned char* utf8, int offset)
     return pos;
 }
 
-const char* get_codepage_name(int cp)
+const char * get_codepage_name(int cp)
 {
-    if (cp < 0 || cp>= NUM_CODEPAGES)
-        return name_codepages[NUM_CODEPAGES];
-    return name_codepages[cp];
+    if (cp < 0 || cp >= NUM_CODEPAGES)
+        cp = NUM_CODEPAGES;
+    return cp_info[cp].name;
 }
+
+#if 0 /* not needed just now */
+void unicode_init(void)
+{
+    cp_lock_init();
+}
+#endif