1 files changed, 2763 insertions, 1218 deletions

diff --git a/firmware/common/dircache.c b/firmware/common/dircache.c
index b53fc4d7a6..1e580bf3af 100644
--- a/firmware/common/dircache.c
+++ b/firmware/common/dircache.c
@@ -8,6 +8,7 @@
  * $Id$
  *
  * Copyright (C) 2005 by Miika Pekkarinen
+ * Copyright (C) 2014 by Michael Sevakis
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -18,13 +19,7 @@
  * KIND, either express or implied.
  *
  ****************************************************************************/
-
-/* TODO:
-   - Allow cache live updating while transparent rebuild is running.
-*/
-
 #include "config.h"
-
 #include <stdio.h>
 #include <errno.h>
 #include "string-extra.h"
@@ -33,6 +28,8 @@
 #include "debug.h"
 #include "system.h"
 #include "logf.h"
+#include "fileobj_mgr.h"
+#include "pathfuncs.h"
 #include "dircache.h"
 #include "thread.h"
 #include "kernel.h"
@@ -42,393 +39,1626 @@
 #include "dir.h"
 #include "storage.h"
 #include "audio.h"
-#if CONFIG_RTC
-#include "time.h"
-#include "timefuncs.h"
-#endif
 #include "rbpaths.h"
+#include "linked_list.h"
+#ifdef HAVE_EEPROM_SETTINGS
+#include "crc32.h"
+#endif
 
+/**
+ * Cache memory layout:
+ * x - array of struct dircache_entry
+ * r - reserved buffer
+ * d - name buffer for the name entry of the struct dircache_entry
+ * 0 - zero bytes to assist free name block sentinel scanning (not 0xfe or 0xff)
+ * |xxxxxx|rrrrrrrrr|0|dddddd|0|
+ *
+ * Subsequent x are allocated from the front, d are allocated from the back,
+ * using the reserve buffer for entries added after initial scan.
+ *
+ * After a while the cache may look like:
+ * |xxxxxxxx|rrrrr|0|dddddddd|0|
+ *
+ * After a reboot, the reserve buffer is restored in it's size, so that the
+ * total allocation size grows:
+ * |xxxxxxxx|rrrrrrrrr|0|dddddddd|0|
+ *
+ *
+ * Cache structure:
+ * Format is memory position independent and uses only indexes as links. The
+ * buffer pointers are offset back by one entry to make the array 1-based so
+ * that an index of 0 may be considered an analog of a NULL pointer.
+ *
+ * Entry elements are linked together analagously to the filesystem directory
+ * structure with minor variations that are helpful to the cache's algorithms.
+ * Each volume has a special root structure in the dircache structure, not an
+ * entry in the cache, comprising a forest of volume trees which facilitates
+ * mounting or dismounting of specified volumes on the fly.
+ *
+ * Indexes identifying a volume are computed as: index = -volume - 1
+ * Returning the volume from these indexes is thus: volume = -index - 1
+ * Such indexes are used in root binding and as the 'up' index for an entry
+ * who's parent is the root directory.
+ *
+ * Open files list:
+ * When dircache is made it is the maintainer of the main volume open files
+ * lists, even when it is off. Any files open before dircache is enabled or
+ * initialized must be bound to cache entries by the scan and build operation.
+ * It maintains these lists in a special way.
+ *
+ * Queued (unresolved) bindings are at the back and resolved at the front.
+ * A pointer to the first of each kind of binding is provided to skip to help
+ * iterating one sublist or another.
+ *
+ * r0->r1->r2->q0->q1->q2->NULL
+ * ^resolved0  ^queued0
+ */
 
-/* Queue commands. */
-#define DIRCACHE_BUILD 1
-#define DIRCACHE_STOP  2
+#ifdef DIRCACHE_NATIVE
+#define dircache_lock()   file_internal_lock_WRITER()
+#define dircache_unlock() file_internal_unlock_WRITER()
 
-#if (MEMORYSIZE > 8)
-#define MAX_OPEN_DIRS 12
-#else
-#define MAX_OPEN_DIRS 8
-#endif
-static DIR_CACHED opendirs[MAX_OPEN_DIRS];
-static char       opendir_dnames[MAX_OPEN_DIRS][MAX_PATH];
+/* scan and build parameter data */
+struct sab_component;
+struct sab
+{
+    struct filestr_base   stream;    /* scan directory stream */
+    struct file_base_info info;      /* scanned entry info */
+    bool volatile         quit;      /* halt all scanning */
+    struct sab_component  *stackend; /* end of stack pointer */
+    struct sab_component  *top;      /* current top of stack */
+    struct sab_component
+    {
+        int volatile      idx;       /* cache index of directory */
+        int               *downp;    /* pointer to ce->down */
+        int *volatile     prevp;     /* previous item accessed */
+    } stack[];                       /* "recursion" stack */
+};
+
+#else /* !DIRCACHE_NATIVE */
+
+#error need locking scheme
+#define FILESYS_WRITER_LOCK()
+#define FILESYS_WRITER_UNLOCK()
 
-#define MAX_PENDING_BINDINGS 2
-struct fdbind_queue {
+struct sab_component
+{
+};
+
+struct sab
+{
+#ifdef HAVE_MUTLIVOLUME
+    int volume;
+#endif /* HAVE_MULTIVOLUME */
     char path[MAX_PATH];
-    int fd;
+    unsigned int append;
+};
+#endif /* DIRCACHE_NATIVE */
+
+enum
+{
+    FRONTIER_SETTLED = 0x0, /* dir entry contents are complete */
+    FRONTIER_NEW     = 0x1, /* dir entry contents are in-progress */
+    FRONTIER_ZONED   = 0x2, /* frontier entry permanent mark (very sticky!) */
+    FRONTIER_RENEW   = 0x4, /* override FRONTIER_ZONED sticky (not stored) */
 };
 
-/* Unions with char to make pointer arithmetic simpler and avoid casting */
-struct dircache_entry {
-    struct dirinfo info;
+enum
+{
+    DCM_BUILD,              /* build a volume */
+    DCM_PROCEED,            /* merged DCM_BUILD messages */
+    DCM_FIRST = DCM_BUILD,
+    DCM_LAST  = DCM_PROCEED,
+};
+
+#define MAX_TINYNAME   sizeof (uint32_t)
+#define DC_MAX_NAME    MIN(MAX_NAME, UINT8_MAX)
+
+/* Throw some warnings if about the limits if things may not work */
+#if MAX_NAME > UINT8_MAX
+#warning Need more than 8 bits in name length bitfield
+#endif
+
+#if DIRCACHE_LIMIT > ((1 << 24)-255)
+#warning Names may not be addressable with 24 bits
+#endif
+
+/* data structure used by cache entries */
+struct dircache_entry
+{
+    int         next;              /* next at same level */
     union {
-        struct dircache_entry *next;
-        char* next_char;
+    int         down;              /* first at child level (if directory) */
+    file_size_t filesize;          /* size of file in bytes (if file) */
     };
+    int         up;                /* parent index (-volume-1 if root) */
     union {
-        struct dircache_entry *up;
-        char* up_char;
+    struct {
+    uint32_t    name         : 24; /* indirect storage (.tinyname == 0) */
+    uint32_t    length       :  8; /* length of name indexed by 'name' */
     };
-    union {
-        struct dircache_entry *down;
-        char* down_char;
+    unsigned char namebuf[MAX_TINYNAME]; /* direct storage (.tinyname == 1) */
     };
-    long startcluster;
-    char *d_name;
-};
-
-/* Cache Layout:
- *
- * x - array of struct dircache_entry
- * r - reserved buffer
- * d - name buffer for the d_name entry of the struct dircache_entry
- * |xxxxxx|rrrrrrrrr|dddddd|
- *
- * subsequent x are allocated from the front, d are allocated from the back,
- * using the reserve buffer for entries added after initial scan
- *
- * after a while the cache may look like:
- * |xxxxxxxx|rrrrr|dddddddd|
- *
- * after a reboot, the reserve buffer is restored in it's size, so that the
- * total allocation size grows
- * |xxxxxxxx|rrrrrrrrr|dddddddd|
- */
-/* this points to the beginnging of the buffer and the first entry */
-static struct dircache_entry *dircache_root;
-/* these point to the start and end of the name buffer (d above) */
-static char                  *d_names_start, *d_names_end;
-/* put "." and ".." into the d_names buffer to enable easy pointer logic */
-static char                  *dot, *dotdot;
-#ifdef HAVE_MULTIVOLUME
-static struct dircache_entry *append_position;
+    uint32_t    direntry     : 16; /* entry # in parent - max 0xffff */
+    uint32_t    direntries   :  5; /* # of entries used - max 21 */
+    uint32_t    tinyname     :  1; /* if == 1, name fits in .namebuf */
+    uint32_t    frontier     :  2; /* (FRONTIER_* bitflags) */
+    uint32_t    attr         :  8; /* entry file attributes */
+#ifdef DIRCACHE_NATIVE
+    long        firstcluster;      /* first file cluster - max 0x0ffffff4 */
+    uint16_t    wrtdate;           /* FAT write date */
+    uint16_t    wrttime;           /* FAT write time */
+#else
+    time_t      mtime;             /* file last-modified time */
 #endif
+    dc_serial_t serialnum;         /* entry serial number */
+};
 
-static DIR_CACHED opendirs[MAX_OPEN_DIRS];
-static struct dircache_entry *fd_bindings[MAX_OPEN_FILES];
-
-static bool dircache_initialized = false;
-static bool dircache_initializing = false;
-static bool thread_enabled = false;
-static unsigned long allocated_size = 0;
-static unsigned long dircache_size = 0;
-static unsigned long entry_count = 0;
-static unsigned long reserve_used = 0;
-static unsigned int  cache_build_ticks = 0;
-static unsigned long appflags = 0;
+/* spare us some tedium */
+#define ENTRYSIZE   (sizeof (struct dircache_entry))
 
+/* thread and kernel stuff */
 static struct event_queue dircache_queue SHAREDBSS_ATTR;
-static long dircache_stack[(DEFAULT_STACK_SIZE + 0x400)/sizeof(long)];
+static uintptr_t dircache_stack[DIRCACHE_STACK_SIZE / sizeof (uintptr_t)];
 static const char dircache_thread_name[] = "dircache";
 
-static struct fdbind_queue fdbind_cache[MAX_PENDING_BINDINGS];
-static int fdbind_idx = 0;
+/* struct that is both used during run time and for persistent storage */
+static struct dircache
+{
+    /* cache-wide data */
+    int          free_list;           /* first index of free entry list */
+    size_t       size;                /* total size of data (including holes) */
+    size_t       sizeused;            /* bytes of .size bytes actually used */
+    union {
+    unsigned int numentries;          /* entry count (including holes) */
+#ifdef HAVE_EEPROM_SETTINGS
+    size_t       sizeentries;         /* used when persisting */
+#endif
+    };
+    int          names;               /* index of first name in name block */
+    size_t       sizenames;           /* size of all names (including holes) */
+    size_t       namesfree;           /* amount of wasted name space */
+    int          nextnamefree;        /* hint of next free name in buffer */
+    /* per-volume data */
+    struct dircache_volume            /* per volume cache data */
+    {
+        uint32_t       status   : 2;  /* cache status of this volume */
+        uint32_t       frontier : 2;  /* (FRONTIER_* bitflags) */
+        dc_serial_t    serialnum;     /* dircache serial number of root */
+        int            root_down;     /* index of first entry of volume root */
+        union {
+        long           start_tick;    /* when did scan start (scanning) */
+        long           build_ticks;   /* how long to build volume? (ready) */
+        };
+    } dcvol[NUM_VOLUMES];
+    /* these remain unchanged between cache resets */
+    size_t       last_size;           /* last reported size at boot */
+    size_t       reserve_used;        /* reserved used at last build */
+    dc_serial_t  last_serialnum;      /* last serialnumber generated */
+} dircache;
+
+/* struct that is used only for the cache in main memory */
+struct dircache_runinfo
+{
+    /* cache setting and build info */
+    int          suspended;        /* dircache suspend count */
+    bool         enabled;          /* dircache master enable switch */
+    unsigned int thread_id;        /* current/last thread id */
+    bool         thread_done;      /* thread has exited */
+    /* cache buffer info */
+    int          handle;           /* buflib buffer handle */
+    size_t       bufsize;          /* size of buflib allocation - 1 */
+    int          buflocked;        /* don't move due to other allocs */
+    union {
+    void                  *p;      /* address of buffer - ENTRYSIZE */
+    struct dircache_entry *pentry; /* alias of .p to assist entry resolution */
+    unsigned char         *pname;  /* alias of .p to assist name resolution */
+    };
+    struct buflib_callbacks ops;   /* buflib ops callbacks */
+    /* per-volume data */
+    struct dircache_runinfo_volume
+    {
+        struct file_base_binding *resolved0; /* first resolved binding in list */
+        struct file_base_binding *queued0;   /* first queued binding in list */
+        struct sab               *sabp;      /* if building, struct sab in use */
+    } dcrivol[NUM_VOLUMES];
+} dircache_runinfo;
+
+#define BINDING_NEXT(bindp) \
+    ((struct file_base_binding *)(bindp)->node.next)
+
+#define FOR_EACH_BINDING(start, p) \
+    for (struct file_base_binding *p = (start); p; p = BINDING_NEXT(p))
+
+#define FOR_EACH_CACHE_ENTRY(ce) \
+    for (struct dircache_entry *ce = &dircache_runinfo.pentry[1],  \
+                               *_ceend = ce + dircache.numentries; \
+         ce < _ceend; ce++) if (ce->serialnum)
+
+#define FOR_EACH_SAB_COMP(sabp, p) \
+    for (struct sab_component *p = (sabp)->top; p < (sabp)->stackend; p++)
+
+/* "overloaded" macros to get volume structures */
+#define DCVOL_i(i)               (&dircache.dcvol[i])
+#define DCVOL_volume(volume)     (&dircache.dcvol[volume])
+#define DCVOL_infop(infop)       (&dircache.dcvol[BASEINFO_VOL(infop)])
+#define DCVOL_dirinfop(dirinfop) (&dircache.dcvol[BASEINFO_VOL(dirinfop)])
+#define DCVOL(x)                 DCVOL_##x(x)
+
+#define DCRIVOL_i(i)             (&dircache_runinfo.dcrivol[i])
+#define DCRIVOL_infop(infop)     (&dircache_runinfo.dcrivol[BASEINFO_VOL(infop)])
+#define DCRIVOL_bindp(bindp)     (&dircache_runinfo.dcrivol[BASEBINDING_VOL(bindp)])
+#define DCRIVOL(x)               DCRIVOL_##x(x)
+
+/* reserve over 75% full? */
+#define DIRCACHE_STUFFED(reserve_used) \
+    ((reserve_used) > 3*DIRCACHE_RESERVE / 4)
 
-/* --- Internal cache structure control functions --- */
+#ifdef HAVE_EEPROM_SETTINGS
+/**
+ * remove the snapshot file
+ */
+static int remove_dircache_file(void)
+{
+    return remove(DIRCACHE_FILE);
+}
+
+/**
+ * open or create the snapshot file
+ */
+static int open_dircache_file(int oflag)
+{
+    return open(DIRCACHE_FILE, oflag, 0666);
+}
+#endif /* HAVE_EEPROM_SETTINGS */
 
-static inline struct dircache_entry* get_entry(int id)
+#ifdef DIRCACHE_DUMPSTER
+/**
+ * clean up the memory allocation to make viewing in a hex editor more friendly
+ * and highlight problems
+ */
+static inline void dumpster_clean_buffer(void *p, size_t size)
 {
-    return &dircache_root[id];
+    memset(p, 0xAA, size);
 }
+#endif /* DIRCACHE_DUMPSTER */
 
-/* flag to make sure buffer doesn't move due to other allocs.
- * this is set to true completely during dircache build */
-static bool dont_move = false;
-static int dircache_handle;
-static int move_callback(int handle, void* current, void* new)
+/**
+ * relocate the cache when the buffer has moved
+ */
+static int move_callback(int handle, void *current, void *new)
 {
-    (void)handle;
-    if (dont_move)
+    if (dircache_runinfo.buflocked)
         return BUFLIB_CB_CANNOT_MOVE;
 
-#define UPDATE(x) if (x) { x = PTR_ADD(x, diff); }
-    /* relocate the cache */
-    ptrdiff_t diff = new - current;
-    for(unsigned i = 0; i < entry_count; i++)
+    dircache_runinfo.p = new - ENTRYSIZE;
+
+    return BUFLIB_CB_OK;
+    (void)handle; (void)current;
+}
+
+/**
+ * add a "don't move" lock count
+ */
+static inline void buffer_lock(void)
+{
+    dircache_runinfo.buflocked++;
+}
+
+/**
+ * remove a "don't move" lock count
+ */
+static inline void buffer_unlock(void)
+{
+    dircache_runinfo.buflocked--;
+}
+
+
+/** Open file bindings management **/
+
+/* compare the basic file information and return 'true' if they are logically
+   equivalent or the same item, else return 'false' if not */
+static inline bool binding_compare(const struct file_base_info *infop1,
+                                   const struct file_base_info *infop2)
+{
+#ifdef DIRCACHE_NATIVE
+    return fat_file_is_same(&infop1->fatfile, &infop2->fatfile);
+#else
+    #error hey watch it!
+#endif
+}
+
+/**
+ * bind a file to the cache; "queued" or "resolved" depending upon whether or
+ * not it has entry information
+ */
+static void binding_open(struct file_base_binding *bindp)
+{
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(bindp);
+    if (bindp->info.dcfile.serialnum)
     {
-        UPDATE(dircache_root[i].d_name);
-        UPDATE(dircache_root[i].next_char);
-        UPDATE(dircache_root[i].up_char);
-        UPDATE(dircache_root[i].down_char);
+        /* already resolved */
+        dcrivolp->resolved0 = bindp;
+        file_binding_insert_first(bindp);
     }
-    dircache_root = new;
-    UPDATE(d_names_start);
-    UPDATE(d_names_end);
-    UPDATE(dot);
-    UPDATE(dotdot);
+    else
+    {
+        if (dcrivolp->queued0 == NULL)
+            dcrivolp->queued0 = bindp;
 
-    for(unsigned i = 0; i < MAX_OPEN_FILES; i++)
-        UPDATE(fd_bindings[i]);
+        file_binding_insert_last(bindp);
+    }
+}
 
-#ifdef HAVE_MULTIVOLUME
-    UPDATE(append_position);
-#endif
-    return BUFLIB_CB_OK;
+/**
+ * remove a binding from the cache
+ */
+static void binding_close(struct file_base_binding *bindp)
+{
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(bindp);
+
+    if (bindp == dcrivolp->queued0)
+        dcrivolp->queued0 = BINDING_NEXT(bindp);
+    else if (bindp == dcrivolp->resolved0)
+    {
+        struct file_base_binding *nextp = BINDING_NEXT(bindp);
+        dcrivolp->resolved0 = (nextp == dcrivolp->queued0) ? NULL : nextp;
+    }
+
+    file_binding_remove(bindp);
+    /* no need to reset it */
 }
 
-static struct buflib_callbacks ops = {
-    .move_callback = move_callback,
-    .shrink_callback = NULL,
-};
+/**
+ * resolve a queued binding with the information from the given source file
+ */
+static void binding_resolve(const struct file_base_info *infop)
+{
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(infop);
+
+    /* quickly check the queued list to see if it's there */
+    struct file_base_binding *prevp = NULL;
+    FOR_EACH_BINDING(dcrivolp->queued0, p)
+    {
+        if (!binding_compare(infop, &p->info))
+        {
+            prevp = p;
+            continue;
+        }
+
+        if (p == dcrivolp->queued0)
+            dcrivolp->queued0 = BINDING_NEXT(p);
+        else
+        {
+            file_binding_remove_next(prevp, p);
+            file_binding_insert_first(p);
+        }
+
+        dcrivolp->resolved0 = p;
+
+        /* srcinfop may be the actual one */
+        if (&p->info != infop)
+            p->info.dcfile = infop->dcfile;
+
+        break;
+    }
+}
 
-#ifdef HAVE_EEPROM_SETTINGS
 /**
- * Open the dircache file to save a snapshot on disk
+ * dissolve a resolved binding on its volume
  */
-static int open_dircache_file(unsigned flags, int permissions)
+static void binding_dissolve(struct file_base_binding *prevp,
+                             struct file_base_binding *bindp)
 {
-    if (permissions != 0)
-        return open(DIRCACHE_FILE, flags, permissions);
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(bindp);
 
-    return open(DIRCACHE_FILE, flags);
+    if (bindp == dcrivolp->resolved0)
+    {
+        struct file_base_binding *nextp = BINDING_NEXT(bindp);
+        dcrivolp->resolved0 = (nextp == dcrivolp->queued0) ? NULL : nextp;
+    }
+
+    if (dcrivolp->queued0 == NULL)
+        dcrivolp->queued0 = bindp;
+
+    file_binding_remove_next(prevp, bindp);
+    file_binding_insert_last(bindp);
+
+    dircache_dcfile_init(&bindp->info.dcfile);
 }
 
 /**
- * Remove the snapshot file
+ * dissolve all resolved bindings on a given volume
  */
-static int remove_dircache_file(void)
+static void binding_dissolve_volume(struct dircache_runinfo_volume *dcrivolp)
 {
-    return remove(DIRCACHE_FILE);
+    if (!dcrivolp->resolved0)
+        return;
+
+    FOR_EACH_BINDING(dcrivolp->resolved0, p)
+    {
+        if (p == dcrivolp->queued0)
+            break;
+
+        dircache_dcfile_init(&p->info.dcfile);
+    }
+
+    dcrivolp->queued0 = dcrivolp->resolved0;
+    dcrivolp->resolved0 = NULL;
 }
-#endif
-/** 
- * Internal function to allocate a new dircache_entry from memory.
+
+
+/** Dircache buffer management **/
+
+/**
+ * allocate the cache's memory block
  */
-static struct dircache_entry* allocate_entry(void)
+static int alloc_cache(size_t size)
 {
-    struct dircache_entry *next_entry;
-    
-    if (dircache_size > allocated_size - MAX_PATH*2)
+    /* pad with one extra-- see alloc_name() and free_name() */
+    return core_alloc_ex("dircache", size + 1, &dircache_runinfo.ops);
+}
+
+/**
+ * put the allocation in dircache control
+ */
+static void set_buffer(int handle, size_t size)
+{
+    void *p = core_get_data(handle);
+
+#ifdef DIRCACHE_DUMPSTER
+    dumpster_clean_buffer(p, size);
+#endif /* DIRCACHE_DUMPSTER */
+
+    /* set it up as a 1-based array */
+    dircache_runinfo.p = p - ENTRYSIZE;
+
+    if (dircache_runinfo.handle != handle)
     {
-        logf("size limit reached");
-        return NULL;
+        /* new buffer */
+        dircache_runinfo.handle = handle;
+        dircache_runinfo.bufsize = size;
+        dircache.names = size + ENTRYSIZE;
+        dircache_runinfo.pname[dircache.names - 1] = 0;
+        dircache_runinfo.pname[dircache.names    ] = 0;
     }
-    
-    next_entry = &dircache_root[entry_count++];
-    next_entry->d_name = NULL;
-    next_entry->up = NULL;
-    next_entry->down = NULL;
-    next_entry->next = NULL;
+}
 
-    dircache_size += sizeof(struct dircache_entry);
+/**
+ * remove the allocation from dircache control and return the handle
+ */
+static int reset_buffer(void)
+{
+    int handle = dircache_runinfo.handle;
+    if (handle > 0)
+    {
+        /* don't mind .p; it might get changed by the callback even after
+           this call; buffer presence is determined by the following: */
+        dircache_runinfo.handle  = 0;
+        dircache_runinfo.bufsize = 0;
+    }
+
+    return handle;
+}
+
+/**
+ * return the number of bytes remaining in the buffer
+ */
+static size_t dircache_buf_remaining(void)
+{
+    if (!dircache_runinfo.handle)
+        return 0;
+
+    return dircache_runinfo.bufsize - dircache.size;
+}
+
+/**
+ * return the amount of reserve space used
+ */
+static size_t reserve_buf_used(void)
+{
+    size_t remaining = dircache_buf_remaining();
+    return (remaining < DIRCACHE_RESERVE) ?
+                DIRCACHE_RESERVE - remaining : 0;
+}
+
+
+/** Internal cache structure control functions **/
+
+/**
+ * generate the next serial number in the sequence
+ */
+static dc_serial_t next_serialnum(void)
+{
+    dc_serial_t serialnum = MAX(dircache.last_serialnum + 1, 1);
+    dircache.last_serialnum = serialnum;
+    return serialnum;
+}
+
+/**
+ * return the dircache volume pointer for the special index
+ */
+static struct dircache_volume * get_idx_dcvolp(int idx)
+{
+    if (idx >= 0)
+        return NULL;
+
+    return &dircache.dcvol[IF_MV_VOL(-idx - 1)];
+}
+
+/**
+ * return the cache entry referenced by idx (NULL if outside buffer)
+ */
+static struct dircache_entry * get_entry(int idx)
+{
+    if (idx <= 0 || (unsigned int)idx > dircache.numentries)
+        return NULL;
 
-    return next_entry;
+    return &dircache_runinfo.pentry[idx];
 }
 
 /**
- * Internal function to allocate a dircache_entry and set 
- * ->next entry pointers.
+ * return the index of the cache entry (0 if outside buffer)
  */
-static struct dircache_entry* dircache_gen_next(struct dircache_entry *ce)
+static int get_index(const struct dircache_entry *ce)
 {
-    struct dircache_entry *next_entry;
+    if (!PTR_IN_ARRAY(dircache_runinfo.pentry + 1, ce,
+                      dircache.numentries + 1))
+    {
+        return 0;
+    }
+
+    return ce - dircache_runinfo.pentry;
+}
 
-    if ( (next_entry = allocate_entry()) == NULL)
+/**
+ *  return the sublist down pointer for the sublist that contains entry 'idx'
+ */
+static int * get_downidxp(int idx)
+{
+    /* NOTE: 'idx' must refer to a directory or the result is undefined */
+    if (idx == 0 || idx < -NUM_VOLUMES)
         return NULL;
-    next_entry->up = ce->up;
-    ce->next = next_entry;
-    
-    return next_entry;
+
+    if (idx > 0)
+    {
+        /* a normal entry */
+        struct dircache_entry *ce = get_entry(idx);
+        return ce ? &ce->down : NULL;
+    }
+    else
+    {
+        /* a volume root */
+        return &get_idx_dcvolp(idx)->root_down;
+    }
 }
 
-/*
- * Internal function to allocate a dircache_entry and set
- * ->down entry pointers.
+/**
+ * return a pointer to the index referencing the cache entry that 'idx'
+ * references
  */
-static struct dircache_entry* dircache_gen_down(struct dircache_entry *ce)
+static int * get_previdxp(int idx)
 {
-    struct dircache_entry *next_entry;
+    struct dircache_entry *ce = get_entry(idx);
 
-    if ( (next_entry = allocate_entry()) == NULL)
+    int *prevp = get_downidxp(ce->up);
+    if (!prevp)
         return NULL;
-    next_entry->up = ce;
-    ce->down = next_entry;
-    
-    return next_entry;
+
+    while (1)
+    {
+        int next = *prevp;
+        if (!next || next == idx)
+            break;
+
+        prevp = &get_entry(next)->next;
+    }
+
+    return prevp;
+}
+
+/**
+ * if required, adjust the lists and directory read of any scan and build in
+ * progress
+ */
+static void sab_sync_scan(struct sab *sabp, int *prevp, int *nextp)
+{
+    struct sab_component *abovep = NULL;
+    FOR_EACH_SAB_COMP(sabp, p)
+    {
+        if (nextp != p->prevp)
+        {
+            abovep = p;
+            continue;
+        }
+
+        /* removing an item being scanned; set the component position to the
+           entry before this */
+        p->prevp = prevp;
+
+        if (p == sabp->top)
+        {
+            /* removed at item in the directory who's immediate contents are
+               being scanned */
+            if (prevp == p->downp)
+            {
+                /* was first item; rewind it */
+                dircache_rewinddir_internal(&sabp->info);
+            }
+            else
+            {
+                struct dircache_entry *ceprev =
+                    container_of(prevp, struct dircache_entry, next);
+            #ifdef DIRCACHE_NATIVE
+                sabp->info.fatfile.e.entry   = ceprev->direntry;
+                sabp->info.fatfile.e.entries = ceprev->direntries;
+            #endif
+            }
+        }
+        else if (abovep)
+        {
+            /* the directory being scanned or a parent of it has been removed;
+               abort its build or cache traversal */
+            abovep->idx = 0;
+        }
+
+        break;
+    }
+}
+
+/**
+ * get a pointer to an allocated name given a cache index
+ */
+static inline unsigned char * get_name(int nameidx)
+{
+    return &dircache_runinfo.pname[nameidx];
+}
+
+/**
+ * get the cache buffer index of the given name
+ */
+static inline int get_nameidx(const unsigned char *pname)
+{
+    return pname - dircache_runinfo.pname;
+}
+
+/**
+ * copy the entry's name to a buffer (which assumed to be of sufficient size)
+ */
+static void entry_name_copy(char *dst, const struct dircache_entry *ce)
+{
+    if (LIKELY(!ce->tinyname))
+    {
+        strmemcpy(dst, get_name(ce->name), ce->length);
+        return;
+    }
+
+    const unsigned char *src = ce->namebuf;
+    size_t len = 0;
+    while (len++ < MAX_TINYNAME && *src)
+        *dst++ = *src++;
+
+    *dst = '\0';
+}
+
+/**
+ * set the namesfree hint to a new position
+ */
+static void set_namesfree_hint(const unsigned char *hintp)
+{
+    int hintidx = get_nameidx(hintp);
+
+    if (hintidx >= (int)(dircache.names + dircache.sizenames))
+        hintidx = dircache.names;
+
+    dircache.nextnamefree = hintidx;
 }
 
 /**
- * Returns true if there is an event waiting in the queue
- * that requires the current operation to be aborted.
+ * allocate a buffer to use for a new name
  */
-static bool check_event_queue(void)
+static int alloc_name(size_t size)
+{
+    int nameidx = 0;
+
+    if (dircache.namesfree >= size)
+    {
+        /* scan for a free gap starting at the hint point - first fit */
+        unsigned char *start = get_name(dircache.nextnamefree), *p = start;
+        unsigned char *namesend = get_name(dircache.names + dircache.sizenames);
+        size_t gapsize = 0;
+
+        while (gapsize < size)
+        {
+            if ((p = memchr(p, 0xff, namesend - p)))
+            {
+                /* found a sentinel; see if there are enough in a row */
+                gapsize = 1;
+                while (*++p == 0xff && gapsize < size)
+                    gapsize++;
+            }
+            else
+            {
+                if (namesend == start)
+                    break; /* exhausted */
+
+                /* wrap */
+                namesend = start;
+                p = get_name(dircache.names);
+
+                if (p == namesend)
+                    break; /* initial hint was at names start */
+            }
+        }
+
+        if (gapsize >= size)
+        {
+            unsigned char *namep = p - gapsize;
+            nameidx = get_nameidx(namep);
+
+            if (*p == 0xff)
+            {
+                /* if only a tiny block remains after buffer, claim it too */
+                size_t tinysize = 1;
+                while (*++p == 0xff && tinysize <= MAX_TINYNAME)
+                    tinysize++;
+
+                if (tinysize <= MAX_TINYNAME)
+                {
+                    /* mark with tiny block sentinel */
+                    memset(p - tinysize, 0xfe, tinysize);
+                    size += tinysize;
+                }
+            }
+
+            dircache.namesfree -= size;
+            dircache.sizeused += size;
+            set_namesfree_hint(namep + size);
+        }
+    }
+
+    if (!nameidx)
+    {
+        /* no sufficiently long free gaps; allocate anew */
+        if (dircache_buf_remaining() <= size)
+        {
+            dircache.last_size = 0;
+            return 0;
+        }
+
+        dircache.names     -= size;
+        dircache.sizenames += size;
+        nameidx             = dircache.names;
+        dircache.size      += size;
+        dircache.sizeused  += size;
+        *get_name(dircache.names - 1) = 0;
+    }
+
+    return nameidx;
+}
+
+/**
+ * mark a name as free and note that its bytes are available
+ */
+static void free_name(int nameidx, size_t size)
+{
+    unsigned char *beg = get_name(nameidx);
+    unsigned char *end = beg + size;
+
+    /* merge with any adjacent tiny blocks */
+    while (beg[-1] == 0xfe)
+        --beg;
+
+    while (end[1] == 0xfe)
+        ++end;
+
+    size = end - beg;
+    memset(beg, 0xff, size);
+    dircache.namesfree += size;
+    dircache.sizeused -= size;
+    set_namesfree_hint(beg);
+}
+
+/**
+ * allocate and assign a name to the entry
+ */
+static bool entry_assign_name(struct dircache_entry *ce,
+                              const unsigned char *name, size_t size)
+{
+    unsigned char *copyto;
+
+    if (size <= MAX_TINYNAME)
+    {
+        copyto = ce->namebuf;
+
+        if (size < MAX_TINYNAME)
+            copyto[size] = '\0';
+
+        ce->tinyname = 1;
+    }
+    else
+    {
+        if (size > DC_MAX_NAME)
+            size = DC_MAX_NAME;
+
+        int nameidx = alloc_name(size);
+        if (!nameidx)
+            return false;
+
+        copyto = get_name(nameidx);
+
+        ce->tinyname = 0;
+        ce->name     = nameidx;
+        ce->length   = size;
+    }
+
+    memcpy(copyto, name, size);
+    return true;
+}
+
+/**
+ * free the name for the entry
+ */
+static void entry_unassign_name(struct dircache_entry *ce)
+{
+    if (!ce->tinyname)
+        free_name(ce->name, ce->length);
+}
+
+/**
+ * assign a new name to the entry
+ */
+static bool entry_reassign_name(struct dircache_entry *ce,
+                                const unsigned char *newname)
+{
+    size_t oldlen = ce->tinyname ? 0 : ce->length;
+    size_t newlen = strlen(newname);
+
+    if (oldlen == newlen || (oldlen == 0 && newlen <= MAX_TINYNAME))
+    {
+        char *p = mempcpy(oldlen == 0 ? ce->namebuf : get_name(ce->name),
+                          newname, newlen);
+        if (newlen < MAX_TINYNAME)
+            *p = '\0';
+        return true;
+    }
+
+    /* needs a new name allocation; if the new name fits in the freed block,
+       it will use it immediately without a lengthy search */
+    entry_unassign_name(ce);
+    return entry_assign_name(ce, newname, newlen);
+}
+
+/**
+ * allocate a dircache_entry from memory using freed ones if available
+ */
+static int alloc_entry(struct dircache_entry **res)
+{
+    struct dircache_entry *ce;
+    int idx = dircache.free_list;
+
+    if (idx)
+    {
+        /* reuse a freed entry */
+        ce = get_entry(idx);
+        dircache.free_list = ce->next;
+    }
+    else if (dircache_buf_remaining() > ENTRYSIZE)
+    {
+        /* allocate a new one */
+        idx = ++dircache.numentries;
+        dircache.size += ENTRYSIZE;
+        ce = get_entry(idx);
+    }
+    else
+    {
+        dircache.last_size = 0;
+        *res = NULL;
+        return 0;
+    }
+
+    dircache.sizeused += ENTRYSIZE;
+
+    ce->next      = 0;
+    ce->up        = 0;
+    ce->down      = 0;
+    ce->length    = 0;
+    ce->frontier  = FRONTIER_SETTLED;
+    ce->serialnum = next_serialnum();
+
+    *res = ce;
+    return idx;
+}
+
+/**
+ * free an entry's allocations in the cache; must not be linked to anything
+ * by this time (orphan!)
+ */
+static void free_orphan_entry(struct dircache_runinfo_volume *dcrivolp,
+                              struct dircache_entry *ce, int idx)
+{
+    if (dcrivolp)
+    {
+        /* was an established entry; find any associated resolved binding and
+           dissolve it;  bindings are kept strictly synchronized with changes
+           to the storage so a simple serial number comparison is sufficient */
+        struct file_base_binding *prevp = NULL;
+        FOR_EACH_BINDING(dcrivolp->resolved0, p)
+        {
+            if (p == dcrivolp->queued0)
+                break;
+
+            if (ce->serialnum == p->info.dcfile.serialnum)
+            {
+                binding_dissolve(prevp, p);
+                break;
+            }
+
+            prevp = p;
+        }
+    }
+
+    entry_unassign_name(ce);
+
+    /* no serialnum says "it's free" (for cache-wide iterators) */
+    ce->serialnum = 0;
+
+    /* add to free list */
+    ce->next = dircache.free_list;
+    dircache.free_list = idx;
+    dircache.sizeused -= ENTRYSIZE;
+}
+
+/**
+ * allocates a new entry of with the name specified by 'basename'
+ */
+static int create_entry(const char *basename,
+                        struct dircache_entry **res)
+{
+    int idx = alloc_entry(res);
+    if (!idx)
+    {
+        logf("size limit reached (entry)");
+        return 0; /* full */
+    }
+
+    if (!entry_assign_name(*res, basename, strlen(basename)))
+    {
+        free_orphan_entry(NULL, *res, idx);
+        logf("size limit reached (name)");
+        return 0; /* really full! */
+    }
+
+    return idx;
+}
+
+/**
+ * unlink the entry at *prevp and adjust the scanner if needed
+ */
+static void remove_entry(struct dircache_runinfo_volume *dcrivolp,
+                         struct dircache_entry *ce, int *prevp)
+{
+    /* unlink it from its list */
+    *prevp = ce->next;
+
+    if (dcrivolp)
+    {
+        /* adjust scanner iterator if needed */
+        struct sab *sabp = dcrivolp->sabp;
+        if (sabp)
+            sab_sync_scan(sabp, prevp, &ce->next);
+    }
+}
+
+/**
+ * free the entire subtree in the referenced parent down index
+ */
+static void free_subentries(struct dircache_runinfo_volume *dcrivolp, int *downp)
+{
+    while (1)
+    {
+        int idx = *downp;
+        struct dircache_entry *ce = get_entry(idx);
+        if (!ce)
+            break;
+
+        if ((ce->attr & ATTR_DIRECTORY) && ce->down)
+            free_subentries(dcrivolp, &ce->down);
+
+        remove_entry(dcrivolp, ce, downp);
+        free_orphan_entry(dcrivolp, ce, idx);
+    }
+}
+
+/**
+ * free the specified file entry and its children
+ */
+static void free_file_entry(struct file_base_info *infop)
+{
+    int idx = infop->dcfile.idx;
+    if (idx <= 0)
+        return; /* can't remove a root/invalid */
+
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(infop);
+
+    struct dircache_entry *ce = get_entry(idx);
+    if ((ce->attr & ATTR_DIRECTORY) && ce->down)
+    {
+        /* gonna get all this contents (normally the "." and "..") */
+        free_subentries(dcrivolp, &ce->down);
+    }
+
+    remove_entry(dcrivolp, ce, get_previdxp(idx));
+    free_orphan_entry(dcrivolp, ce, idx);
+}
+
+/**
+ * insert the new entry into the parent, sorted into position
+ */
+static void insert_file_entry(struct file_base_info *dirinfop,
+                              struct dircache_entry *ce)
+{
+    /* DIRCACHE_NATIVE: the entires are sorted into the spot it would be on
+     * the storage medium based upon the directory entry number, in-progress
+     * scans will catch it or miss it in just the same way they would if
+     * directly scanning the disk. If this is behind an init scan, it gets
+     * added anyway; if in front of it, then scanning will compare what it
+     * finds in order to avoid adding a duplicate.
+     *
+     * All others: the order of entries of the host filesystem is not known so
+     * this must be placed at the end so that a build scan won't miss it and
+     * add a duplicate since it will be comparing any entries it finds in front
+     * of it.
+     */
+    int diridx = dirinfop->dcfile.idx;
+    int *nextp = get_downidxp(diridx);
+
+    while (8675309)
+    {
+        struct dircache_entry *nextce = get_entry(*nextp);
+        if (!nextce)
+            break;
+
+#ifdef DIRCACHE_NATIVE
+        if (nextce->direntry > ce->direntry)
+            break;
+
+        /* now, nothing should be equal to ours or that is a bug since it
+           would already exist (and it shouldn't because it's just been
+           created or moved) */
+#endif /* DIRCACHE_NATIVE */
+
+        nextp = &nextce->next;
+    }
+
+    ce->up   = diridx;
+    ce->next = *nextp;
+    *nextp   = get_index(ce);
+}
+
+/**
+ * unlink the entry from its parent and return its pointer to the caller
+ */
+static struct dircache_entry * remove_file_entry(struct file_base_info *infop)
+{
+    struct dircache_runinfo_volume *dcrivolp = DCRIVOL(infop);
+    struct dircache_entry *ce = get_entry(infop->dcfile.idx);
+    remove_entry(dcrivolp, ce, get_previdxp(infop->dcfile.idx));
+    return ce;
+}
+
+/**
+ * set the frontier indicator for the given cache index
+ */
+static void establish_frontier(int idx, uint32_t code)
+{
+    if (idx < 0)
+    {
+        int volume = IF_MV_VOL(-idx - 1);
+        uint32_t val = dircache.dcvol[volume].frontier;
+        if (code & FRONTIER_RENEW)
+            val &= ~FRONTIER_ZONED;
+        dircache.dcvol[volume].frontier = code | (val & FRONTIER_ZONED);
+    }
+    else if (idx > 0)
+    {
+        struct dircache_entry *ce = get_entry(idx);
+        uint32_t val = ce->frontier;
+        if (code & FRONTIER_RENEW)
+            val &= ~FRONTIER_ZONED;
+        ce->frontier = code | (val & FRONTIER_ZONED);
+    }
+}
+
+/**
+ * remove all messages from the queue, responding to anyone waiting
+ */
+static void clear_dircache_queue(void)
 {
     struct queue_event ev;
-    
-    if(!queue_peek(&dircache_queue, &ev))
-        return false;
-    
-    switch (ev.id)
-    {
-        case DIRCACHE_STOP:
-        case SYS_USB_CONNECTED:
-#ifdef HAVE_HOTSWAP
-        case SYS_FS_CHANGED:
-#endif
-            return true;
+
+    while (1)
+    {
+        queue_wait_w_tmo(&dircache_queue, &ev, 0);
+        if (ev.id == SYS_TIMEOUT)
+            break;
+
+        /* respond to any synchronous build queries; since we're already
+           building and thusly allocated, any additional requests can be
+           processed async */
+        if (ev.id == DCM_BUILD)
+        {
+            int *rcp = (int *)ev.data;
+            if (rcp)
+                *rcp = 0;
+        }
     }
-    
-    return false;
 }
 
-#if (CONFIG_PLATFORM & PLATFORM_NATIVE)
-/* scan and build static data (avoid redundancy on stack) */
-static struct
+/**
+ * service dircache_queue during a scan and build
+ */
+static void process_events(void)
 {
-#ifdef HAVE_MULTIVOLUME
-    int volume;
-#endif
-    struct fat_dir *dir;
-    struct fat_direntry *direntry;
-}sab;
-
-static int sab_process_dir(unsigned long startcluster, struct dircache_entry *ce)
-{
-    /* normally, opendir expects a full fat_dir as parent but in our case,
-     * it's completely useless because we don't modify anything
-     * WARNING: this heavily relies on current FAT implementation ! */
-    
-    /* those field are necessary to update the FAT entry in case of modification
-       here we don't touch anything so we put dummy values */
-    sab.dir->entry = 0;
-    sab.dir->entrycount = 0;
-    sab.dir->file.firstcluster = 0;
-    /* open directory */
-    int rc = fat_opendir(IF_MV(sab.volume,) sab.dir, startcluster, sab.dir);
-    if(rc < 0)
-    {
-        logf("fat_opendir failed: %d", rc);
-        return rc;
+    yield();
+
+    /* only count externally generated commands */
+    if (!queue_peek_ex(&dircache_queue, NULL, 0, QPEEK_FILTER1(DCM_BUILD)))
+        return;
+
+    clear_dircache_queue();
+
+    /* this reminds us to keep moving after we're done here; a volume we passed
+       up earlier could have been mounted and need refreshing; it just condenses
+       a slew of requests into one and isn't mistaken for an externally generated
+       command */
+    queue_post(&dircache_queue, DCM_PROCEED, 0);
+}
+
+#if defined (DIRCACHE_NATIVE)
+/**
+ * scan and build the contents of a subdirectory
+ */
+static void sab_process_sub(struct sab *sabp)
+{
+    struct fat_direntry *const fatentp = get_dir_fatent();
+    struct filestr_base *const streamp = &sabp->stream;
+    struct file_base_info *const infop = &sabp->info;
+
+    int idx = infop->dcfile.idx;
+    int *downp = get_downidxp(idx);
+    if (!downp)
+        return;
+
+    while (1)
+    {
+        struct sab_component *compp = --sabp->top;
+        compp->idx   = idx;
+        compp->downp = downp;
+        compp->prevp = downp;
+
+        /* open directory stream */
+        filestr_base_init(streamp);
+        fileobj_fileop_open(streamp, infop, FO_DIRECTORY);
+        fat_rewind(&streamp->fatstr);
+        uncached_rewinddir_internal(infop);
+
+        const long dircluster = streamp->infop->fatfile.firstcluster;
+
+        /* first pass: read directory */
+        while (1)
+        {
+            if (sabp->stack + 1 < sabp->stackend)
+            {
+                /* release control and process queued events */
+                dircache_unlock();
+                process_events();
+                dircache_lock();
+
+                if (sabp->quit || !compp->idx)
+                    break;
+            }
+            /* else an immediate-contents directory scan */
+
+            int rc = uncached_readdir_internal(streamp, infop, fatentp);
+            if (rc <= 0)
+            {
+                if (rc < 0)
+                    sabp->quit = true;
+                else
+                    compp->prevp = downp; /* rewind list */
+
+                break;
+            }
+
+            struct dircache_entry *ce;
+            int prev = *compp->prevp;
+
+            if (prev)
+            {
+                /* there are entries ahead of us; they will be what was just
+                   read or something to be subsequently read; if it belongs
+                   ahead of this one, insert a new entry before it; if it's
+                   the entry just scanned, do nothing further and continue
+                   with the next */
+                ce = get_entry(prev);
+                if (ce->direntry == infop->fatfile.e.entry)
+                {
+                    compp->prevp = &ce->next;
+                    continue; /* already there */
+                }
+            }
+
+            int idx = create_entry(fatentp->name, &ce);
+            if (!idx)
+            {
+                sabp->quit = true;
+                break;
+            }
+
+            /* link it in */
+            ce->up = compp->idx;
+            ce->next = prev;
+            *compp->prevp = idx;
+            compp->prevp = &ce->next;
+
+            if (!(fatentp->attr & ATTR_DIRECTORY))
+                ce->filesize = fatentp->filesize;
+            else if (!is_dotdir_name(fatentp->name))
+                ce->frontier = FRONTIER_NEW; /* this needs scanning */
+
+            /* copy remaining FS info */
+            ce->direntry     = infop->fatfile.e.entry;
+            ce->direntries   = infop->fatfile.e.entries;
+            ce->attr         = fatentp->attr;
+            ce->firstcluster = fatentp->firstcluster;
+            ce->wrtdate      = fatentp->wrtdate;
+            ce->wrttime      = fatentp->wrttime;
+
+            /* resolve queued user bindings */
+            infop->fatfile.firstcluster = fatentp->firstcluster;
+            infop->fatfile.dircluster   = dircluster;
+            infop->dcfile.idx           = idx;
+            infop->dcfile.serialnum     = ce->serialnum;
+            binding_resolve(infop);
+        } /* end while */
+
+        close_stream_internal(streamp);
+
+        if (sabp->quit)
+            return;
+
+        establish_frontier(compp->idx, FRONTIER_SETTLED);
+
+        /* second pass: "recurse!" */
+        struct dircache_entry *ce = NULL;
+
+        while (1)
+        {
+            idx = compp->idx && compp > sabp->stack ? *compp->prevp : 0;
+            if (idx)
+            {
+                ce = get_entry(idx);
+                compp->prevp = &ce->next;
+
+                if (ce->frontier != FRONTIER_SETTLED)
+                    break;
+            }
+            else
+            {
+                /* directory completed or removed/deepest level */
+                compp = ++sabp->top;
+                if (compp >= sabp->stackend)
+                    return; /* scan completed/initial directory removed */
+            }
+        }
+
+        /* even if it got zoned from outside it is about to be scanned in
+           its entirety and may be considered new again */
+        ce->frontier = FRONTIER_NEW;
+        downp = &ce->down;
+
+        /* set up info for next open
+         * IF_MV: "volume" was set when scan began */
+        infop->fatfile.firstcluster = ce->firstcluster;
+        infop->fatfile.dircluster   = dircluster;
+        infop->fatfile.e.entry      = ce->direntry;
+        infop->fatfile.e.entries    = ce->direntries;
+    } /* end while */
+}
+
+/**
+ * scan and build the contents of a directory or volume root
+ */
+static void sab_process_dir(struct file_base_info *infop, bool issab)
+{
+    /* infop should have been fully opened meaning that all its parent
+       directory information is filled in and intact; the binding information
+       should also filled in beforehand */
+
+    /* allocate the stack right now to the max demand */
+    struct dirsab
+    {
+        struct sab           sab;
+        struct sab_component stack[issab ? DIRCACHE_MAX_DEPTH : 1];
+    } dirsab;
+    struct sab *sabp = &dirsab.sab;
+
+    sabp->quit     = false;
+    sabp->stackend = &sabp->stack[ARRAYLEN(dirsab.stack)];
+    sabp->top      = sabp->stackend;
+    sabp->info     = *infop;
+
+    if (issab)
+        DCRIVOL(infop)->sabp = sabp;
+
+    establish_frontier(infop->dcfile.idx, FRONTIER_NEW | FRONTIER_RENEW);
+    sab_process_sub(sabp);
+
+    if (issab)
+        DCRIVOL(infop)->sabp = NULL;
+}
+
+/**
+ * scan and build the entire tree for a volume
+ */
+static void sab_process_volume(struct dircache_volume *dcvolp)
+{
+    int rc;
+
+    int volume = IF_MV_VOL(dcvolp - dircache.dcvol);
+    int idx = -volume - 1;
+
+    logf("dircache - building volume %d", volume);
+
+    /* gather everything sab_process_dir() needs in order to begin a scan */
+    struct file_base_info info;
+    rc = fat_open_rootdir(IF_MV(volume,) &info.fatfile);
+    if (rc < 0)
+    {
+        /* probably not mounted */
+        logf("SAB - no root %d: %d", volume, rc);
+        establish_frontier(idx, FRONTIER_NEW);
+        return;
     }
-    
-    /* first pass : read dir */
-    struct dircache_entry *first_ce = ce;
-    
-    /* read through directory */
-    while((rc = fat_getnext(sab.dir, sab.direntry)) >= 0 && sab.direntry->name[0])
+
+    info.dcfile.idx       = idx;
+    info.dcfile.serialnum = dcvolp->serialnum;
+    sab_process_dir(&info, true);
+}
+
+/**
+ * this function is the back end to the public API's like readdir()
+ */
+int dircache_readdir_dirent(struct filestr_base *stream,
+                            struct dirscan_info *scanp,
+                            struct dirent *entry)
+{
+    struct file_base_info *dirinfop = stream->infop;
+
+    if (!dirinfop->dcfile.serialnum)
+        goto read_uncached; /* no parent cached => no entries cached */
+
+    struct dircache_volume *dcvolp = DCVOL(dirinfop);
+
+    int diridx = dirinfop->dcfile.idx;
+    unsigned int frontier = diridx <= 0 ? dcvolp->frontier :
+                                          get_entry(diridx)->frontier;
+
+    /* if not settled, just readthrough; no binding information is needed for
+       this; if it becomes settled, we'll get scan->dcfile caught up and do
+       subsequent reads with the cache */
+    if (frontier != FRONTIER_SETTLED)
+        goto read_uncached;
+
+    int idx = scanp->dcscan.idx;
+    struct dircache_entry *ce;
+
+    unsigned int direntry = scanp->fatscan.entry;
+    while (1)
     {
-        if(!strcmp(".", sab.direntry->name) ||
-                !strcmp("..", sab.direntry->name))
-            continue;
+        if (idx == 0 || direntry == FAT_RW_VAL) /* rewound? */
+        {
+            idx = diridx <= 0 ? dcvolp->root_down : get_entry(diridx)->down;
+            break;
+        }
 
-        size_t size = strlen(sab.direntry->name) + 1;
-        ce->d_name = (d_names_start -= size);
-        ce->startcluster = sab.direntry->firstcluster;
-        ce->info.size = sab.direntry->filesize;
-        ce->info.attribute = sab.direntry->attr;
-        ce->info.wrtdate = sab.direntry->wrtdate;
-        ce->info.wrttime = sab.direntry->wrttime;
+        /* validate last entry scanned; it might have been replaced between
+           calls or not there at all any more; if so, start the cache reader
+           at the beginning and fast-forward to the correct point as indicated
+           by the FS scanner */
+        ce = get_entry(idx);
+        if (ce && ce->serialnum == scanp->dcscan.serialnum)
+        {
+            idx = ce->next;
+            break;
+        }
 
-        strcpy(ce->d_name, sab.direntry->name); 
-        dircache_size += size;
-        
-        if(ce->info.attribute & FAT_ATTR_DIRECTORY)
-            dircache_gen_down(ce);
-                
-        ce = dircache_gen_next(ce);
-        if(ce == NULL)
-            return -5;
-        
-        /* When simulator is used, it's only safe to yield here. */
-        if(thread_enabled)
+        idx = 0;
+    }
+
+    while (1)
+    {
+        ce = get_entry(idx);
+        if (!ce)
         {
-            /* Stop if we got an external signal. */
-            if(check_event_queue())
-                return -6;
-            yield();
+            empty_dirent(entry);
+            scanp->fatscan.entries = 0;
+            return 0; /* end of dir */
         }
+
+        if (ce->direntry > direntry || direntry == FAT_RW_VAL)
+            break; /* cache reader is caught up to FS scan */
+
+        idx = ce->next;
     }
-    
-    /* add "." and ".." */
-    ce->d_name = dot;
-    ce->info.attribute = FAT_ATTR_DIRECTORY;
-    ce->startcluster = startcluster;
-    ce->info.size = 0;
-    ce->down = first_ce;
-    
-    ce = dircache_gen_next(ce);
-    
-    ce->d_name = dotdot;
-    ce->info.attribute = FAT_ATTR_DIRECTORY;
-    ce->startcluster = (first_ce->up ? first_ce->up->startcluster : 0);
-    ce->info.size = 0;
-    ce->down = first_ce->up;
-    
-    /* second pass: recurse ! */
-    ce = first_ce;
-    
-    while(rc >= 0 && ce)
-    {
-        if(ce->d_name != NULL && ce->down != NULL && strcmp(ce->d_name, ".")
-                && strcmp(ce->d_name, ".."))
-            rc = sab_process_dir(ce->startcluster, ce->down);
-        
-        ce = ce->next;
-    }
-    
+
+    /* basic dirent information */
+    entry_name_copy(entry->d_name, ce);
+    entry->info.attr    = ce->attr;
+    entry->info.size    = (ce->attr & ATTR_DIRECTORY) ? 0 : ce->filesize;
+    entry->info.wrtdate = ce->wrtdate;
+    entry->info.wrttime = ce->wrttime;
+
+    /* FS scan information */
+    scanp->fatscan.entry    = ce->direntry;
+    scanp->fatscan.entries  = ce->direntries;
+
+    /* dircache scan information */
+    scanp->dcscan.idx       = idx;
+    scanp->dcscan.serialnum = ce->serialnum;
+
+    /* return whether this needs decoding */
+    int rc = ce->direntries == 1 ? 2 : 1;
+
+    yield();
     return rc;
+
+read_uncached:
+    dircache_dcfile_init(&scanp->dcscan);
+    return uncached_readdir_dirent(stream, scanp, entry);
 }
 
-/* used during the generation */
-static struct fat_dir sab_fat_dir;
+/**
+ * rewind the directory scan cursor
+ */
+void dircache_rewinddir_dirent(struct dirscan_info *scanp)
+{
+    uncached_rewinddir_dirent(scanp);
+    dircache_dcfile_init(&scanp->dcscan);
+}
 
-static int dircache_scan_and_build(IF_MV(int volume,) struct dircache_entry *ce)
+/**
+ * this function is the back end to file API internal scanning, which requires
+ * much more detail about the directory entries; this is allowed to make
+ * assumptions about cache state because the cache will not be altered during
+ * the scan process; an additional important property of internal scanning is
+ * that any available binding information is not ignored even when a scan
+ * directory is frontier zoned.
+ */
+int dircache_readdir_internal(struct filestr_base *stream,
+                              struct file_base_info *infop,
+                              struct fat_direntry *fatent)
 {
-    memset(ce, 0, sizeof(struct dircache_entry));
+    /* call with writer exclusion */
+    struct file_base_info *dirinfop = stream->infop;
+    struct dircache_volume *dcvolp = DCVOL(dirinfop);
 
-#ifdef HAVE_MULTIVOLUME
-    if (volume > 0)
-    {
-        /* broken for 100+ volumes because the format string is too small
-         * and we use that for size calculation */
-        const size_t max_len = VOL_ENUM_POS + 3;
-        ce->d_name = (d_names_start -= max_len);
-        snprintf(ce->d_name, max_len, VOL_NAMES, volume);
-        dircache_size += max_len;
-        ce->info.attribute = FAT_ATTR_DIRECTORY | FAT_ATTR_VOLUME;
-        ce->info.size = 0;
-        append_position = dircache_gen_next(ce);
-        ce = dircache_gen_down(ce);
+    /* assume binding "not found" */
+    infop->dcfile.serialnum = 0;
+
+    /* is parent cached? if not, readthrough because nothing is here yet */
+    if (!dirinfop->dcfile.serialnum)
+        return uncached_readdir_internal(stream, infop, fatent);
+
+    int diridx = dirinfop->dcfile.idx;
+    unsigned int frontier = diridx < 0 ?
+        dcvolp->frontier : get_entry(diridx)->frontier;
+
+    int idx = infop->dcfile.idx;
+    if (idx == 0) /* rewound? */
+        idx = diridx <= 0 ? dcvolp->root_down : get_entry(diridx)->down;
+    else
+        idx = get_entry(idx)->next;
+
+    struct dircache_entry *ce = get_entry(idx);
+    if (frontier != FRONTIER_SETTLED)
+    {
+        /* the directory being read is reported to be incompletely cached;
+           readthrough and if the entry exists, return it with its binding
+           information; otherwise return the uncached read result while
+           maintaining the last index */
+        int rc = uncached_readdir_internal(stream, infop, fatent);
+        if (rc <= 0 || !ce || ce->direntry > infop->fatfile.e.entry)
+            return rc;
+
+        /* entry matches next one to read */
+    }
+    else if (!ce)
+    {
+        /* end of dir */
+        fat_empty_fat_direntry(fatent);
+        infop->fatfile.e.entries = 0;
+        return 0;
     }
-#endif
 
-    struct fat_direntry direntry; /* ditto */
-#ifdef HAVE_MULTIVOLUME
-    sab.volume = volume;
-#endif
-    sab.dir = &sab_fat_dir;
-    sab.direntry = &direntry;
-    
-    return sab_process_dir(0, ce);
+    /* FS entry information that we maintain */
+    entry_name_copy(fatent->name, ce);
+    fatent->shortname[0]     = '\0';
+    fatent->attr             = ce->attr;
+    /* file code file scanning does not need time information */
+    fatent->filesize         = (ce->attr & ATTR_DIRECTORY) ? 0 : ce->filesize;
+    fatent->firstcluster     = ce->firstcluster;
+
+    /* FS entry directory information */
+    infop->fatfile.e.entry   = ce->direntry;
+    infop->fatfile.e.entries = ce->direntries;
+
+    /* dircache file binding information */
+    infop->dcfile.idx        = idx;
+    infop->dcfile.serialnum  = ce->serialnum;
+
+    /* return whether this needs decoding */
+    int rc = ce->direntries == 1 ? 2 : 1;
+
+    if (frontier == FRONTIER_SETTLED)
+    {
+        static long next_yield;
+        if (TIME_AFTER(current_tick, next_yield))
+        {
+            yield();
+            next_yield = current_tick + HZ/50;
+        }
+    }
+
+    return rc;
 }
-#elif (CONFIG_PLATFORM & PLATFORM_HOSTED) /* PLATFORM_HOSTED */
+
+/**
+ * rewind the scan position for an internal scan
+ */
+void dircache_rewinddir_internal(struct file_base_info *infop)
+{
+    uncached_rewinddir_internal(infop);
+    dircache_dcfile_init(&infop->dcfile);
+}
+
+#else /* !DIRCACHE_NATIVE (for all others) */
+
+#####################
+/* we require access to the host functions */
+#undef opendir
+#undef readdir
+#undef closedir
+#undef rewinddir
+
 static char sab_path[MAX_PATH];
 
 static int sab_process_dir(struct dircache_entry *ce)
 {
     struct dirent_uncached *entry;
     struct dircache_entry *first_ce = ce;
-    DIR_UNCACHED *dir = opendir_uncached(sab_path);
+    DIR *dir = opendir(sab_path);
     if(dir == NULL)
     {
         logf("Failed to opendir_uncached(%s)", sab_path);
         return -1;
     }
-    
-    while((entry = readdir_uncached(dir)))
+
+    while (1)
     {
-        if(!strcmp(".", entry->d_name) ||
-                !strcmp("..", entry->d_name))
+        if (!(entry = readdir(dir)))
+            break;
+
+        if (IS_DOTDIR_NAME(entry->d_name))
+        {
+            /* add "." and ".." */
+            ce->info.attribute = ATTR_DIRECTORY;
+            ce->info.size = 0;
+            ce->down = entry->d_name[1] == '\0' ? first_ce : first_ce->up;
+            strcpy(ce->dot_d_name, entry->d_name);
             continue;
+        }
 
         size_t size = strlen(entry->d_name) + 1;
         ce->d_name = (d_names_start -= size);
@@ -436,10 +1666,10 @@ static int sab_process_dir(struct dircache_entry *ce)
 
         strcpy(ce->d_name, entry->d_name);
         dircache_size += size;
-        
+
         if(entry->info.attribute & ATTR_DIRECTORY)
         {
-            dircache_gen_down(ce);
+            dircache_gen_down(ce, ce);
             if(ce->down == NULL)
             {
                 closedir_uncached(dir);
@@ -450,1189 +1680,1504 @@ static int sab_process_dir(struct dircache_entry *ce)
             /* append entry */
             strlcpy(&sab_path[pathpos], "/", sizeof(sab_path) - pathpos);
             strlcpy(&sab_path[pathpos+1], entry->d_name, sizeof(sab_path) - pathpos - 1);
-            
+
             int rc = sab_process_dir(ce->down);
             /* restore path */
             sab_path[pathpos] = '\0';
-            
+
             if(rc < 0)
             {
                 closedir_uncached(dir);
                 return rc;
             }
         }
-        
-        ce = dircache_gen_next(ce);
-        if(ce == NULL)
+
+        ce = dircache_gen_entry(ce);
+        if (ce == NULL)
             return -5;
-        
-        /* When simulator is used, it's only safe to yield here. */
-        if(thread_enabled)
-        {
-            /* Stop if we got an external signal. */
-            if(check_event_queue())
-                return -1;
-            yield();
-        }
+
+        yield();
     }
-    
-    /* add "." and ".." */
-    ce->d_name = dot;
-    ce->info.attribute = ATTR_DIRECTORY;
-    ce->info.size = 0;
-    ce->down = first_ce;
-    
-    ce = dircache_gen_next(ce);
-    
-    ce->d_name = dotdot;
-    ce->info.attribute = ATTR_DIRECTORY;
-    ce->info.size = 0;
-    ce->down = first_ce->up;
-    
+
     closedir_uncached(dir);
-    return 0;
+    return 1;
 }
 
-static int dircache_scan_and_build(IF_MV(int volume,) struct dircache_entry *ce)
+static int sab_process_volume(IF_MV(int volume,) struct dircache_entry *ce)
 {
-    #ifdef HAVE_MULTIVOLUME
-    (void) volume;
-    #endif
     memset(ce, 0, sizeof(struct dircache_entry));
-    
     strlcpy(sab_path, "/", sizeof sab_path);
     return sab_process_dir(ce);
 }
-#endif /* PLATFORM_NATIVE */
 
-/**
- * Internal function to get a pointer to dircache_entry for a given filename.
- *   path: Absolute path to a file or directory (see comment)
- *   go_down: Returns the first entry of the directory given by the path (see comment)
- *
- * As a a special case, accept path="" as an alias for "/".
- * Also if the path omits the first '/', it will be accepted.
- *
- * * If get_down=true:
- *   If path="/", the returned entry is the first of root directory (ie dircache_root)
- *   Otherwise, if 'entry' is the returned value when get_down=false, 
- *   the functions returns entry->down (which can be NULL)
- *
- * * If get_down=false:
- *   If path="/chunk_1/chunk_2/.../chunk_n" then this functions returns the entry
- *   root_entry()->chunk_1->chunk_2->...->chunk_(n-1)
- *   Which means that
- *   dircache_get_entry(path)->d_name == chunk_n
- *
- *   If path="/", the returned entry is NULL.
- *   If the entry doesn't exist, return NULL
- *
- *  NOTE: this functions silently handles double '/'
- */
-static struct dircache_entry* dircache_get_entry(const char *path, bool go_down)
-{
-    char namecopy[MAX_PATH];
-    char* part;
-    char* end;
-    
-    bool at_root = true;
-    struct dircache_entry *cache_entry = dircache_root;
-    
-    strlcpy(namecopy, path, sizeof(namecopy));
-    
-    for(part = strtok_r(namecopy, "/", &end); part; part = strtok_r(NULL, "/", &end))
-    {
-        /* If request another chunk, the current entry has to be directory
-         * and so cache_entry->down has to be non-NULL/
-         * Special case of root because it's already the first entry of the root directory
-         *
-         * NOTE: this is safe even if cache_entry->down is NULL */
-        if(!at_root)
-            cache_entry = cache_entry->down;
-        else
-            at_root = false;
-        
-        /* scan dir for name */
-        while(cache_entry != NULL)
-        {
-            /* skip unused entries */
-            if(cache_entry->d_name == NULL)
-            {
-                cache_entry = cache_entry->next;
-                continue;
-            }
-            /* compare names */
-            if(!strcasecmp(part, cache_entry->d_name))
-                break;
-            /* go to next entry */
-            cache_entry = cache_entry->next;
-        }
-        
-        /* handle not found case */
-        if(cache_entry == NULL)
-            return NULL;
-    }
+int dircache_readdir_r(struct dircache_dirscan *dir, struct dirent *result)
+{
+    if (dircache_state != DIRCACHE_READY)
+        return readdir_r(dir->###########3, result, &result);
 
-    /* NOTE: here cache_entry!=NULL so taking ->down is safe */
-    if(go_down)
-        return at_root ? cache_entry : cache_entry->down;
-    else
-        return at_root ? NULL : cache_entry;
-}
+    bool first = dir->dcinfo.scanidx == REWIND_INDEX;
+    struct dircache_entry *ce = get_entry(first ? dir->dcinfo.index :
+                                                  dir->dcinfo.scanidx);
 
-#ifdef HAVE_EEPROM_SETTINGS
+    ce = first ? ce->down : ce->next;
 
-#define DIRCACHE_MAGIC  0x00d0c0a1
-struct dircache_maindata {
-    long magic;
-    long size;
-    long entry_count;
-    long appflags;
-    struct dircache_entry *root_entry;
-    char *d_names_start;
-};
+    if (ce == NULL)
+        return 0;
+
+    dir->scanidx = ce - dircache_root;
+
+    strlcpy(result->d_name, ce->d_name, sizeof (result->d_name));
+    result->info = ce->dirinfo;
+
+    return 1;
+}
+
+#endif /* DIRCACHE_* */
 
 /**
- * Function to load the internal cache structure from disk to initialize
- * the dircache really fast and little disk access.
+ * reset the cache for the specified volume
  */
-int dircache_load(void)
+static void reset_volume(IF_MV_NONVOID(int volume))
 {
-    struct dircache_maindata maindata;
-    ssize_t bytes_read;
-    int fd;
-        
-    if (dircache_initialized)
-        return -1;
-        
-    logf("Loading directory cache");
-    dircache_size = 0;
-    
-    fd = open_dircache_file(O_RDONLY, 0);
-    if (fd < 0)
-        return -2;
-        
-    bytes_read = read(fd, &maindata, sizeof(struct dircache_maindata));
-    if (bytes_read != sizeof(struct dircache_maindata)
-        || maindata.magic != DIRCACHE_MAGIC || maindata.size <= 0)
+    FOR_EACH_VOLUME(volume, i)
     {
-        logf("Dircache file header error");
-        close(fd);
-        remove_dircache_file();
-        return -3;
-    }
-    
-    allocated_size = maindata.size + DIRCACHE_RESERVE;
-    dircache_handle = core_alloc_ex("dircache", allocated_size, &ops);
-    /* block movement during upcoming I/O */
-    dont_move = true;
-    dircache_root = core_get_data(dircache_handle);
-    ALIGN_BUFFER(dircache_root, allocated_size, sizeof(struct dircache_entry*));
-    entry_count = maindata.entry_count;
-    appflags = maindata.appflags;
-
-    /* read the dircache file into memory,
-     * start with the struct dircache_entries */
-    ssize_t bytes_to_read = entry_count*sizeof(struct dircache_entry);
-    bytes_read = read(fd, dircache_root, bytes_to_read);
-    
-    if (bytes_read != bytes_to_read)
-    {
-        logf("Dircache read failed #1");
-        return -6;
-    }
-
-    /* continue with the d_names. Fix up pointers to them if needed */
-    bytes_to_read = maindata.size - bytes_to_read;
-    d_names_start = (char*)dircache_root + allocated_size - bytes_to_read;
-    bytes_read = read(fd, d_names_start, bytes_to_read);
-    close(fd);
-    remove_dircache_file();
-    if (bytes_read != bytes_to_read)
-    {
-        logf("Dircache read failed #2");
-        return -7;
+        struct dircache_volume *dcvolp = DCVOL(i);
+
+        if (dcvolp->status == DIRCACHE_IDLE)
+            continue; /* idle => nothing happening there */
+
+        struct dircache_runinfo_volume *dcrivolp = DCRIVOL(i);
+
+        /* stop any scan and build on this one */
+        if (dcrivolp->sabp)
+            dcrivolp->sabp->quit = true;
+
+    #ifdef HAVE_MULTIVOLUME
+        /* if this call is for all volumes, subsequent code will just reset
+           the cache memory usage and the freeing of individual entries may
+           be skipped */
+        if (volume >= 0)
+            free_subentries(NULL, &dcvolp->root_down);
+        else
+    #endif
+            binding_dissolve_volume(dcrivolp);
+
+        /* set it back to unscanned */
+        dcvolp->status      = DIRCACHE_IDLE;
+        dcvolp->frontier    = FRONTIER_NEW;
+        dcvolp->root_down   = 0;
+        dcvolp->build_ticks = 0;
+        dcvolp->serialnum   = 0;
     }
+}
 
-    d_names_end = d_names_start + bytes_read;
-    dot = d_names_end - sizeof(".");
-    dotdot = dot - sizeof("..");
+/**
+ * reset the entire cache state for all volumes
+ */
+static void reset_cache(void)
+{
+    if (!dircache_runinfo.handle)
+        return; /* no buffer => nothing cached */
+
+    /* blast all the volumes */
+    reset_volume(IF_MV(-1));
+
+#ifdef DIRCACHE_DUMPSTER
+    dumpster_clean_buffer(dircache_runinfo.p + ENTRYSIZE,
+                          dircache_runinfo.bufsize);
+#endif /* DIRCACHE_DUMPSTER */
+
+    /* reset the memory */
+    dircache.free_list    = 0;
+    dircache.size         = 0;
+    dircache.sizeused     = 0;
+    dircache.numentries   = 0;
+    dircache.names        = dircache_runinfo.bufsize + ENTRYSIZE;
+    dircache.sizenames    = 0;
+    dircache.namesfree    = 0;
+    dircache.nextnamefree = 0;
+    *get_name(dircache.names - 1) = 0;
+    /* dircache.last_serialnum stays */
+    /* dircache.reserve_used stays */
+    /* dircache.last_size stays */
+}
 
-    /* d_names are in reverse order, so the last entry points to the first string */
-    ptrdiff_t offset_d_names = maindata.d_names_start - d_names_start;
-    ptrdiff_t offset_entries = maindata.root_entry - dircache_root;
-    offset_entries *= sizeof(struct dircache_entry); /* make it bytes */
+/**
+ * checks each "idle" volume and builds it
+ */
+static void build_volumes(void)
+{
+    buffer_lock();
 
-    /* offset_entries is less likely to differ, so check if it's 0 in the loop
-     * offset_d_names however is almost always non-zero, since dircache_save()
-     * creates a file which causes the reserve buffer to be used. since
-     * we allocate a new, empty DIRCACHE_RESERVE here, the strings are
-     * farther behind */
-    if (offset_entries != 0 || offset_d_names != 0)
+    for (int i = 0; i < NUM_VOLUMES; i++)
     {
-        for(unsigned i = 0; i < entry_count; i++)
-        {
-            if (dircache_root[i].d_name)
-                dircache_root[i].d_name -= offset_d_names;
+        /* this does reader locking but we already own that */
+        if (!volume_ismounted(IF_MV(i)))
+            continue;
 
-            if (offset_entries == 0)
-                continue;
-            if (dircache_root[i].next_char)
-                dircache_root[i].next_char -= offset_entries;
-            if (dircache_root[i].up_char)
-                dircache_root[i].up_char -= offset_entries;
-            if (dircache_root[i].down_char)
-                dircache_root[i].down_char -= offset_entries;
-        }
+        struct dircache_volume *dcvolp = DCVOL(i);
+
+        /* can't already be "scanning" because that's us; doesn't retry
+           "ready" volumes */
+        if (dcvolp->status == DIRCACHE_READY)
+            continue;
+
+        /* measure how long it takes to build the cache for each volume */
+        if (!dcvolp->serialnum)
+            dcvolp->serialnum = next_serialnum();
+
+        dcvolp->status = DIRCACHE_SCANNING;
+        dcvolp->start_tick = current_tick;
+
+        sab_process_volume(dcvolp);
+
+        if (dircache_runinfo.suspended)
+            break;
+
+        /* whatever happened, it's ready unless reset */
+        dcvolp->build_ticks = current_tick - dcvolp->start_tick;
+        dcvolp->status = DIRCACHE_READY;
     }
 
-    /* Cache successfully loaded. */
-    dircache_size = maindata.size;
-    reserve_used = 0;
-    logf("Done, %ld KiB used", dircache_size / 1024);
-    dircache_initialized = true;
-    memset(fd_bindings, 0, sizeof(fd_bindings));
-    dont_move = false;
+    size_t reserve_used = reserve_buf_used();
+    if (reserve_used > dircache.reserve_used)
+        dircache.reserve_used = reserve_used;
+
+    if (DIRCACHE_STUFFED(reserve_used))
+        dircache.last_size = 0; /* reset */
+    else if (dircache.size > dircache.last_size ||
+        dircache.last_size - dircache.size > DIRCACHE_RESERVE)
+        dircache.last_size = dircache.size;
+
+    logf("Done, %ld KiB used", dircache.size / 1024);
 
-    return 0;
+    buffer_unlock();
 }
 
 /**
- * Function to save the internal cache stucture to disk for fast loading
- * on boot.
+ * allocate buffer and return whether or not a synchronous build should take
+ * place; if 'realloced' is NULL, it's just a query about what will happen
  */
-int dircache_save(void)
+static int prepare_build(bool *realloced)
 {
-    struct dircache_maindata maindata;
-    int fd;
-    unsigned long bytes_written;
+    /* called holding dircache lock */
+    size_t size = dircache.last_size;
 
-    remove_dircache_file();
-    
-    if (!dircache_initialized)
-        return -1;
+#ifdef HAVE_EEPROM_SETTINGS
+    if (realloced)
+    {
+        dircache_unlock();
+        remove_dircache_file();
+        dircache_lock();
 
-    logf("Saving directory cache");
-    dont_move = true;
-    fd = open_dircache_file(O_WRONLY | O_CREAT | O_TRUNC, 0666);
+        if (dircache_runinfo.suspended)
+            return -1;
+    }
+#endif /* HAVE_EEPROM_SETTINGS */
+
+    bool stuffed = DIRCACHE_STUFFED(dircache.reserve_used);
+    if (dircache_runinfo.bufsize > size && !stuffed)
+    {
+        if (realloced)
+            *realloced = false;
 
-    maindata.magic = DIRCACHE_MAGIC;
-    maindata.size = dircache_size;
-    maindata.root_entry = dircache_root;
-    maindata.d_names_start = d_names_start;
-    maindata.entry_count = entry_count;
-    maindata.appflags = appflags;
+        return 0; /* start a transparent rebuild */
+    }
 
-    /* Save the info structure */
-    bytes_written = write(fd, &maindata, sizeof(struct dircache_maindata));
-    if (bytes_written != sizeof(struct dircache_maindata))
+    int syncbuild = size > 0 && !stuffed ? 0 : 1;
+
+    if (!realloced)
+        return syncbuild;
+
+    if (syncbuild)
     {
-        close(fd);
-        logf("dircache: write failed #1");
-        return -2;
+        /* start a non-transparent rebuild */
+        /* we'll use the entire audiobuf to allocate the dircache */
+        size = audio_buffer_available() + dircache_runinfo.bufsize;
+        /* try to allocate at least the min and no more than the limit */
+        size = MAX(DIRCACHE_MIN, MIN(size, DIRCACHE_LIMIT));
     }
+    else
+    {
+        /* start a transparent rebuild */
+        size = MAX(size, DIRCACHE_RESERVE) + DIRCACHE_RESERVE*2;
+    }
+
+    *realloced = true;
+    reset_cache();
 
-    /* Dump whole directory cache to disk
-     * start by writing the dircache_entries */
-    size_t bytes_to_write = entry_count*sizeof(struct dircache_entry);
-    bytes_written = write(fd, dircache_root, bytes_to_write);
-    if (bytes_written != bytes_to_write)
+    buffer_lock();
+
+    int handle = reset_buffer();
+    dircache_unlock();
+
+    if (handle > 0)
+        core_free(handle);
+
+    handle = alloc_cache(size);
+
+    dircache_lock();
+
+    if (dircache_runinfo.suspended && handle > 0)
     {
-        logf("dircache: write failed #2");
-        return -3;
+        /* if we got suspended, don't keep this huge buffer around */
+        dircache_unlock();
+        core_free(handle);
+        handle = 0;
+        dircache_lock();
     }
 
-    /* continue with the d_names */
-    bytes_to_write = d_names_end - d_names_start;
-    bytes_written = write(fd, d_names_start, bytes_to_write);
-    close(fd);
-    if (bytes_written != bytes_to_write)
+    if (handle <= 0)
     {
-        logf("dircache: write failed #3");
-        return -4;
+        buffer_unlock();
+        return -1;
     }
 
-    dont_move = false;
-    return 0;
+    set_buffer(handle, size);
+    buffer_unlock();
+
+    return syncbuild;
 }
-#endif /* HAVE_EEPROM_SETTINGS */
 
 /**
- * Internal function which scans the disk and creates the dircache structure.
+ * compact the dircache buffer after a successful full build
  */
-static int dircache_do_rebuild(void)
+static void compact_cache(void)
 {
-    struct dircache_entry* root_entry;
-    unsigned int start_tick;
-    int i;
-    
-    /* Measure how long it takes build the cache. */
-    start_tick = current_tick;
-    dircache_initializing = true;
-    appflags = 0;
+    /* called holding dircache lock */
+    if (dircache_runinfo.suspended)
+        return;
 
-    /* reset dircache and alloc root entry */
-    entry_count = 0;
-    root_entry = allocate_entry();
-    dont_move = true;
+    void *p = dircache_runinfo.p + ENTRYSIZE;
+    size_t leadsize = dircache.numentries * ENTRYSIZE + DIRCACHE_RESERVE;
 
-#ifdef HAVE_MULTIVOLUME
-    append_position = root_entry;
+    void *dst = p + leadsize;
+    void *src = get_name(dircache.names - 1);
+    if (dst >= src)
+        return; /* cache got bigger than expected; never mind that */
+
+    /* slide the names up in memory */
+    memmove(dst, src, dircache.sizenames + 2);
 
-    for (i = NUM_VOLUMES; i >= 0; i--)
+    /* fix up name indexes */
+    ptrdiff_t offset = dst - src;
+
+    FOR_EACH_CACHE_ENTRY(ce)
     {
-        if (fat_ismounted(i))
-        {
-#endif
-            cpu_boost(true);
-#ifdef HAVE_MULTIVOLUME
-            if (dircache_scan_and_build(IF_MV(i,) append_position) < 0)
-#else
-            if (dircache_scan_and_build(IF_MV(0,) root_entry) < 0)
-#endif /* HAVE_MULTIVOLUME */
-            {
-                logf("dircache_scan_and_build failed");
-                cpu_boost(false);
-                dircache_size = 0;
-                dircache_initializing = false;
-                dont_move = false;
-                return -2;
-            }
-            cpu_boost(false);
-#ifdef HAVE_MULTIVOLUME
-        }
+        if (!ce->tinyname)
+             ce->name += offset;
     }
-#endif
 
-    logf("Done, %ld KiB used", dircache_size / 1024);
-    
-    dircache_initialized = true;
-    dircache_initializing = false;
-    cache_build_ticks = current_tick - start_tick;
-    
-    /* Initialized fd bindings. */
-    memset(fd_bindings, 0, sizeof(fd_bindings));
-    for (i = 0; i < fdbind_idx; i++)
-        dircache_bind(fdbind_cache[i].fd, fdbind_cache[i].path);
-    fdbind_idx = 0;
-    
-    if (thread_enabled)
-    {
-        if (allocated_size - dircache_size < DIRCACHE_RESERVE)
-            reserve_used = DIRCACHE_RESERVE - (allocated_size - dircache_size);
-    }
-
-    dont_move = false;
-    return 1;
-}
+    dircache.names += offset;
 
-/*
- * Free all associated resources, if any */
-static void dircache_free(void)
-{
-    if (dircache_handle > 0)
-        dircache_handle = core_free(dircache_handle);
-    dircache_size = allocated_size = 0;
+    /* assumes beelzelib doesn't do things like calling callbacks or changing
+       the pointer as a result of the shrink operation; it doesn't as of now
+       but if it ever does that may very well cause deadlock problems since
+       we're holding filesystem locks */
+    size_t newsize = leadsize + dircache.sizenames + 1;
+    core_shrink(dircache_runinfo.handle, p, newsize + 1);
+    dircache_runinfo.bufsize = newsize;
+    dircache.reserve_used = 0;
 }
 
 /**
- * Internal thread that controls transparent cache building.
+ * internal thread that controls cache building; exits when no more requests
+ * are pending or the cache is suspended
  */
 static void dircache_thread(void)
 {
     struct queue_event ev;
 
+    /* calls made within the loop reopen the lock */
+    dircache_lock();
+
     while (1)
     {
-        queue_wait(&dircache_queue, &ev);
-
-        switch (ev.id)
+        queue_wait_w_tmo(&dircache_queue, &ev, 0);
+        if (ev.id == SYS_TIMEOUT || dircache_runinfo.suspended)
         {
-#ifdef HAVE_HOTSWAP
-            case SYS_FS_CHANGED:
-                if (!dircache_initialized)
-                    break;
-                dircache_initialized = false;
-#endif
-            case DIRCACHE_BUILD:
-                thread_enabled = true;
-                if (dircache_do_rebuild() < 0)
-                    dircache_free();
-                thread_enabled = false;
-                break ;
-                
-            case DIRCACHE_STOP:
-                logf("Stopped the rebuilding.");
-                dircache_initialized = false;
-                break ;
-            
-            case SYS_USB_CONNECTED:
-                usb_acknowledge(SYS_USB_CONNECTED_ACK);
-                usb_wait_for_disconnect(&dircache_queue);
-                break ;
+            /* nothing left to do/suspended */
+            if (dircache_runinfo.suspended)
+                clear_dircache_queue();
+            dircache_runinfo.thread_done = true;
+            break;
         }
-    }
-}
 
-static void generate_dot_d_names(void)
-{
-    dot = (d_names_start -= sizeof("."));
-    dotdot = (d_names_start -= sizeof(".."));
-    dircache_size += sizeof(".") + sizeof("..");
-    strcpy(dot, ".");
-    strcpy(dotdot, "..");
+        /* background-only builds are not allowed if a synchronous build is
+           required first; test what needs to be done and if it checks out,
+           do it for real below */
+        int *rcp = (int *)ev.data;
+
+        if (!rcp && prepare_build(NULL) > 0)
+            continue;
+
+        bool realloced;
+        int rc = prepare_build(&realloced);
+        if (rcp)
+            *rcp = rc;
+
+        if (rc < 0)
+            continue;
+
+        trigger_cpu_boost();
+        build_volumes();
+
+        /* if it was reallocated, compact it */
+        if (realloced)
+            compact_cache();
+     }
+
+     dircache_unlock();
 }
 
 /**
- * Start scanning the disk to build the dircache.
- * Either transparent or non-transparent build method is used.
+ * post a scan and build message to the thread, starting it if required
  */
-int dircache_build(int last_size)
+static bool dircache_thread_post(int volatile *rcp)
 {
-    if (dircache_initialized || thread_enabled)
-        return -3;
+    if (dircache_runinfo.thread_done)
+    {
+        /* mustn't recreate until it exits so that the stack isn't reused */
+        thread_wait(dircache_runinfo.thread_id);
+        dircache_runinfo.thread_done = false;
+        dircache_runinfo.thread_id = create_thread(
+            dircache_thread, dircache_stack, sizeof (dircache_stack), 0,
+            dircache_thread_name IF_PRIO(, PRIORITY_BACKGROUND)
+            IF_COP(, CPU));
+    }
 
-    logf("Building directory cache");
-#ifdef HAVE_EEPROM_SETTINGS
-    remove_dircache_file();
-#endif
+    bool started = dircache_runinfo.thread_id != 0;
 
-    /* Background build, dircache has been previously allocated and */
-    if (allocated_size > MAX(last_size, 0))
-    {
-        d_names_start = d_names_end;
-        dircache_size = 0;
-        reserve_used = 0;
-        thread_enabled = true;
-        dircache_initializing = true;
-        generate_dot_d_names();
-        
-        queue_post(&dircache_queue, DIRCACHE_BUILD, 0);
-        return 2;
-    }
-
-    /* start by freeing, as we possibly re-allocate */
-    dircache_free();
-
-    if (last_size > DIRCACHE_RESERVE && last_size < DIRCACHE_LIMIT )
-    {
-        allocated_size = last_size + DIRCACHE_RESERVE;
-        dircache_handle = core_alloc_ex("dircache", allocated_size, &ops);
-        dircache_root = core_get_data(dircache_handle);
-        ALIGN_BUFFER(dircache_root, allocated_size, sizeof(struct dircache_entry*));
-        d_names_start = d_names_end = ((char*)dircache_root)+allocated_size-1;
-        dircache_size = 0;
-        thread_enabled = true;
-        generate_dot_d_names();
-
-        /* Start a transparent rebuild. */
-        queue_post(&dircache_queue, DIRCACHE_BUILD, 0);
-        return 3;
-    }
-
-    /* We'll use the entire audiobuf to allocate the dircache
-     * struct dircache_entrys are allocated from the beginning
-     * and their corresponding d_name from the end
-     * after generation the buffer will be compacted with DIRCACHE_RESERVE
-     * free bytes inbetween */
-    size_t available = audio_buffer_available();
-    /* try to allocate at least 1MB, the more the better */
-    if (available < 1<<20) available = 1<<20;
-    if (available > DIRCACHE_LIMIT) available = DIRCACHE_LIMIT;
-
-    dircache_handle = core_alloc_ex("dircache", available, &ops);
-    if (dircache_handle <= 0)
-        return -1; /* that was not successful, should try rebooting */
-    char* buf = core_get_data(dircache_handle);
-    dircache_root = (struct dircache_entry*)ALIGN_UP(buf,
-                                                sizeof(struct dircache_entry*));
-    d_names_start = d_names_end = buf + available - 1;
-    dircache_size = 0;
-    generate_dot_d_names();
-
-    /* Start a non-transparent rebuild. */
-    int res = dircache_do_rebuild();
-    if (res < 0)
-        goto fail;
-
-    /* now compact the dircache buffer */
-    char* dst = ((char*)&dircache_root[entry_count] + DIRCACHE_RESERVE);
-    ptrdiff_t offset = d_names_start - dst;
-    if (offset <= 0) /* something went wrong */
-    {
-        res = -1;
-        goto fail;
-    }
-
-    /* memmove d_names down, there's a possibility of overlap
-     * equivaent to dircache_size - entry_count*sizeof(struct dircache_entry) */
-    ptrdiff_t size_to_move = d_names_end - d_names_start;
-    memmove(dst, d_names_start, size_to_move);
-    
-    /* fix up pointers to the d_names */
-    for(unsigned i = 0; i < entry_count; i++)
-        dircache_root[i].d_name -= offset;
-
-    d_names_start -= offset;
-    d_names_end -= offset;
-    dot -= offset;
-    dotdot -= offset;
-    
-    /* equivalent to dircache_size + DIRCACHE_RESERVE + align */
-    allocated_size = (d_names_end - buf);
-    reserve_used = 0;
-
-    core_shrink(dircache_handle, dircache_root, allocated_size);
-    return res;
-fail:
-    dircache_disable();
-    return res;
-}
-
-/**
- * Main initialization function that must be called before any other
- * operations within the dircache.
- */
-void dircache_init(void)
-{
-    int i;
-    int thread_id __attribute__((unused));
-    
-    dircache_initialized = false;
-    dircache_initializing = false;
-    
-    memset(opendirs, 0, sizeof(opendirs));
-    for (i = 0; i < MAX_OPEN_DIRS; i++)
-    {
-        opendirs[i].theent.d_name = opendir_dnames[i];
-    }
-    
-    queue_init(&dircache_queue, true);
-    thread_id = create_thread(dircache_thread, dircache_stack,
-                sizeof(dircache_stack), 0, dircache_thread_name
-                IF_PRIO(, PRIORITY_BACKGROUND)
-                IF_COP(, CPU));
-#ifdef HAVE_IO_PRIORITY
-    thread_set_io_priority(thread_id,IO_PRIORITY_BACKGROUND);
-#endif
+    if (started)
+        queue_post(&dircache_queue, DCM_BUILD, (intptr_t)rcp);
 
+    return started;
 }
 
 /**
- * Returns true if dircache has been initialized and is ready to be used.
+ * wait for the dircache thread to finish building; intended for waiting for a
+ * non-transparent build to finish when dircache_resume() returns > 0
  */
-bool dircache_is_enabled(void)
+void dircache_wait(void)
 {
-    return dircache_initialized;
+    thread_wait(dircache_runinfo.thread_id);
 }
 
 /**
- * Returns true if dircache is being initialized.
+ * call after mounting a volume or all volumes
  */
-bool dircache_is_initializing(void)
+void dircache_mount(void)
 {
-    return dircache_initializing || thread_enabled;
+    /* call with writer exclusion */
+    if (dircache_runinfo.suspended)
+        return;
+
+    dircache_thread_post(NULL);
 }
 
 /**
- * Set application flags used to determine if dircache is still intact.
+ * call after unmounting a volume; specifying < 0 for all or >= 0 for the
+ * specific one
  */
-void dircache_set_appflag(long mask)
+void dircache_unmount(IF_MV_NONVOID(int volume))
 {
-    appflags |= mask;
+    /* call with writer exclusion */
+    if (dircache_runinfo.suspended)
+        return;
+
+#ifdef HAVE_MULTIVOLUME
+    if (volume >= 0)
+        reset_volume(volume);
+    else
+#endif /* HAVE_MULTIVOLUME */
+        reset_cache();
+}
+
+/* backend to dircache_suspend() and dircache_disable() */
+static void dircache_suspend_internal(bool freeit)
+{
+    if (dircache_runinfo.suspended++ > 0 &&
+        (!freeit || dircache_runinfo.handle <= 0))
+        return;
+
+    unsigned int thread_id = dircache_runinfo.thread_id;
+
+    reset_cache();
+    clear_dircache_queue();
+
+    /* grab the buffer away into our control; the cache won't need it now */
+    int handle = 0;
+    if (freeit)
+        handle = reset_buffer();
+
+    dircache_unlock();
+
+    if (handle > 0)
+        core_free(handle);
+
+    thread_wait(thread_id);
+
+    dircache_lock();
+}
+
+/* backend to dircache_resume() and dircache_enable() */
+static int dircache_resume_internal(bool build_now)
+{
+    int volatile rc = 0;
+
+    if (dircache_runinfo.suspended == 0 || --dircache_runinfo.suspended == 0)
+        rc = build_now && dircache_runinfo.enabled ? 1 : 0;
+
+    if (rc)
+        rc = dircache_thread_post(&rc) ? INT_MIN : -1;
+
+    if (rc == INT_MIN)
+    {
+        dircache_unlock();
+
+        while (rc == INT_MIN) /* poll for response */
+            sleep(0);
+
+        dircache_lock();
+    }
+
+    return rc < 0 ? rc * 10 - 2 : rc;
 }
 
 /**
- * Get application flags used to determine if dircache is still intact.
+ * service to dircache_enable() and dircache_load(); "build_now" starts a build
+ * immediately if the cache was not enabled
  */
-bool dircache_get_appflag(long mask)
+static int dircache_enable_internal(bool build_now)
 {
-    return dircache_is_enabled() && (appflags & mask);
+    int rc = 0;
+
+    if (!dircache_runinfo.enabled)
+    {
+        dircache_runinfo.enabled = true;
+        rc = dircache_resume_internal(build_now);
+    }
+
+    return rc;
 }
 
 /**
- * Returns the current number of entries (directories and files) in the cache.
+ * service to dircache_disable()
  */
-int dircache_get_entry_count(void)
+static void dircache_disable_internal(void)
 {
-    return entry_count;
+    if (dircache_runinfo.enabled)
+    {
+        dircache_runinfo.enabled = false;
+        dircache_suspend_internal(true);
+    }
 }
 
 /**
- * Returns the allocated space for dircache (without reserve space).
+ * disables dircache without freeing the buffer (so it can be re-enabled
+ * afterwards with dircache_resume(); usually called when accepting an USB
+ * connection
  */
-int dircache_get_cache_size(void)
+void dircache_suspend(void)
 {
-    return dircache_is_enabled() ? dircache_size : 0;
+    dircache_lock();
+    dircache_suspend_internal(false);
+    dircache_unlock();
 }
 
 /**
- * Returns how many bytes of the reserve allocation for live cache
- * updates have been used.
+ * re-enables the dircache if previously suspended by dircache_suspend
+ * or dircache_steal_buffer(), re-using the already allocated buffer if
+ * available
+ *
+ * returns: 0 if the background build is started or dircache is still
+ *          suspended
+ *          > 0 if the build is non-background
+ *          < 0 upon failure
  */
-int dircache_get_reserve_used(void)
+int dircache_resume(void)
 {
-    return dircache_is_enabled() ? reserve_used : 0;
+    dircache_lock();
+    int rc = dircache_resume_internal(true);
+    dircache_unlock();
+    return rc;
 }
 
 /**
- * Returns the time in kernel ticks that took to build the cache.
+ * as dircache_resume() but globally enables it; called by settings and init
  */
-int dircache_get_build_ticks(void)
+int dircache_enable(void)
 {
-    return dircache_is_enabled() ? cache_build_ticks : 0;
+    dircache_lock();
+    int rc = dircache_enable_internal(true);
+    dircache_unlock();
+    return rc;
 }
 
 /**
- * Disables dircache without freeing the buffer (so it can be re-enabled
- * afterwards with dircache_resume() or dircache_build()), usually
- * called when accepting an usb connection */
-void dircache_suspend(void)
+ * as dircache_suspend() but also frees the buffer; usually called on shutdown
+ * or when deactivated
+ */
+void dircache_disable(void)
 {
-    int i;
-    bool cache_in_use;
-    
-    if (thread_enabled)
-        queue_post(&dircache_queue, DIRCACHE_STOP, 0);
-    
-    while (thread_enabled)
-        sleep(1);
-    dircache_initialized = false;
-
-    logf("Waiting for cached dirs to release");
-    do {
-        cache_in_use = false;
-        for (i = 0; i < MAX_OPEN_DIRS; i++) {
-            if (!opendirs[i].regulardir && opendirs[i].busy)
-            {
-                cache_in_use = true;
-                sleep(1);
-                break ;
-            }
-        }
-    } while (cache_in_use) ;
-    
-    logf("Cache released");
-    entry_count = 0;
+    dircache_lock();
+    dircache_disable_internal();
+    dircache_unlock();
 }
 
 /**
- * Re-enables the dircache if previous suspended by dircache_suspend
- * or dircache_steal_buffer(), re-using the already allocated buffer
- *
- * Returns true if the background build is started, false otherwise
- * (e.g. if no buffer was previously allocated)
+ * have dircache give up its allocation; call dircache_resume() to restart it
  */
-bool dircache_resume(void)
+void dircache_free_buffer(void)
 {
-    bool ret = allocated_size > 0;
-    if (ret) /* only resume if already allocated */
-        ret = (dircache_build(0) > 0);
+    dircache_lock();
+    dircache_suspend_internal(true);
+    dircache_unlock();
+}
+
+
+/** Dircache live updating **/
 
-    return (allocated_size > 0);
+/**
+ * obtain binding information for the file's root volume; this is the starting
+ * point for internal path parsing and binding
+ */
+void dircache_get_rootinfo(struct file_base_info *infop)
+{
+    int volume = BASEINFO_VOL(infop);
+    struct dircache_volume *dcvolp = DCVOL(volume);
+
+    if (dcvolp->serialnum)
+    {
+        /* root has a binding */
+        infop->dcfile.idx       = -volume - 1;
+        infop->dcfile.serialnum = dcvolp->serialnum;
+    }
+    else
+    {
+        /* root is idle */
+        dircache_dcfile_init(&infop->dcfile);
+    }
 }
 
 /**
- * Disables the dircache entirely. Usually called on shutdown or when
- * deactivated
+ * called by file code when the first reference to a file or directory is
+ * opened
  */
-void dircache_disable(void)
+void dircache_bind_file(struct file_base_binding *bindp)
+{
+    /* requires write exclusion */
+    logf("dc open: %u", (unsigned int)bindp->info.dcfile.serialnum);
+    binding_open(bindp);
+}
+
+/**
+ * called by file code when the last reference to a file or directory is
+ * closed
+ */
+void dircache_unbind_file(struct file_base_binding *bindp)
 {
-    dircache_suspend();
-    dircache_free();
+    /* requires write exclusion */
+    logf("dc close: %u", (unsigned int)bindp->info.dcfile.serialnum);
+    binding_close(bindp);
 }
 
 /**
- * Steal the allocated dircache buffer and disable dircache.
+ * called by file code when a file is newly created
  */
-void* dircache_steal_buffer(size_t *size)
+void dircache_fileop_create(struct file_base_info *dirinfop,
+                            struct file_base_binding *bindp,
+                            const char *basename,
+                            const struct dirinfo_native *dinp)
 {
-    dircache_suspend();
-    if (dircache_size == 0)
+    /* requires write exclusion */
+    logf("dc create: %u \"%s\"",
+         (unsigned int)bindp->info.dcfile.serialnum, basename);
+
+    if (!dirinfop->dcfile.serialnum)
     {
-        *size = 0;
-        return NULL;
+        /* no parent binding => no child binding */
+        return;
     }
 
-    /* since we give up the buffer (without freeing), it must not move anymore */
-    dont_move = true;
-    *size = dircache_size + (DIRCACHE_RESERVE-reserve_used);
-    
-    return dircache_root;
+    struct dircache_entry *ce;
+    int idx = create_entry(basename, &ce);
+    if (idx == 0)
+    {
+        /* failed allocation; parent cache contents are not complete */
+        establish_frontier(dirinfop->dcfile.idx, FRONTIER_ZONED);
+        return;
+    }
+
+    struct file_base_info *infop = &bindp->info;
+
+#ifdef DIRCACHE_NATIVE
+    ce->firstcluster = infop->fatfile.firstcluster;
+    ce->direntry     = infop->fatfile.e.entry;
+    ce->direntries   = infop->fatfile.e.entries;
+    ce->wrtdate      = dinp->wrtdate;
+    ce->wrttime      = dinp->wrttime;
+#else
+    ce->mtime        = dinp->mtime;
+#endif
+    ce->attr         = dinp->attr;
+    if (!(dinp->attr & ATTR_DIRECTORY))
+        ce->filesize = dinp->size;
+
+    insert_file_entry(dirinfop, ce);
+
+    /* file binding will have been queued when it was opened; just resolve */
+    infop->dcfile.idx       = idx;
+    infop->dcfile.serialnum = ce->serialnum;
+    binding_resolve(infop);
+
+    if ((dinp->attr & ATTR_DIRECTORY) && !is_dotdir_name(basename))
+    {
+        /* scan-in the contents of the new directory at this level only */
+        buffer_lock();
+        sab_process_dir(infop, false);
+        buffer_unlock();
+    }
 }
 
 /**
- * Usermode function to return dircache_entry index to the given path.
+ * called by file code when a file or directory is removed
  */
-static int dircache_get_entry_id_ex(const char *filename, bool go_down)
+void dircache_fileop_remove(struct file_base_binding *bindp)
 {
-    if (!dircache_initialized || filename == NULL)
-        return -1;
-    
-    struct dircache_entry* res = dircache_get_entry(filename, go_down);
-    return res ? res - dircache_root : -1;
-}
+    /* requires write exclusion */
+    logf("dc remove: %u\n", (unsigned int)bindp->info.dcfile.serialnum);
 
-int dircache_get_entry_id(const char* filename)
-{
-    return dircache_get_entry_id_ex(filename, false);
+    if (!bindp->info.dcfile.serialnum)
+        return; /* no binding yet */
+
+    free_file_entry(&bindp->info);
+
+    /* if binding was resolved; it should now be queued via above call */
 }
 
 /**
- * Internal: Get the startcluster for the index
+ * called by file code when a file is renamed
  */
-long _dircache_get_entry_startcluster(int id)
+void dircache_fileop_rename(struct file_base_info *dirinfop,
+                            struct file_base_binding *bindp,
+                            const char *basename)
 {
-    return get_entry(id)->startcluster;
+    /* requires write exclusion */
+    logf("dc rename: %u \"%s\"",
+         (unsigned int)bindp->info.dcfile.serialnum, basename);
+
+    if (!dirinfop->dcfile.serialnum)
+    {
+        /* new parent directory not cached; there is nowhere to put it so
+           nuke it */
+        if (bindp->info.dcfile.serialnum)
+            free_file_entry(&bindp->info);
+        /* else no entry anyway */
+
+        return;
+    }
+
+    if (!bindp->info.dcfile.serialnum)
+    {
+        /* binding not resolved on the old file but it's going into a resolved
+           parent which means the parent would be missing an entry in the cache;
+           downgrade the parent */
+        establish_frontier(dirinfop->dcfile.idx, FRONTIER_ZONED);
+        return;
+    }
+
+    /* unlink the entry but keep it; it needs to be re-sorted since the
+       underlying FS probably changed the order */
+    struct dircache_entry *ce = remove_file_entry(&bindp->info);
+
+#ifdef DIRCACHE_NATIVE
+    /* update other name-related information before inserting */
+    ce->direntry   = bindp->info.fatfile.e.entry;
+    ce->direntries = bindp->info.fatfile.e.entries;
+#endif
+
+    /* place it into its new home */
+    insert_file_entry(dirinfop, ce);
+
+    /* lastly, update the entry name itself */
+    if (entry_reassign_name(ce, basename))
+    {
+        /* it's not really the same one now so re-stamp it */
+        dc_serial_t serialnum = next_serialnum();
+        ce->serialnum = serialnum;
+        bindp->info.dcfile.serialnum = serialnum;
+    }
+    else
+    {
+        /* it cannot be kept around without a valid name */
+        free_file_entry(&bindp->info);
+        establish_frontier(dirinfop->dcfile.idx, FRONTIER_ZONED);
+    }
 }
 
 /**
- * Internal: Get the struct dirinfo for the index
+ * called by file code to synchronize file entry information
  */
-struct dirinfo* _dircache_get_entry_dirinfo(int id)
+void dircache_fileop_sync(struct file_base_binding *bindp,
+                          const struct dirinfo_native *dinp)
 {
-    return &get_entry(id)->info;
+    /* requires write exclusion */
+    struct file_base_info *infop = &bindp->info;
+    logf("dc sync: %u\n", (unsigned int)infop->dcfile.serialnum);
+
+    if (!infop->dcfile.serialnum)
+        return; /* binding unresolved */
+
+    struct dircache_entry *ce = get_entry(infop->dcfile.idx);
+    if (!ce)
+    {
+        logf("  bad index %d", infop->dcfile.idx);
+        return; /* a root (should never be called for this) */
+    }
+
+#ifdef DIRCACHE_NATIVE
+    ce->firstcluster = infop->fatfile.firstcluster;
+    ce->wrtdate      = dinp->wrtdate;
+    ce->wrttime      = dinp->wrttime;
+#else
+    ce->mtime        = dinp->mtime;
+#endif
+    ce->attr         = dinp->attr;
+    if (!(dinp->attr & ATTR_DIRECTORY))
+        ce->filesize = dinp->size;
 }
 
-/*
- * build a path from an entry upto the root using recursion
- *
- * it appends '/' after strlcat, therefore buf[0] needs to be prepared with '/'
- * and it will leave a trailing '/'
- *
- * returns the position of that trailing '/' so it can be deleted afterwards
- * (or, in case of truncation, the position of the nul byte */
-static size_t copy_path_helper(const struct dircache_entry *entry, char *buf, size_t size)
+
+/** Dircache paths and files **/
+
+#ifdef DIRCACHE_DUMPSTER
+/* helper for dircache_get_path() */
+static ssize_t get_path_sub(int idx, char *buf, size_t size)
 {
-    int offset = 1;
-    /* has parent? */
-    if (entry->up)
-        offset += copy_path_helper(entry->up, buf, size);
+    if (idx == 0)
+        return -2; /* entry is an orphan split from any root */
 
-    size_t len = strlcpy(buf+offset, entry->d_name, size - offset) + offset;
-    if (len < size)
+    ssize_t offset;
+    char *cename;
+
+    if (idx > 0)
     {
-        buf[len++] = '/';
-        buf[len]   = '\0';
+        /* go all the way up then move back down from the root */
+        struct dircache_entry *ce = get_entry(idx);
+        offset = get_path_sub(ce->up, buf, size) - 1;
+        if (offset < 0)
+            return -3;
+
+        cename = alloca(MAX_NAME + 1);
+        entry_name_copy(cename, ce);
     }
-    return len-1;
+    else /* idx < 0 */
+    {
+        offset = 0;
+        cename = "";
+
+    #ifdef HAVE_MULTIVOLUME
+        int volume = IF_MV_VOL(-idx - 1);
+        if (volume > 0)
+        {
+            /* prepend the volume specifier for volumes > 0 */
+            cename = alloca(VOL_MAX_LEN+1);
+            get_volume_name(volume, cename);
+        }
+    #endif /* HAVE_MULTIVOLUME */
+    }
+
+    return offset + path_append(buf + offset, PA_SEP_HARD, cename,
+                                size > (size_t)offset ? size - offset : 0);
 }
+#endif /* DIRCACHE_DUMPSTER */
+
+#if 0
+
 /**
- * Function to copy the full absolute path from dircache to the given buffer
- * using the given dircache_entry pointer.
- *
- * Returns the size of the resulting string, or 0 if an error occured
+ * retrieve and validate the file's entry/binding serial number
+ * the dircache file's serial number must match the indexed entry's or the
+ * file reference is stale
  */
-size_t dircache_copy_path(int index, char *buf, size_t size)
+static dc_serial_t get_file_serialnum(const struct dircache_file *dcfilep)
 {
-    if (!size || !buf || index < 0)
+    int idx = dcfilep->idx;
+
+    if (idx == 0 || idx < -NUM_VOLUMES)
         return 0;
 
-    buf[0] = '/';
-    size_t res = copy_path_helper(&dircache_root[index], buf, size - 1);
-    /* fixup trailing '/' */
-    buf[res] = '\0';
-    return res;
+    dc_serial_t serialnum;
+
+    if (idx > 0)
+    {
+        struct dircache_entry *ce = get_entry(idx);
+        serialnum = ce ? ce->serialnum : 0;
+    }
+    else
+    {
+        serialnum = get_idx_dcvolp(idx)->serialnum;
+    }
+
+    return serialnum == dcfilep->serialnum ? serialnum : 0;
 }
 
-/* --- Directory cache live updating functions --- */
-static int block_until_ready(void)
+/**
+ * usermode function to construct a full absolute path from dircache into the
+ * given buffer given the dircache file info
+ *
+ * returns:
+ *   success - the length of the string, not including the trailing null
+ *   failure - a negative value
+ *
+ * successful return value is as strlcpy()
+ *
+ * errors:
+ *   ENOENT - the file or directory does not exist
+ */
+ssize_t dircache_get_path(const struct dircache_file *dcfilep, char *buf,
+                          size_t size)
 {
-    /* Block until dircache has been built. */
-    while (!dircache_initialized && dircache_is_initializing())
-        sleep(1);
-    
-    if (!dircache_initialized)
-        return -1;
-    
-    return 0;
+    /* if missing buffer space, still return what's needed a la strlcpy */
+    if (!buf)
+        size = 0;
+    else if (size)
+        *buf = '\0';
+
+    ssize_t len = -1;
+
+    dircache_lock();
+
+    /* first and foremost, there must be a cache and the serial number must
+       check out */
+    if (dircache_runinfo.handle && get_file_serialnum(dcfilep))
+        len = get_path_sub(dcfilep->idx, buf, size);
+
+    if (len < 0)
+        errno = ENOENT;
+
+    dircache_unlock();
+    return len;
 }
 
-static struct dircache_entry* dircache_new_entry(const char *path, int attribute)
-{
-    struct dircache_entry *entry;
-    char basedir[MAX_PATH*2];
-    char *new;
-    long last_cache_size = dircache_size;
+/**
+ * searches the sublist starting at 'idx' for the named component
+ */
 
-    strlcpy(basedir, path, sizeof(basedir));
-    new = strrchr(basedir, '/');
-    if (new == NULL)
+/* helper for get_file_sub() */
+static struct dircache_entry *
+get_file_sub_scan(int idx, const char *name, size_t length, int *idxp)
+{
+    struct dircache_entry *ce = get_entry(idx);
+    if (ce)
     {
-        logf("error occurred");
-        dircache_initialized = false;
-        return NULL;
+        char entname[MAX_NAME+1];
+        name = strmemdupa(name, length);
+
+        do
+        {
+            entry_name_copy(entname, ce);
+            if (!strcasecmp(entname, name))
+            {
+                *idxp = idx;
+                break;
+            }
+
+            idx = ce->next;
+        }
+        while ((ce = get_entry(idx)));
     }
 
-    *new = '\0';
-    new++;
+    return ce;
+}
+
+/**
+ * searches for the subcomponent of *pathp
+ */
+
+/* helper for dircache_get_file() */
+static int get_file_sub(const char **pathp, int *downp, int *idxp)
+{
+    int rc;
+    const char *name;
+    rc = parse_path_component(pathp, &name, false);
+    if (rc <= 0)
+        return rc;
+    else if (rc >= MAX_PATH)
+        return ENAMETOOLONG; /* that's just unpossible, man */
+
+    struct dircache_entry *ce = get_file_sub_scan(*downp, name, rc, idxp);
 
-    entry = dircache_get_entry(basedir, true);
-    if (entry == NULL)
+    if (!ce)
+        rc = RC_NOT_FOUND;   /* not there; tellibry solly */
+    else if (!*pathp)
+        rc = RC_PATH_ENDED;  /* done */
+    else if (!(ce->attr & ATTR_DIRECTORY))
+        rc = ENOTDIR;        /* a parent component must be a directory */
+    else
+        while ((rc = get_file_sub(pathp, &ce->down, idxp)) == RC_CONTINUE);
+
+    switch (rc)
     {
-        logf("basedir not found!");
-        logf("%s", basedir);
-        dircache_initialized = false;
-        return NULL;
+    case RC_GO_UP:           /* hit ".."; drop to previous level */
+        return RC_CONTINUE;
+    case RC_PATH_ENDED:      /* success! */
+        return RC_FOUND;
+    default:                 /* component not found or error */
+        return rc;
     }
+}
 
-    if (reserve_used + 2*sizeof(struct dircache_entry) + strlen(new)+1
-        >= DIRCACHE_RESERVE)
+/**
+ * usermode function to return dircache file info for the given path
+ *
+ * returns:
+ *   success: the volume number that is specified for the file
+ *   failure: a negative value
+ *
+ * errors:
+ *   ENOENT       - the file or directory does not exist or path is empty
+ *   ENAMETOOLONG - a component of the path is too long
+ *   ENOTDIR      - a component of the path is not a directory
+ */
+int dircache_get_file(const char *path, struct dircache_file *dcfilep)
+{
+    if (!path_is_absolute(path) || !dcfilep)
     {
-        logf("not enough space");
-        dircache_initialized = false;
-        return NULL;
+        errno = ENOENT;
+        return -1;
     }
-    
-    while (entry->next != NULL)
-        entry = entry->next;
 
-    if (entry->d_name != NULL)
+    dircache_lock();
+
+    if (!dircache_runinfo.handle)
     {
-        entry = dircache_gen_next(entry);
-        if (entry == NULL)
+        dircache_unlock();
+        errno = ENOENT;
+        return -2;
+    }
+
+    int volume = 0;
+    int idx = 0;
+    dc_serial_t serialnum = 0;
+    struct dircache_volume *dcvolp = NULL;
+    struct dircache_entry *ce = NULL;
+
+    int rc = RC_GO_UP;
+
+    while (rc == RC_CONTINUE || rc == RC_GO_UP)
+    {
+    #ifdef HAVE_MULTIVOLUME
+        if (rc == RC_GO_UP)
         {
-            dircache_initialized = false;
-            return NULL;
+            volume = path_strip_volume(path, &path, false);
+            if (!CHECK_VOL(volume))
+            {
+                rc = ENXIO;
+                break;
+            }
         }
-    }
+    #endif /* HAVE_MULTIVOLUME */
+
+        dcvolp = DCVOL(volume);
 
-    size_t size = strlen(new) + 1;
-    entry->d_name = (d_names_start -= size);
-    entry->startcluster = 0;
-    memset(&entry->info, 0, sizeof(entry->info));
-    entry->info.attribute = attribute;
+        int *downp = &dcvolp->root_down;
+        if (*downp <= 0)
+        {
+            rc = ENXIO;
+            break;
+        }
 
-    strcpy(entry->d_name, new);
-    dircache_size += size;
+        rc = get_file_sub(&path, downp, &idx);
+    }
 
-    if (attribute & ATTR_DIRECTORY)
+    switch (rc)
     {
-        logf("gen_down");
-        dircache_gen_down(entry);
+    case RC_FOUND:      /* hit: component found */
+        serialnum = ce->serialnum;
+        rc = volume;
+        break;
+    case RC_PATH_ENDED: /* hit: it's a root (volume or system) */
+        idx = -volume - 1;
+        serialnum = dcvolp->serialnum;
+        rc = volume;
+        break;
+    case RC_NOT_FOUND:  /* miss */
+        rc = ENOENT;
+    default:
+        idx = 0;
+        errno = rc;
+        rc = -3;
+        break;
     }
-        
-    reserve_used += dircache_size - last_cache_size;
 
-    return entry;
+    dcfilep->idx       = idx;
+    dcfilep->serialnum = serialnum;
+
+    dircache_unlock();
+    return rc;
 }
+#endif /* 0 */
+
 
-void dircache_bind(int fd, const char *path)
+/** Debug screen/info stuff **/
+
+/**
+ * return cache state parameters
+ */
+void dircache_get_info(struct dircache_info *info)
 {
-    struct dircache_entry *entry;
-    
-    /* Queue requests until dircache has been built. */
-    if (!dircache_initialized && dircache_is_initializing())
+    static const char * const status_descriptions[] =
+    {
+        [DIRCACHE_IDLE]     = "Idle",
+        [DIRCACHE_SCANNING] = "Scanning",
+        [DIRCACHE_READY]    = "Ready",
+    };
+
+    if (!info)
+        return;
+
+    memset(info, 0, sizeof (*info));
+
+    dircache_lock();
+
+    enum dircache_status status = DIRCACHE_IDLE;
+
+    for (unsigned int volume = 0; volume < NUM_VOLUMES; volume++)
     {
-        if (fdbind_idx >= MAX_PENDING_BINDINGS)
-            return ;
-        strlcpy(fdbind_cache[fdbind_idx].path, path, 
-                sizeof(fdbind_cache[fdbind_idx].path));
-        fdbind_cache[fdbind_idx].fd = fd;
-        fdbind_idx++;
-        return ;
+        struct dircache_volume *dcvolp = DCVOL(volume);
+        enum dircache_status volstatus = dcvolp->status;
+
+        switch (volstatus)
+        {
+        case DIRCACHE_SCANNING:
+            /* if any one is scanning then overall status is "scanning" */
+            status = volstatus;
+
+            /* sum the time the scanning has taken so far */
+            info->build_ticks += current_tick - dcvolp->start_tick;
+            break;
+        case DIRCACHE_READY:
+            /* if all the rest are idle and at least one is ready, then
+               status is "ready". */
+            if (status == DIRCACHE_IDLE)
+                status = DIRCACHE_READY;
+
+            /* sum the build ticks of all "ready" volumes */
+            info->build_ticks += dcvolp->build_ticks;
+            break;
+        case DIRCACHE_IDLE:
+            /* if all are idle; then the whole cache is "idle" */
+            break;
+        }
     }
-    
-    if (!dircache_initialized)
-        return ;
 
-    logf("bind: %d/%s", fd, path);
-    entry = dircache_get_entry(path, false);
-    if (entry == NULL)
+    info->status     = status;
+    info->statusdesc = status_descriptions[status];
+    info->last_size  = dircache.last_size;
+    info->size_limit = DIRCACHE_LIMIT;
+    info->reserve    = DIRCACHE_RESERVE;
+
+    /* report usage only if there is something ready or being built */
+    if (status != DIRCACHE_IDLE)
     {
-        logf("not found!");
-        dircache_initialized = false;
-        return ;
+        info->reserve_used = reserve_buf_used();
+        info->size         = dircache.size;
+        info->sizeused     = dircache.sizeused;
+        info->entry_count  = dircache.numentries;
     }
 
-    fd_bindings[fd] = entry;
+    dircache_unlock();
 }
 
-void dircache_update_filesize(int fd, long newsize, long startcluster)
+#ifdef DIRCACHE_DUMPSTER
+/**
+ * dump RAW binary of buffer and CSV of all valid paths and volumes to disk
+ */
+void dircache_dump(void)
 {
-    if (!dircache_initialized || fd < 0)
-        return ;
-
-    if (fd_bindings[fd] == NULL)
+    /* open both now so they're in the cache */
+    int fdbin = open(DIRCACHE_DUMPSTER_BIN, O_WRONLY|O_CREAT|O_TRUNC, 0666);
+    int fdcsv = open(DIRCACHE_DUMPSTER_CSV, O_WRONLY|O_CREAT|O_TRUNC, 0666);
+    if (fdbin < 0 || fdcsv < 0)
     {
-        logf("dircache fd(%d) access error", fd);
-        dircache_initialized = false;
-        return ;
+        close(fdbin);
+        return;
     }
-    
-    fd_bindings[fd]->info.size = newsize;
-    fd_bindings[fd]->startcluster = startcluster;
-}
-void dircache_update_filetime(int fd)
-{
-#if CONFIG_RTC == 0
-    (void)fd;
-#else
-    short year;
-    struct tm *now = get_time();
-    if (!dircache_initialized || fd < 0)
-        return ;
-
-    if (fd_bindings[fd] == NULL)
-    {
-        logf("dircache fd access error");
-        dircache_initialized = false;
-        return ;
-    }
-    year = now->tm_year+1900-1980;
-    fd_bindings[fd]->info.wrtdate = (((year)&0x7f)<<9)           |
-                                    (((now->tm_mon+1)&0xf)<<5)   |
-                                    (((now->tm_mday)&0x1f));
-    fd_bindings[fd]->info.wrttime = (((now->tm_hour)&0x1f)<<11)  |
-                                    (((now->tm_min)&0x3f)<<5)    |
-                                    (((now->tm_sec/2)&0x1f));
-#endif
-}
 
-void dircache_mkdir(const char *path)
-{ /* Test ok. */
-    if (block_until_ready())
-        return ;
-        
-        
-    logf("mkdir: %s", path);
-    dircache_new_entry(path, ATTR_DIRECTORY);
-}
-
-void dircache_rmdir(const char *path)
-{ /* Test ok. */
-    struct dircache_entry *entry;
-    
-    if (block_until_ready())
-        return ;
-        
-    logf("rmdir: %s", path);
-    entry = dircache_get_entry(path, false);
-    if (entry == NULL || entry->down == NULL)
-    {
-        logf("not found or not a directory!");
-        dircache_initialized = false;
-        return ;
-    }
-
-    entry->down = NULL;
-    entry->d_name = NULL;
-}
-
-/* Remove a file from cache */
-void dircache_remove(const char *name)
-{ /* Test ok. */
-    struct dircache_entry *entry;
-    
-    if (block_until_ready())
-        return ;
-        
-    logf("remove: %s", name);
-    
-    entry = dircache_get_entry(name, false);
-
-    if (entry == NULL)
-    {
-        logf("not found!");
-        dircache_initialized = false;
-        return ;
-    }
-    
-    entry->d_name = NULL;
-}
-
-void dircache_rename(const char *oldpath, const char *newpath)
-{ /* Test ok. */
-    struct dircache_entry *entry, *newentry;
-    struct dircache_entry oldentry;
-    char absolute_path[MAX_PATH*2];
-    char *p;
-    
-    if (block_until_ready())
-        return ;
-        
-    logf("rename: %s->%s", oldpath, newpath);
-    
-    entry = dircache_get_entry(oldpath, false);
-    if (entry == NULL)
-    {
-        logf("not found!");
-        dircache_initialized = false;
-        return ;
-    }
-
-    /* Delete the old entry. */
-    entry->d_name = NULL;
-
-    /** If we rename the same filename twice in a row, we need to
-     * save the data, because the entry will be re-used. */
-    oldentry = *entry;
-
-    /* Generate the absolute path for destination if necessary. */
-    if (newpath[0] != '/')
-    {
-        strlcpy(absolute_path, oldpath, sizeof(absolute_path));
-        p = strrchr(absolute_path, '/');
-        if (!p)
+    trigger_cpu_boost();
+    dircache_lock();
+
+    if (dircache_runinfo.handle)
+    {
+        buffer_lock();
+
+        /* bin */
+        write(fdbin, dircache_runinfo.p + ENTRYSIZE,
+              dircache_runinfo.bufsize + 1);
+
+        /* CSV */
+        fdprintf(fdcsv, "\"Index\",\"Serialnum\","
+                        "\"Path\",\"Frontier\","
+                        "\"Attribute\",\"File Size\","
+                        "\"Mod Date\",\"Mod Time\"\n");
+
+        FOR_EACH_VOLUME(-1, volume)
         {
-            logf("Invalid path");
-            dircache_initialized = false;
-            return ;
+            struct dircache_volume *dcvolp = DCVOL(volume);
+            if (dcvolp->status == DIRCACHE_IDLE)
+                continue;
+
+        #ifdef HAVE_MULTIVOLUME
+            char name[VOL_MAX_LEN+1];
+            get_volume_name(volume, name);
+        #endif
+            fdprintf(fdcsv,
+                     "%d,%lu,"
+                     "\"%c" IF_MV("%s") "\",%u,"
+                     "0x%08X,0,"
+                     "\"\",\"\"\n",
+                     -volume-1, dcvolp->serialnum,
+                     PATH_SEPCH, IF_MV(name,) dcvolp->frontier,
+                     ATTR_DIRECTORY | ATTR_VOLUME);
         }
-        
-        *p = '\0';
-        strlcpy(p, absolute_path, sizeof(absolute_path)-strlen(p));
-        newpath = absolute_path;
-    }
-    
-    newentry = dircache_new_entry(newpath, entry->info.attribute);
-    if (newentry == NULL)
-    {
-        dircache_initialized = false;
-        return ;
+
+        FOR_EACH_CACHE_ENTRY(ce)
+        {
+        #ifdef DIRCACHE_NATIVE
+            time_t mtime = fattime_mktime(ce->wrtdate, ce->wrttime);
+        #else
+            time_t mtime = ce->mtime;
+        #endif
+            struct tm tm;
+            gmtime_r(&mtime, &tm);
+
+            char buf[DC_MAX_NAME + 2];
+            *buf = '\0';
+            int idx = get_index(ce);
+            get_path_sub(idx, buf, sizeof (buf));
+
+            fdprintf(fdcsv,
+                     "%d,%lu,"
+                     "\"%s\",%u,"
+                     "0x%08X,%lu,"
+                     "%04d/%02d/%02d,"
+                     "%02d:%02d:%02d\n",
+                     idx, ce->serialnum,
+                     buf, ce->frontier,
+                     ce->attr, (ce->attr & ATTR_DIRECTORY) ?
+                                0ul : (unsigned long)ce->filesize,
+                     tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+                     tm.tm_hour, tm.tm_min, tm.tm_sec);
+        }
+
+        buffer_unlock();
     }
 
-    newentry->down = oldentry.down;
-    newentry->startcluster = oldentry.startcluster;
-    newentry->info.size    = oldentry.info.size;
-    newentry->info.wrtdate = oldentry.info.wrtdate;
-    newentry->info.wrttime = oldentry.info.wrttime;
+    dircache_unlock();
+    cancel_cpu_boost();
+
+    close(fdbin);
+    close(fdcsv);
 }
+#endif /* DIRCACHE_DUMPSTER */
+
 
-void dircache_add_file(const char *path, long startcluster)
+/** Misc. stuff **/
+
+/**
+ * set the dircache file to initial values
+ */
+void dircache_dcfile_init(struct dircache_file *dcfilep)
 {
-    struct dircache_entry *entry;
-    
-    if (block_until_ready())
-        return ;
-    
-    logf("add file: %s", path);
-    entry = dircache_new_entry(path, 0);
-    if (entry == NULL)
-        return ;
-    
-    entry->startcluster = startcluster;
+    dcfilep->idx       = 0;
+    dcfilep->serialnum = 0;
 }
 
-static bool is_disable_msg_pending(void)
+#ifdef HAVE_EEPROM_SETTINGS
+
+#ifdef HAVE_HOTSWAP
+/* NOTE: This is hazardous to the filesystem of any sort of removable
+         storage unless it may be determined that the filesystem from save
+         to load is identical. If it's not possible to do so in a timely
+         manner, it's not worth persisting the cache. */
+  #warning "Don't do this; you'll find the consequences unpleasant."
+#endif
+
+/* dircache persistence file header magic */
+#define DIRCACHE_MAGIC  0x00d0c0a1
+
+/* dircache persistence file header */
+struct dircache_maindata
 {
-    return check_event_queue();
+    uint32_t        magic;      /* DIRCACHE_MAGIC */
+    struct dircache dircache;   /* metadata of the cache! */
+    uint32_t        datacrc;    /* CRC32 of data */
+    uint32_t        hdrcrc;     /* CRC32 of header through datacrc */
+} __attribute__((packed, aligned (4)));
+
+/**
+ * verify that the clean status is A-ok
+ */
+static bool dircache_is_clean(bool saving)
+{
+    if (saving)
+        return dircache.dcvol[0].status == DIRCACHE_READY;
+    else
+    {
+        return dircache.dcvol[0].status == DIRCACHE_IDLE &&
+               !dircache_runinfo.enabled;
+    }
 }
 
-DIR_CACHED* opendir_cached(const char* name)
+/**
+ * function to load the internal cache structure from disk to initialize
+ * the dircache really fast with little disk access.
+ */
+int dircache_load(void)
 {
-    int dd;
-    DIR_CACHED* pdir = opendirs;
+    logf("Loading directory cache");
+    int fd = open_dircache_file(O_RDONLY);
+    if (fd < 0)
+        return -1;
+
+    int rc = -1;
+
+    ssize_t size;
+    struct dircache_maindata maindata;
+    uint32_t crc;
+    int handle = 0;
+    bool hasbuffer = false;
 
-    if ( name[0] != '/' )
+    size = sizeof (maindata);
+    if (read(fd, &maindata, size) != size)
     {
-        DEBUGF("Only absolute paths supported right now\n");
-        return NULL;
+        logf("dircache: header read failed");
+        goto error_nolock;
     }
 
-    /* find a free dir descriptor */
-    for ( dd=0; dd<MAX_OPEN_DIRS; dd++, pdir++)
-        if ( !pdir->busy )
-            break;
+    /* sanity check the header */
+    if (maindata.magic != DIRCACHE_MAGIC)
+    {
+        logf("dircache: invalid header magic");
+        goto error_nolock;
+    }
 
-    if ( dd == MAX_OPEN_DIRS )
+    crc = crc_32(&maindata, offsetof(struct dircache_maindata, hdrcrc),
+                 0xffffffff);
+    if (crc != maindata.hdrcrc)
     {
-        DEBUGF("Too many dirs open\n");
-        errno = EMFILE;
-        return NULL;
+        logf("dircache: invalid header CRC32");
+        goto error_nolock;
     }
-    
-    pdir->busy = true;
 
-    if (!dircache_initialized || is_disable_msg_pending())
+    if (maindata.dircache.size !=
+            maindata.dircache.sizeentries + maindata.dircache.sizenames ||
+        ALIGN_DOWN(maindata.dircache.size, ENTRYSIZE) != maindata.dircache.size ||
+        filesize(fd) - sizeof (maindata) != maindata.dircache.size)
     {
-        pdir->internal_entry = -1;
-        pdir->regulardir = opendir_uncached(name);   
+        logf("dircache: file header error");
+        goto error_nolock;
     }
-    else
+
+    /* allocate so that exactly the reserve size remains */
+    size_t bufsize = maindata.dircache.size + DIRCACHE_RESERVE + 1;
+    handle = alloc_cache(bufsize);
+    if (handle <= 0)
+    {
+        logf("dircache: failed load allocation");
+        goto error_nolock;
+    }
+
+    dircache_lock();
+    buffer_lock();
+
+    if (!dircache_is_clean(false))
+        goto error;
+
+    /* from this point on, we're actually dealing with the cache in RAM */
+    dircache = maindata.dircache;
+
+    set_buffer(handle, bufsize);
+    hasbuffer = true;
+
+    /* convert back to in-RAM representation */
+    dircache.numentries = maindata.dircache.sizeentries / ENTRYSIZE;
+
+    /* read the dircache file into memory; start with the entries */
+    size = maindata.dircache.sizeentries;
+    if (read(fd, dircache_runinfo.pentry + 1, size) != size)
     {
-        pdir->regulardir = NULL;
-        pdir->internal_entry = dircache_get_entry_id_ex(name, true);
-        pdir->theent.info.attribute = -1; /* used to make readdir_cached aware of the first call */
+        logf("dircache read failed #1");
+        goto error;
     }
 
-    if (pdir->internal_entry == -1 && pdir->regulardir == NULL)
+    crc = crc_32(dircache_runinfo.pentry + 1, size, 0xffffffff);
+
+    /* continue with the names; fix up indexes to them if needed */
+    dircache.names -= maindata.dircache.sizenames;
+    *get_name(dircache.names - 1) = 0;
+
+    size = maindata.dircache.sizenames;
+    if (read(fd, get_name(dircache.names), size) != size)
     {
-        pdir->busy = false;
-        return NULL;
+        logf("dircache read failed #2");
+        goto error;
     }
 
-    return pdir;
-}
+    crc = crc_32(get_name(dircache.names), size, crc);
+    if (crc != maindata.datacrc)
+    {
+        logf("dircache: data failed CRC32");
+        goto error;
+    }
 
-struct dirent_cached* readdir_cached(DIR_CACHED* dir)
-{
-    struct dircache_entry *ce = get_entry(dir->internal_entry);
-    struct dirent_uncached *regentry;
-    
-    if (!dir->busy)
-        return NULL;
+    /* only names will be changed in relative position so fix up those
+       references */
+    ssize_t offset = dircache.names - maindata.dircache.names;
+    if (offset != 0)
+    {
+        /* nothing should be open besides the dircache file itself therefore
+           no bindings need be resolved; the cache will have its own entry
+           but that should get cleaned up when removing the file */
+        FOR_EACH_CACHE_ENTRY(ce)
+        {
+            if (!ce->tinyname)
+                ce->name += offset;
+        }
+    }
 
-    if (dir->regulardir != NULL)
-    {
-        regentry = readdir_uncached(dir->regulardir);
-        if (regentry == NULL)
-            return NULL;
-
-        strlcpy(dir->theent.d_name, regentry->d_name, MAX_PATH);
-        dir->theent.startcluster = regentry->startcluster;
-        dir->theent.info = regentry->info;
-        
-        return &dir->theent;
-    }
-    
-    /* if theent.attribute=-1 then this is the first call */
-    /* otherwise, this is is not so we first take the entry's ->next */
-    /* NOTE: normal file can't have attribute=-1 */
-    if(dir->theent.info.attribute != -1)
-        ce = ce->next;
-    /* skip unused entries */
-    while(ce != NULL && ce->d_name == NULL)
-        ce = ce->next;
-    
-    if (ce == NULL)
-            return NULL;
+    dircache.reserve_used = 0;
+
+    /* enable the cache but do not try to build it */
+    dircache_enable_internal(false);
+
+    /* cache successfully loaded */
+    logf("Done, %ld KiB used", dircache.size / 1024);
+    rc = 0;
+error:
+    if (rc < 0 && hasbuffer)
+        reset_buffer();
 
-    strlcpy(dir->theent.d_name, ce->d_name, MAX_PATH);
-    /* Can't do `dir->theent = *ce`
-       because that modifies the d_name pointer. */
-    dir->theent.startcluster = ce->startcluster;
-    dir->theent.info = ce->info;
-    dir->internal_entry = ce - dircache_root;
+    buffer_unlock();
+    dircache_unlock();
 
-    //logf("-> %s", ce->d_name);
-    return &dir->theent;
+error_nolock:
+    if (rc < 0 && handle > 0)
+        core_free(handle);
+
+    if (fd >= 0)
+        close(fd);
+
+    remove_dircache_file();
+    return rc;
 }
 
-int closedir_cached(DIR_CACHED* dir)
+/**
+ * function to save the internal cache stucture to disk for fast loading
+ * on boot
+ */
+int dircache_save(void)
 {
-    if (!dir->busy)
+    logf("Saving directory cache");
+
+    int fd = open_dircache_file(O_WRONLY|O_CREAT|O_TRUNC|O_APPEND);
+    if (fd < 0)
         return -1;
-        
-    dir->busy=false;
-    if (dir->regulardir != NULL)
-        return closedir_uncached(dir->regulardir);
-    
-    return 0;
-}
 
-int mkdir_cached(const char *name)
-{
-    int rc=mkdir_uncached(name);
-    if (rc >= 0)
-        dircache_mkdir(name);
-    return(rc);
+    dircache_lock();
+    buffer_lock();
+
+    int rc = -1;
+
+    if (!dircache_is_clean(true))
+        goto error;
+
+    /* save the header structure along with the cache metadata */
+    ssize_t size;
+    uint32_t crc;
+    struct dircache_maindata maindata =
+    {
+        .magic    = DIRCACHE_MAGIC,
+        .dircache = dircache,
+    };
+
+    /* store the size since it better detects an invalid header */
+    maindata.dircache.sizeentries = maindata.dircache.numentries * ENTRYSIZE;
+
+    /* write the template header */
+    size = sizeof (maindata);
+    if (write(fd, &maindata, size) != size)
+    {
+        logf("dircache: write failed #1");
+        goto error;
+    }
+
+    /* write the dircache entries */
+    size = maindata.dircache.sizeentries;
+    if (write(fd, dircache_runinfo.pentry + 1, size) != size)
+    {
+        logf("dircache: write failed #2");
+        goto error;
+    }
+
+    crc = crc_32(dircache_runinfo.pentry + 1, size, 0xffffffff);
+
+    /* continue with the names */
+    size = maindata.dircache.sizenames;
+    if (write(fd, get_name(dircache.names), size) != size)
+    {
+        logf("dircache: write failed #3");
+        goto error;
+    }
+
+    crc = crc_32(get_name(dircache.names), size, crc);
+    maindata.datacrc = crc;
+
+    /* rewrite the header with CRC info */
+    if (lseek(fd, 0, SEEK_SET) != 0)
+    {
+        logf("dircache: seek failed");
+        goto error;
+    }
+
+    crc = crc_32(&maindata, offsetof(struct dircache_maindata, hdrcrc),
+                 0xffffffff);
+    maindata.hdrcrc = crc;
+
+    if (write(fd, &maindata, sizeof (maindata)) != sizeof (maindata))
+    {
+        logf("dircache: write failed #4");
+        goto error;
+    }
+
+    /* as of now, no changes to the volumes should be allowed at all since
+       that makes what was saved completely invalid */
+    rc = 0;
+error:
+    buffer_unlock();
+    dircache_unlock();
+
+    if (rc < 0)
+        remove_dircache_file();
+
+    close(fd);
+    return rc;
 }
+#endif /* HAVE_EEPROM_SETTINGS */
 
-int rmdir_cached(const char* name)
+/**
+ * main one-time initialization function that must be called before any other
+ * operations within the dircache
+ */
+void dircache_init(size_t last_size)
 {
-    int rc=rmdir_uncached(name);
-    if(rc >= 0)
-        dircache_rmdir(name);
-    return(rc);
+    queue_init(&dircache_queue, false);
+
+    dircache.last_size = MIN(last_size, DIRCACHE_LIMIT);
+
+    struct dircache_runinfo *dcrip = &dircache_runinfo;
+    dcrip->suspended         = 1;
+    dcrip->thread_done       = true;
+    dcrip->ops.move_callback = move_callback;
 }