Do some debug and preparatory work for ramcache and playlist

The file system rework introduced incompatibility between dircache
and the tagcache ramcache and playlist dircache path caching. This
update makes changes to filesystem code to reintegrate all that.

It also fixes a couple bugs that were found when vetting all the
code. The filestream cache was being reset without regard to
the stream even if it was shared in write mode (made work of
.playlist_control). Better handling of unmounting gives files a
better go at force-closing them without risk to disk integrity.

Did some miscellaneous pedantic changes. Improved efficiency of
testing a file's existence (a little) since the path parser will
be shared between file code and parsing for the sake of finding
dircache references, not duplicated as before.

This commit doesn't reenable said items just for the sake of
keeping changes separate and related.

Plan for the next is to enable dircache again for the playlists
(easy peasy) and reenable tagcache ramcache but *without* the
dircache path caching because it's rather substantial to change
in itself. The ramcache will still function without dircache.

Change-Id: I7e2a9910b866251fa8333e1275f72fcfc8425d2d

1 files changed, 178 insertions, 173 deletions

diff --git a/firmware/common/dircache.c b/firmware/common/dircache.c
index b93ee73fc6..a3538ff96f 100644
--- a/firmware/common/dircache.c
+++ b/firmware/common/dircache.c
@@ -41,9 +41,7 @@
 #include "audio.h"
 #include "rbpaths.h"
 #include "linked_list.h"
-#ifdef HAVE_EEPROM_SETTINGS
 #include "crc32.h"
-#endif
 
 /**
  * Cache memory layout:
@@ -1457,7 +1455,7 @@ int dircache_readdir_dirent(struct filestr_base *stream,
     unsigned int direntry = scanp->fatscan.entry;
     while (1)
     {
-        if (idx == 0 || direntry == FAT_RW_VAL) /* rewound? */
+        if (idx == 0 || direntry == FAT_DIRSCAN_RW_VAL) /* rewound? */
         {
             idx = diridx <= 0 ? dcvolp->root_down : get_entry(diridx)->down;
             break;
@@ -1487,7 +1485,7 @@ int dircache_readdir_dirent(struct filestr_base *stream,
             return 0; /* end of dir */
         }
 
-        if (ce->direntry > direntry || direntry == FAT_RW_VAL)
+        if (ce->direntry > direntry || direntry == FAT_DIRSCAN_RW_VAL)
             break; /* cache reader is caught up to FS scan */
 
         idx = ce->next;
@@ -1549,7 +1547,12 @@ int dircache_readdir_internal(struct filestr_base *stream,
 
     /* is parent cached? if not, readthrough because nothing is here yet */
     if (!dirinfop->dcfile.serialnum)
+    {
+        if (stream->flags & FF_CACHEONLY)
+            goto read_eod;
+
         return uncached_readdir_internal(stream, infop, fatent);
+    }
 
     int diridx = dirinfop->dcfile.idx;
     unsigned int frontier = diridx < 0 ?
@@ -1562,7 +1565,8 @@ int dircache_readdir_internal(struct filestr_base *stream,
         idx = get_entry(idx)->next;
 
     struct dircache_entry *ce = get_entry(idx);
-    if (frontier != FRONTIER_SETTLED)
+
+    if (frontier != FRONTIER_SETTLED && !(stream->flags & FF_CACHEONLY))
     {
         /* the directory being read is reported to be incompletely cached;
            readthrough and if the entry exists, return it with its binding
@@ -1577,9 +1581,7 @@ int dircache_readdir_internal(struct filestr_base *stream,
     else if (!ce)
     {
         /* end of dir */
-        fat_empty_fat_direntry(fatent);
-        infop->fatfile.e.entries = 0;
-        return 0;
+        goto read_eod;
     }
 
     /* FS entry information that we maintain */
@@ -1612,6 +1614,11 @@ int dircache_readdir_internal(struct filestr_base *stream,
     }
 
     return rc;
+
+read_eod:
+    fat_empty_fat_direntry(fatent);
+    infop->fatfile.e.entries = 0;
+    return 0;    
 }
 
 /**
@@ -2451,30 +2458,41 @@ void dircache_fileop_sync(struct file_base_binding *bindp,
 
 /** Dircache paths and files **/
 
-#ifdef DIRCACHE_DUMPSTER
-/* helper for dircache_get_path() */
-static ssize_t get_path_sub(int idx, char *buf, size_t size)
+/**
+ * helper for returning a path and serial hash represented by an index
+ */
+struct get_path_sub_data
+{
+    char        *buf;
+    size_t      size;
+    dc_serial_t serialhash;
+};
+
+static ssize_t get_path_sub(int idx, struct get_path_sub_data *data)
 {
     if (idx == 0)
-        return -2; /* entry is an orphan split from any root */
+        return -1; /* entry is an orphan split from any root */
 
-    ssize_t offset;
+    ssize_t len;
     char *cename;
 
     if (idx > 0)
     {
-        /* go all the way up then move back down from the root */
         struct dircache_entry *ce = get_entry(idx);
-        offset = get_path_sub(ce->up, buf, size) - 1;
-        if (offset < 0)
-            return -3;
+
+        data->serialhash = dc_hash_serialnum(ce->serialnum, data->serialhash);
+
+        /* go all the way up then move back down from the root */
+        len = get_path_sub(ce->up, data) - 1;
+        if (len < 0)
+            return -2;
 
         cename = alloca(MAX_NAME + 1);
         entry_name_copy(cename, ce);
     }
     else /* idx < 0 */
     {
-        offset = 0;
+        len = 0;
         cename = "";
 
     #ifdef HAVE_MULTIVOLUME
@@ -2486,14 +2504,14 @@ static ssize_t get_path_sub(int idx, char *buf, size_t size)
             get_volume_name(volume, cename);
         }
     #endif /* HAVE_MULTIVOLUME */
+
+        data->serialhash = dc_hash_serialnum(get_idx_dcvolp(idx)->serialnum,
+                                             data->serialhash);
     }
 
-    return offset + path_append(buf + offset, PA_SEP_HARD, cename,
-                                size > (size_t)offset ? size - offset : 0);
+    return len + path_append(data->buf + len, PA_SEP_HARD, cename,
+                             data->size > (size_t)len ? data->size - len : 0);
 }
-#endif /* DIRCACHE_DUMPSTER */
-
-#if 0
 
 /**
  * retrieve and validate the file's entry/binding serial number
@@ -2523,201 +2541,173 @@ static dc_serial_t get_file_serialnum(const struct dircache_file *dcfilep)
 }
 
 /**
+ * Obtain the hash of the serial numbers of the canonical path, index to root
+ */
+static dc_serial_t get_file_serialhash(const struct dircache_file *dcfilep)
+{
+    int idx = dcfilep->idx;
+
+    dc_serial_t h = DC_SERHASH_START;
+
+    while (idx > 0)
+    {
+        struct dircache_entry *ce = get_entry(idx);
+        h = dc_hash_serialnum(ce->serialnum, h);
+        idx = ce->up;
+    }
+
+    h = dc_hash_serialnum(get_idx_dcvolp(idx)->serialnum, h);
+
+    return h;
+}
+
+/**
+ * Initialize the fileref
+ */
+void dircache_fileref_init(struct dircache_fileref *dcfrefp)
+{
+    dircache_dcfile_init(&dcfrefp->dcfile);
+    dcfrefp->serialhash = DC_SERHASH_START;
+}
+
+/**
  * usermode function to construct a full absolute path from dircache into the
  * given buffer given the dircache file info
  *
  * returns:
- *   success - the length of the string, not including the trailing null
+ *   success - the length of the string, not including the trailing null or the
+ *             buffer length required if the buffer is too small (return is >=
+ *             size)
  *   failure - a negative value
  *
- * successful return value is as strlcpy()
- *
  * errors:
- *   ENOENT - the file or directory does not exist
+ *   ENOENT - No such file or directory
  */
-ssize_t dircache_get_path(const struct dircache_file *dcfilep, char *buf,
-                          size_t size)
+ssize_t dircache_get_fileref_path(const struct dircache_fileref *dcfrefp, char *buf,
+                                  size_t size)
 {
+    ssize_t rc;
+
     /* if missing buffer space, still return what's needed a la strlcpy */
     if (!buf)
         size = 0;
     else if (size)
         *buf = '\0';
 
-    ssize_t len = -1;
-
     dircache_lock();
 
     /* first and foremost, there must be a cache and the serial number must
        check out */
-    if (dircache_runinfo.handle && get_file_serialnum(dcfilep))
-        len = get_path_sub(dcfilep->idx, buf, size);
-
-    if (len < 0)
-        errno = ENOENT;
-
-    dircache_unlock();
-    return len;
-}
+    if (!dircache_runinfo.handle)
+        FILE_ERROR(ENOENT, -1);
 
-/**
- * searches the sublist starting at 'idx' for the named component
- */
+    if (get_file_serialnum(&dcfrefp->dcfile) == 0)
+        FILE_ERROR(ENOENT, -2);
 
-/* helper for get_file_sub() */
-static struct dircache_entry *
-get_file_sub_scan(int idx, const char *name, size_t length, int *idxp)
-{
-    struct dircache_entry *ce = get_entry(idx);
-    if (ce)
+    struct get_path_sub_data data =
     {
-        char entname[MAX_NAME+1];
-        name = strmemdupa(name, length);
-
-        do
-        {
-            entry_name_copy(entname, ce);
-            if (!strcasecmp(entname, name))
-            {
-                *idxp = idx;
-                break;
-            }
-
-            idx = ce->next;
-        }
-        while ((ce = get_entry(idx)));
-    }
-
-    return ce;
-}
-
-/**
- * searches for the subcomponent of *pathp
- */
-
-/* helper for dircache_get_file() */
-static int get_file_sub(const char **pathp, int *downp, int *idxp)
-{
-    int rc;
-    const char *name;
-    rc = parse_path_component(pathp, &name, false);
-    if (rc <= 0)
-        return rc;
-    else if (rc >= MAX_PATH)
-        return ENAMETOOLONG; /* that's just unpossible, man */
+        .buf        = buf,
+        .size       = size,
+        .serialhash = DC_SERHASH_START,
+    };
 
-    struct dircache_entry *ce = get_file_sub_scan(*downp, name, rc, idxp);
+    rc = get_path_sub(dcfrefp->dcfile.idx, &data);
+    if (rc < 0)
+        FILE_ERROR(ENOENT, rc * 10 - 3);
 
-    if (!ce)
-        rc = RC_NOT_FOUND;   /* not there; tellibry solly */
-    else if (!*pathp)
-        rc = RC_PATH_ENDED;  /* done */
-    else if (!(ce->attr & ATTR_DIRECTORY))
-        rc = ENOTDIR;        /* a parent component must be a directory */
-    else
-        while ((rc = get_file_sub(pathp, &ce->down, idxp)) == RC_CONTINUE);
+    if (data.serialhash != dcfrefp->serialhash)
+        FILE_ERROR(ENOENT, -4);
 
-    switch (rc)
-    {
-    case RC_GO_UP:           /* hit ".."; drop to previous level */
-        return RC_CONTINUE;
-    case RC_PATH_ENDED:      /* success! */
-        return RC_FOUND;
-    default:                 /* component not found or error */
-        return rc;
-    }
+file_error:
+    dircache_unlock();
+    return rc;
 }
 
 /**
- * usermode function to return dircache file info for the given path
+ * Test a path to various levels of rigor and optionally return dircache file
+ * info for the given path
  *
  * returns:
- *   success: the volume number that is specified for the file
+ *   success: 0
  *   failure: a negative value
  *
- * errors:
- *   ENOENT       - the file or directory does not exist or path is empty
- *   ENAMETOOLONG - a component of the path is too long
- *   ENOTDIR      - a component of the path is not a directory
+ * errors (including but not limited to):
+ *   EFAULT       - Bad address
+ *   EINVAL       - Invalid argument
+ *   ENAMETOOLONG - File or path name too long
+ *   ENOENT       - No such file or directory
+ *   ENOTDIR      - Not a directory
+ *   ENXIO        - No such device or address
  */
-int dircache_get_file(const char *path, struct dircache_file *dcfilep)
+int dircache_search(unsigned int flags, struct dircache_fileref *dcfrefp, const char *path)
 {
-    if (!path_is_absolute(path) || !dcfilep)
-    {
-        errno = ENOENT;
-        return -1;
-    }
+    int rc;
+
+    if (!(flags & (DCS_FILEREF | DCS_CACHED_PATH)))
+        FILE_ERROR_RETURN(EINVAL, -1); /* search nothing? */
 
     dircache_lock();
 
     if (!dircache_runinfo.handle)
+        FILE_ERROR(ENOENT, -2);
+
+    if (flags & DCS_FILEREF)
     {
-        dircache_unlock();
-        errno = ENOENT;
-        return -2;
-    }
+        if (!dcfrefp)
+            FILE_ERROR(EFAULT, -3);
 
-    int volume = 0;
-    int idx = 0;
-    dc_serial_t serialnum = 0;
-    struct dircache_volume *dcvolp = NULL;
-    struct dircache_entry *ce = NULL;
+        if (get_file_serialnum(&dcfrefp->dcfile) != 0)
+        {
+            if (!(flags & _DCS_VERIFY_FLAG))
+                goto file_success; /* no robust verification wanted */
+
+            if (get_file_serialhash(&dcfrefp->dcfile) == dcfrefp->serialhash)
+                goto file_success; /* reference is most likely still valid */
+        }
 
-    int rc = RC_GO_UP;
+        if (!(flags & DCS_CACHED_PATH))
+            FILE_ERROR(ENOENT, -4); /* no path search wanted */
+    }
 
-    while (rc == RC_CONTINUE || rc == RC_GO_UP)
+    if (flags & DCS_CACHED_PATH)
     {
-    #ifdef HAVE_MULTIVOLUME
-        if (rc == RC_GO_UP)
+        const bool update = flags & DCS_UPDATE_FILEREF;
+        struct path_component_info *compinfop = NULL;
+
+        if (update)
         {
-            volume = path_strip_volume(path, &path, false);
-            if (!CHECK_VOL(volume))
-            {
-                rc = ENXIO;
-                break;
-            }
-        }
-    #endif /* HAVE_MULTIVOLUME */
+            if (!dcfrefp)
+                FILE_ERROR(EFAULT, -5);
 
-        dcvolp = DCVOL(volume);
+            compinfop = alloca(sizeof (*compinfop));  
+        }
 
-        int *downp = &dcvolp->root_down;
-        if (*downp <= 0)
+        struct filestr_base stream;
+        rc = open_stream_internal(path, FF_ANYTYPE | FF_PROBE | FF_SELFINFO |
+                                  ((flags & _DCS_STORAGE_FLAG) ? 0 : FF_CACHEONLY),
+                                  &stream, compinfop);
+        if (rc <= 0)
         {
-            rc = ENXIO;
-            break;
+            if (update)
+                dircache_fileref_init(dcfrefp);
+  
+            FILE_ERROR(rc ? ERRNO : ENOENT, rc * 10 - 6);
         }
 
-        rc = get_file_sub(&path, downp, &idx);
-    }
-
-    switch (rc)
-    {
-    case RC_FOUND:      /* hit: component found */
-        serialnum = ce->serialnum;
-        rc = volume;
-        break;
-    case RC_PATH_ENDED: /* hit: it's a root (volume or system) */
-        idx = -volume - 1;
-        serialnum = dcvolp->serialnum;
-        rc = volume;
-        break;
-    case RC_NOT_FOUND:  /* miss */
-        rc = ENOENT;
-    default:
-        idx = 0;
-        errno = rc;
-        rc = -3;
-        break;
+        if (update)
+        {
+            dcfrefp->dcfile     = compinfop->info.dcfile;
+            dcfrefp->serialhash = get_file_serialhash(&compinfop->info.dcfile);
+        }
     }
 
-    dcfilep->idx       = idx;
-    dcfilep->serialnum = serialnum;
+file_success:
+    rc = 0;
 
+file_error:
     dircache_unlock();
-    return rc;
+    return rc;    
 }
-#endif /* 0 */
 
 
 /** Debug screen/info stuff **/
@@ -2737,13 +2727,12 @@ void dircache_get_info(struct dircache_info *info)
     if (!info)
         return;
 
-    memset(info, 0, sizeof (*info));
-
     dircache_lock();
 
     enum dircache_status status = DIRCACHE_IDLE;
+    info->build_ticks = 0;
 
-    for (unsigned int volume = 0; volume < NUM_VOLUMES; volume++)
+    FOR_EACH_VOLUME(-1, volume)
     {
         struct dircache_volume *dcvolp = DCVOL(volume);
         enum dircache_status volstatus = dcvolp->status;
@@ -2781,11 +2770,18 @@ void dircache_get_info(struct dircache_info *info)
     /* report usage only if there is something ready or being built */
     if (status != DIRCACHE_IDLE)
     {
-        info->reserve_used = reserve_buf_used();
         info->size         = dircache.size;
         info->sizeused     = dircache.sizeused;
+        info->reserve_used = reserve_buf_used();
         info->entry_count  = dircache.numentries;
     }
+    else
+    {
+        info->size         = 0;
+        info->sizeused     = 0;
+        info->reserve_used = 0;
+        info->entry_count  = 0;
+    }
 
     dircache_unlock();
 }
@@ -2817,7 +2813,7 @@ void dircache_dump(void)
               dircache_runinfo.bufsize + 1);
 
         /* CSV */
-        fdprintf(fdcsv, "\"Index\",\"Serialnum\","
+        fdprintf(fdcsv, "\"Index\",\"Serialnum\",\"Serialhash\","
                         "\"Path\",\"Frontier\","
                         "\"Attribute\",\"File Size\","
                         "\"Mod Date\",\"Mod Time\"\n");
@@ -2833,11 +2829,12 @@ void dircache_dump(void)
             get_volume_name(volume, name);
         #endif
             fdprintf(fdcsv,
-                     "%d,%lu,"
+                     "%d," DC_SERIAL_FMT "," DC_SERIAL_FMT ","
                      "\"%c" IF_MV("%s") "\",%u,"
                      "0x%08X,0,"
                      "\"\",\"\"\n",
                      -volume-1, dcvolp->serialnum,
+                         dc_hash_serialnum(dcvolp->serialnum, DC_SERHASH_START), 
                      PATH_SEPCH, IF_MV(name,) dcvolp->frontier,
                      ATTR_DIRECTORY | ATTR_VOLUME);
         }
@@ -2855,15 +2852,23 @@ void dircache_dump(void)
             char buf[DC_MAX_NAME + 2];
             *buf = '\0';
             int idx = get_index(ce);
-            get_path_sub(idx, buf, sizeof (buf));
+
+            struct get_path_sub_data data =
+            {
+                .buf        = buf,
+                .size       = sizeof (buf),
+                .serialhash = DC_SERHASH_START,
+            };
+
+            get_path_sub(idx, &data);
 
             fdprintf(fdcsv,
-                     "%d,%lu,"
+                     "%d," DC_SERIAL_FMT "," DC_SERIAL_FMT ","
                      "\"%s\",%u,"
                      "0x%08X,%lu,"
                      "%04d/%02d/%02d,"
                      "%02d:%02d:%02d\n",
-                     idx, ce->serialnum,
+                     idx, ce->serialnum, data.serialhash,
                      buf, ce->frontier,
                      ce->attr, (ce->attr & ATTR_DIRECTORY) ?
                                 0ul : (unsigned long)ce->filesize,