talk: Smarter cache management for TALK_PARTIAL_LOAD.

Previously the clip cache of TALK_PARTIAL_LOAD reserved space N clips, each slot
was as big as the maximum sized clip which was necessary to replace clips
in-memory in MRU-style.

The cache management now uses buflib to allocate and free each clip, using the
clip's real size. This allows the clip cache to be much more compact, because
no space is wasted for the max. sized clip. This makes use of buflib's ability
to easily manage differently-sized memory chunks by moving them to make free
space.

As an example: for english.voice TALK_PARTIAL_LOAD allocated 288k in advance.
for just 64 clips. With this patch ~70 clips can be stored in a 100k buffer.
This, the memory usage is cut by 2/3 and almost optimal (there's still the
buflib per-alloc cookie overhead).

As a result the TALK_PARTIAL_LOAD buffer is restricted to 100k which still
allows for more clips than previously, on average.

Change-Id: I257654071e9a95770cd6db2c2765f020befce412

1 files changed, 373 insertions, 282 deletions

diff --git a/apps/talk.c b/apps/talk.c
index 0904e41c3b..4c4e6184b8 100644
--- a/apps/talk.c
+++ b/apps/talk.c
@@ -87,6 +87,8 @@ const char* const file_thumbnail_ext = ".talk";
 #define MAX_THUMBNAIL_BUFSIZE 0x10000
 #endif
 
+#define DEFAULT_VOICE_LANG "english"
+
 /***************** Data types *****************/
 
 struct clip_entry /* one entry of the index table */
@@ -114,16 +116,7 @@ struct voicefile_header /* file format of our voice file */
  * The Ondios have slow storage access and loading the entire voice file would
  * take several seconds, so we use the same mechanism. */
 #define TALK_PARTIAL_LOAD
-#endif
-
-#ifdef TALK_PARTIAL_LOAD
-static long           max_clipsize; /* size of the biggest clip */
-static long           buffered_id[QUEUE_SIZE];  /* IDs of the talk clips */
-static uint8_t        clip_age[QUEUE_SIZE];
-#if QUEUE_SIZE > 255
-#   error clip_age[] type too small
-#endif
-static int cache_hits, cache_misses;
+#define MAX_CLIP_BUFFER_SIZE 100000 /* 70+ clips should fit into 100k */
 #endif
 
 /* Multiple thumbnails can be loaded back-to-back in this buffer. */
@@ -147,9 +140,6 @@ static struct mutex queue_mutex SHAREDBSS_ATTR;
 #endif /* CONFIG_CODEC */
 static int sent; /* how many bytes handed over to playback, owned by ISR */
 static unsigned char curr_hd[3]; /* current frame header, for re-sync */
-static int silence_offset; /* VOICE_PAUSE clip, used for termination */
-static long silence_length; /* length of the VOICE_PAUSE clip */
-static unsigned long lastclip_offset; /* address of latest clip, for silence add */
 static unsigned char last_lang[MAX_FILENAME+1]; /* name of last used lang file (in talk_init) */
 static bool talk_initialized; /* true if talk_init has been called */
 static bool give_buffer_away; /* true if we should give the buffers away in shrink_callback if requested */
@@ -161,27 +151,65 @@ static unsigned long voicefile_size;
 
 struct queue_entry /* one entry of the internal queue */
 {
-    int offset, length;
+    int offset; /* actually a buflib handle if type == HANDLE */
+    int length; /* total length of the clip */
+    int remaining; /* bytes that still need to be deoded */
+    /* for large QUEUE_SIZE values it might be worthwhile to merge the type
+     * into the bits of the above members (as a space saver). For small values
+     * the required extra code outweights this so it's not done here */
+    enum offset_type {
+        TALK_OFFSET,
+        THUMB_OFFSET,
+#ifdef TALK_PARTIAL_LOAD
+        TALK_HANDLE,
+#endif
+    } type;
 };
 
-static struct queue_entry queue[QUEUE_SIZE]; /* queue of scheduled clips */
+#ifdef TALK_PARTIAL_LOAD
+static struct buflib_context clip_ctx;
 
-#define DEFAULT_VOICE_LANG "english"
+struct clip_cache_metadata {
+    long tick;
+    int handle, voice_id;
+};
+
+static int metadata_table_handle;
+static unsigned max_clips;
+static int cache_hits, cache_misses;
+#endif
+
+static struct queue_entry queue[QUEUE_SIZE]; /* queue of scheduled clips */
+static struct queue_entry silence, *last_clip;
 
 /***************** Private implementation *****************/
 
-static int thumb_handle;
-static int talk_handle, talk_handle_locked;
+static int index_handle, talk_handle, thumb_handle;
+
+static int move_callback(int handle, void *current, void *new)
+{
+    (void)handle; (void)current; (void)new;
+#ifdef TALK_PARTIAL_LOAD
+    if (handle == talk_handle)
+        if (!buflib_context_relocate(&clip_ctx, new))
+            return BUFLIB_CB_CANNOT_MOVE;
+#endif
+    return BUFLIB_CB_OK;
+}
+
+
+static struct mutex read_buffer_mutex;
 
-#if CONFIG_CODEC != SWCODEC
 
 /* on HWCODEC only voice xor audio can be active at a time */
 static bool check_audio_status(void)
 {
+#if CONFIG_CODEC != SWCODEC
     if (audio_status()) /* busy, buffer in use */
         return false;
     /* ensure playback is given up on the buffer */
     audio_hard_stop();
+#endif
     return true;
 }
 
@@ -190,31 +218,43 @@ static bool check_audio_status(void)
 static void sync_callback(int handle, bool sync_on)
 {
     (void) handle;
-    (void) sync_on;
-#if CONFIG_CPU == SH7034
     if (sync_on)
-        CHCR3 &= ~0x0001; /* disable the DMA (and therefore the interrupt also) */
+        mutex_lock(&read_buffer_mutex);
     else
-        CHCR3 |=  0x0001; /* re-enable the DMA */
-#endif
+        mutex_unlock(&read_buffer_mutex);
 }
-#else
-#define check_audio_status() (true)
-#endif
 
-static int move_callback(int handle, void *current, void *new)
+static ssize_t read_to_handle_ex(int fd, struct buflib_context *ctx, int handle,
+                                 int handle_offset, size_t count)
 {
-    (void)handle;(void)current;(void)new;
-    if (UNLIKELY(talk_handle_locked))
-        return BUFLIB_CB_CANNOT_MOVE;
-    return BUFLIB_CB_OK;
+    unsigned char *buf;
+    ssize_t ret;
+    mutex_lock(&read_buffer_mutex);
+
+    if (!ctx)
+        buf = core_get_data(handle);
+    else
+        buf = buflib_get_data(ctx, handle);
+
+    buf += handle_offset;
+    ret = read(fd, buf, count);
+
+    mutex_unlock(&read_buffer_mutex);
+
+    return ret;
 }
 
-static int clip_shrink_callback(int handle, unsigned hints, void *start, size_t old_size)
+static ssize_t read_to_handle(int fd, int handle, int handle_offset, size_t count)
 {
-    (void)start;(void)old_size;(void)hints;
+    return read_to_handle_ex(fd, NULL, handle, handle_offset, count);
+}
+
 
-#if (!defined(TALK_PARTIAL_LOAD) || (MEMORYSIZE > 2))
+static int shrink_callback(int handle, unsigned hints, void *start, size_t old_size)
+{
+    (void)start;(void)old_size;(void)hints;
+    int *h;
+#if (MAX_CLIP_BUFFER_SIZE < (MEMORYSIZE<<20) || (MEMORYSIZE > 2))
     /* on low-mem and when the voice buffer size is not limited (i.e.
      * on 2MB HWCODEC) we effectively own the entire buffer because
      * the voicefile takes up all RAM. This blocks other Rockbox parts
@@ -222,12 +262,22 @@ static int clip_shrink_callback(int handle, unsigned hints, void *start, size_t
      * up the buffer and reload when clips are played back. On high-mem
      * or when the clip buffer is limited to a few 100K this provision is
      * not necessary. */
-    if (LIKELY(!talk_handle_locked)
-            && give_buffer_away
+    if (give_buffer_away
             && (hints & BUFLIB_SHRINK_POS_MASK) == BUFLIB_SHRINK_POS_MASK)
 #endif
     {
-        talk_handle = core_free(handle);
+        if (handle == talk_handle)
+            h = &talk_handle;
+        else //if (handle == index_handle)
+            h = &index_handle;
+
+        mutex_lock(&read_buffer_mutex);
+        /* the clip buffer isn't usable without index table */
+        if (handle == index_handle && talk_handle > 0)
+            talk_handle = core_free(talk_handle);
+        *h = core_free(handle);
+        mutex_unlock(&read_buffer_mutex);
+
         return BUFLIB_CB_OK;
     }
     return BUFLIB_CB_CANNOT_SHRINK;
@@ -237,62 +287,31 @@ static int thumb_shrink_callback(int handle, unsigned hints, void *start, size_t
 {
     (void)start;(void)old_size;(void)hints;
 
-    /* be generous about the thumbnail buffer unless currently used */
-    if (LIKELY(!talk_handle_locked) && thumbnail_buf_used == 0)
+    if (handle == thumb_handle && thumbnail_buf_used == 0)
     {
+        mutex_lock(&read_buffer_mutex);
         thumb_handle = core_free(handle);
+        mutex_unlock(&read_buffer_mutex);
         return BUFLIB_CB_OK;
     }
+
     return BUFLIB_CB_CANNOT_SHRINK;
 }
 
-static struct buflib_callbacks clip_ops = {
+
+
+static struct buflib_callbacks talk_ops = {
     .move_callback = move_callback,
-#if CONFIG_CODEC != SWCODEC
     .sync_callback = sync_callback,
-#endif
-    .shrink_callback = clip_shrink_callback,
+    .shrink_callback = shrink_callback,
 };
 
 static struct buflib_callbacks thumb_ops = {
     .move_callback = move_callback,
-#if CONFIG_CODEC != SWCODEC
     .sync_callback = sync_callback,
-#endif
     .shrink_callback = thumb_shrink_callback,
 };
 
-
-static int index_handle, index_handle_locked;
-static int index_move_callback(int handle, void *current, void *new)
-{
-    (void)handle;(void)current;(void)new;
-    if (UNLIKELY(index_handle_locked))
-        return BUFLIB_CB_CANNOT_MOVE;
-    return BUFLIB_CB_OK;
-}
-
-static int index_shrink_callback(int handle, unsigned hints, void *start, size_t old_size)
-{
-    (void)start;(void)old_size;
-    if (LIKELY(!index_handle_locked)
-            && give_buffer_away
-            && (hints & BUFLIB_SHRINK_POS_MASK) == BUFLIB_SHRINK_POS_MASK)
-    {
-        index_handle = core_free(handle);
-        /* the clip buffer isn't usable without index table */
-        if (LIKELY(!talk_handle_locked))
-            talk_handle = core_free(talk_handle);
-        return BUFLIB_CB_OK;
-    }
-    return BUFLIB_CB_CANNOT_SHRINK;
-}
-
-static struct buflib_callbacks index_ops = {
-    .move_callback = index_move_callback,
-    .shrink_callback = index_shrink_callback,
-};
-
 static int open_voicefile(void)
 {
     char buf[64];
@@ -310,114 +329,143 @@ static int open_voicefile(void)
 }
 
 
-/* fetch a clip from the voice file */
-static int get_clip(long id, long* p_size)
+static int id2index(int id)
 {
-    int retval = -1;
-    struct clip_entry* clipbuf;
-    size_t clipsize;
-
+    int index = id;
     if (id > VOICEONLY_DELIMITER)
     {   /* voice-only entries use the second part of the table.
            The first string comes after VOICEONLY_DELIMITER so we need to
            substract VOICEONLY_DELIMITER + 1 */
-        id -= VOICEONLY_DELIMITER + 1;
-        if (id >= voicefile.id2_max)
+        index -= VOICEONLY_DELIMITER + 1;
+        if (index >= voicefile.id2_max)
             return -1; /* must be newer than we have */
-        id += voicefile.id1_max; /* table 2 is behind table 1 */
+        index += voicefile.id1_max; /* table 2 is behind table 1 */
     }
     else
     {   /* normal use of the first table */
         if (id >= voicefile.id1_max)
             return -1; /* must be newer than we have */
     }
+    return index;
+}
+
+#ifdef TALK_PARTIAL_LOAD
+static int free_oldest_clip(void)
+{
+    unsigned i;
+    int oldest = 0;
+    long age, now;
+    struct clip_entry* clipbuf;
+    struct clip_cache_metadata *cc = buflib_get_data(&clip_ctx, metadata_table_handle);
+    for(age = i = 0, now = current_tick; i < max_clips; i++)
+    {
+        if (cc[i].handle && (now - cc[i].tick) > age
+                && cc[i].voice_id != VOICE_PAUSE) /* never consider silence */
+        {
+            age = now - cc[i].tick;
+            oldest = i;
+        }
+    }
+    cc = &cc[oldest];
+    cc->handle = buflib_free(&clip_ctx, cc->handle);
+    /* need to clear the LOADED bit too */
+    clipbuf = core_get_data(index_handle);
+    clipbuf[id2index(cc->voice_id)].size &= ~LOADED_MASK;
+
+    return oldest;
+}
+#endif
+/* fetch a clip from the voice file */
+static int get_clip(long id, struct queue_entry *q)
+{
+    int index;
+    int retval = -1;
+    struct clip_entry* clipbuf;
+    size_t clipsize;
+    enum offset_type type;
+
+    index = id2index(id);
+    if (index == -1)
+        return -1;
 
     clipbuf = core_get_data(index_handle);
-    clipsize = clipbuf[id].size;
+    clipsize = clipbuf[index].size;
     if (clipsize == 0) /* clip not included in voicefile */
         return -1;
 
 #ifndef TALK_PARTIAL_LOAD
-    retval = clipbuf[id].offset;
+    retval = clipbuf[index].offset;
+    type   = TALK_OFFSET;
 
 #else
     if (!(clipsize & LOADED_MASK))
     {   /* clip needs loading */
+        struct clip_cache_metadata *cc;
+        int fd, handle, oldest = -1;
+        unsigned i;
         ssize_t ret;
-        int fd, idx = 0;
-        unsigned char *voicebuf;
         cache_misses++;
-        if (id == VOICE_PAUSE) {
-            idx = QUEUE_SIZE;   /* we keep VOICE_PAUSE loaded */
-        } else {
-            int oldest = 0, i;
-            for(i=0; i<QUEUE_SIZE; i++) {
-                if (buffered_id[i] < 0) {
-                    /* found a free entry, that means the buffer isn't
-                     * full yet. */
-                    idx = i;
-                    break;
-                }
-
-                /* find the oldest clip */
-                if(clip_age[i] > oldest) {
-                    idx = i;
-                    oldest = clip_age[i];
-                }
-
-                /* increment age of each loaded clip */
-                clip_age[i]++;
-            }
-            clip_age[idx] = 0; /* reset clip's age */
-        }
-        retval = idx * max_clipsize;
+        /* free clips from cache until this one succeeds to allocate */
+        while ((handle = buflib_alloc(&clip_ctx, clipsize)) < 0)
+            oldest = free_oldest_clip();
+        /* handle should now hold a valid alloc. Load from disk
+         * and insert into cache */
         fd = open_voicefile();
         if (fd < 0)
+        {
+            buflib_free(&clip_ctx, handle);
             return -1; /* open error */
-
-        talk_handle_locked++;
-        voicebuf = core_get_data(talk_handle);
+        }
         clipbuf = core_get_data(index_handle);
-        lseek(fd, clipbuf[id].offset, SEEK_SET);
-        ret = read(fd, &voicebuf[retval], clipsize);
+        lseek(fd, clipbuf[index].offset, SEEK_SET);
+        ret = read_to_handle_ex(fd, &clip_ctx, handle, 0, clipsize);
         close(fd);
-        talk_handle_locked--;
 
         if (ret < 0 || clipsize != (size_t)ret)
+        {
+            buflib_free(&clip_ctx, handle);
             return -1; /* read error */
+        }
 
         clipbuf = core_get_data(index_handle);
-        clipbuf[id].size |= LOADED_MASK; /* mark as loaded */
+        clipbuf[index].size |= LOADED_MASK; /* mark as loaded */
 
-        if (id != VOICE_PAUSE) {
-            if (buffered_id[idx] >= 0) {
-                /* mark previously loaded clip as unloaded */
-                clipbuf[buffered_id[idx]].size &= ~LOADED_MASK;
-            }
-            buffered_id[idx] = id;
+        /* finally insert into metadata table */
+        cc = buflib_get_data(&clip_ctx, metadata_table_handle);
+        if (oldest != -1)
+            /* went through the cache in the above loop already, re-use the slot */
+            cc = &cc[oldest];
+        else
+        {   /* find an empty slot */
+            for(i = 0; cc[i].handle && i < max_clips; i++) ;
+            if (i == max_clips) /* no free slot in the cache table? */
+                i = free_oldest_clip();
+            cc = &cc[i];
         }
+        cc->handle = handle;
+        cc->tick = current_tick;
+        cc->voice_id = id;
+        retval = handle;
     }
     else
-    {   /* clip is in memory already */
-        /* Find where it was loaded */
+    {   /* clip is in memory already; find where it was loaded */
         cache_hits++;
-        if (id == VOICE_PAUSE) {
-            retval = QUEUE_SIZE * max_clipsize;
-        } else {
-            int idx;
-            for (idx=0; idx<QUEUE_SIZE; idx++)
-                if (buffered_id[idx] == id) {
-                    retval = idx * max_clipsize;
-                    clip_age[idx] = 0; /* reset clip's age */
-                    break;
-                }
-        }
+        struct clip_cache_metadata *cc;
+        static int i;
+        cc = buflib_get_data(&clip_ctx, metadata_table_handle);
+        for (i = 0; cc[i].voice_id != id || !cc[i].handle; i++) ;
+        cc[i].tick = current_tick; /* reset age */
         clipsize &= ~LOADED_MASK; /* without the extra bit gives true size */
+        retval = cc[i].handle;
     }
+    type = TALK_HANDLE;
 #endif /* TALK_PARTIAL_LOAD */
 
-    *p_size = clipsize;
-    return retval;
+    q->offset    = retval;
+    q->length    = clipsize;
+    q->remaining = clipsize;
+    q->type      = type;
+    return 0;
 }
 
 static bool load_index_table(int fd, const struct voicefile_header *hdr)
@@ -429,13 +477,11 @@ static bool load_index_table(int fd, const struct voicefile_header *hdr)
         return true;
 
     ssize_t alloc_size = (hdr->id1_max + hdr->id2_max) * sizeof(struct clip_entry);
-    index_handle = core_alloc_ex("voice index", alloc_size, &index_ops);
+    index_handle = core_alloc_ex("voice index", alloc_size, &talk_ops);
     if (index_handle < 0)
         return false;
 
-    index_handle_locked++;
-    buf = core_get_data(index_handle);
-    ret = read(fd, buf, alloc_size);
+    ret = read_to_handle(fd, index_handle, 0, alloc_size);
 
 #ifndef TALK_PARTIAL_LOAD
     int clips_offset, num_clips;
@@ -447,19 +493,20 @@ static bool load_index_table(int fd, const struct voicefile_header *hdr)
     clips_offset += num_clips * sizeof(struct clip_entry); /* skip index */
 #endif
     if (ret == alloc_size)
+    {
+        buf = core_get_data(index_handle);
         for (int i = 0; i < hdr->id1_max + hdr->id2_max; i++)
         {
 #ifdef ROCKBOX_LITTLE_ENDIAN
+            /* doesn't yield() */
             structec_convert(&buf[i], "ll", 1, true);
 #endif
 #ifndef TALK_PARTIAL_LOAD
             buf[i].offset -= clips_offset;
 #endif
         }
-
-    index_handle_locked--;
-
-    if (ret != alloc_size)
+    }
+    else
         index_handle = core_free(index_handle);
 
     return ret == alloc_size;
@@ -481,20 +528,16 @@ static bool load_header(int fd, struct voicefile_header *hdr)
 #ifndef TALK_PARTIAL_LOAD
 static bool load_data(int fd, ssize_t size_to_read)
 {
-    unsigned char *buf;
     ssize_t ret;
 
     if (size_to_read < 0)
         return false;
 
-    talk_handle = core_alloc_ex("voice data", size_to_read, &clip_ops);
+    talk_handle = core_alloc_ex("voice data", size_to_read, &talk_ops);
     if (talk_handle < 0)
         return false;
 
-    talk_handle_locked++;
-    buf = core_get_data(talk_handle);
-    ret = read(fd, buf, size_to_read);
-    talk_handle_locked--;
+    ret = read_to_handle(fd, talk_handle, 0, size_to_read);
 
     if (ret != size_to_read)
         talk_handle = core_free(talk_handle);
@@ -558,23 +601,33 @@ static bool load_voicefile_data(int fd, size_t max_size)
 {
 #ifdef TALK_PARTIAL_LOAD
     (void)fd;
-    /* just allocate, populate on an as-needed basis later */
-    talk_handle = core_alloc_ex("voice data", max_size, &clip_ops);
+    size_t alloc_size;
+    /* just allocate, populate on an as-needed basis later
+     * re-create the clip buffer to ensure clip_ctx is up-to-date */
+    if (talk_handle > 0)
+        talk_handle = core_free(talk_handle);
+    talk_handle = core_alloc_ex("voice data", max_size, &talk_ops);
     if (talk_handle < 0)
         goto load_err_free;
+
+    buflib_init(&clip_ctx, core_get_data(talk_handle), max_size);
+
+    alloc_size = max_clips * sizeof(struct clip_cache_metadata);
+    /* the first alloc is the clip metadata table */
+    metadata_table_handle = buflib_alloc(&clip_ctx, alloc_size);
+    memset(buflib_get_data(&clip_ctx, metadata_table_handle), 0, alloc_size);
+
 #else
-    size_t load_size, clips_size;
     /* load the entire file into memory */
-    clips_size = (voicefile.id1_max+voicefile.id2_max) * sizeof(struct clip_entry);
-    load_size  = max_size - voicefile.table - clips_size;
-    if (!load_data(fd, load_size))
+    if (!load_data(fd, max_size))
         goto load_err_free;
 #endif
 
     /* make sure to have the silence clip, if available
      * return value can be cached globally even for TALK_PARTIAL_LOAD because
      * the VOICE_PAUSE clip is specially handled */
-    silence_offset = get_clip(VOICE_PAUSE, &silence_length);
+    if (get_clip(VOICE_PAUSE, &silence))
+        goto load_err_free;
 
     /* not an error if this fails here, might try again when the
      * actual thumbnails are attempted to be played back */
@@ -587,74 +640,94 @@ load_err_free:
     return false;
 }
 
+/* most, if not all, clips should be well below 32k (largest in english.lang is
+ * 4.5K). Currently there is a problem with voice decoding such that clips
+ * cannot be decoded in chunks. Once that is resolved this buffer could be
+ * smaller and clips be decoded in multiple chunks */
+static unsigned char commit_buffer[32<<10];
+
+static void* commit_transfer(struct queue_entry *qe, size_t *size)
+{
+    void *buf = NULL; /* shut up gcc */
+    static unsigned char *bufpos = commit_buffer;
+    int offset = qe->offset;
+#if CONFIG_CODEC != SWCODEC
+    sent = MIN(qe->remaining, 0xFFFF);
+#else
+    sent = qe->remaining;
+#endif
+    sent = MIN((size_t)sent, sizeof(commit_buffer));
+    switch (qe->type)
+    {
+        case TALK_OFFSET:  buf = core_get_data(talk_handle) + offset; break;
+        case THUMB_OFFSET: buf = core_get_data(thumb_handle) + offset; break;
+#ifdef TALK_PARTIAL_LOAD
+        case TALK_HANDLE:  buf = buflib_get_data(&clip_ctx, offset); break;
+#endif
+    }
+    /* adjust buffer position to what has been played already */
+    buf += (qe->length - qe->remaining);
+    memcpy(bufpos, buf, sent);
+    *size = sent;
+
+
+    return commit_buffer;
+}
+
+static inline bool is_silence(struct queue_entry *qe)
+{
+    if (silence.length > 0) /* silence clip available? */
+        return (qe->offset == silence.offset && qe->type == silence.type);
+    else
+        return false;
+}
+
 /* called in ISR context (on HWCODEC) if mp3 data got consumed */
 static void mp3_callback(const void** start, size_t* size)
 {
-    queue[queue_read].length -= sent; /* we completed this */
-    queue[queue_read].offset += sent;
+    struct queue_entry *qe = &queue[queue_read];
+    qe->remaining -= sent; /* we completed this */
 
-    if (queue[queue_read].length > 0) /* current clip not finished? */
+    if (qe->remaining > 0) /* current clip not finished? */
     {   /* feed the next 64K-1 chunk */
-        int offset;
-#if CONFIG_CODEC != SWCODEC
-        sent = MIN(queue[queue_read].length, 0xFFFF);
-#else
-        sent = queue[queue_read].length;
-#endif
-        offset = queue[queue_read].offset;
-        if ((unsigned long)offset >= voicefile_size)
-            *start = core_get_data(thumb_handle) + offset - voicefile_size;
-        else
-            *start = core_get_data(talk_handle) + offset;
-        *size = sent;
+        *start = commit_transfer(qe, size);
         return;
     }
+
     talk_queue_lock();
-    if(thumb_handle && (unsigned long)queue[queue_read].offset == voicefile_size+thumbnail_buf_used)
-        thumbnail_buf_used = 0;
-    if (sent > 0) /* go to next entry */
+    /* check if thumbnails have been played */
+    if (qe->type == THUMB_OFFSET)
     {
-        queue_read = (queue_read + 1) & QUEUE_MASK;
+        if (qe->remaining == 0 && (qe->length + qe->offset) == thumbnail_buf_used)
+            thumbnail_buf_used = 0;
     }
 
-re_check:
+    /* increment read position for the just played clip */
+    queue_read = (queue_read + 1) & QUEUE_MASK;
 
-    if (QUEUE_LEVEL != 0) /* queue is not empty? */
-    {   /* start next clip */
-        unsigned char *buf;
-#if CONFIG_CODEC != SWCODEC
-        sent = MIN(queue[queue_read].length, 0xFFFF);
-#else
-        sent = queue[queue_read].length;
-#endif
-        lastclip_offset = queue[queue_read].offset;
-        /* offsets larger than voicefile_size denote thumbnail clips */
-        if (lastclip_offset >= voicefile_size)
-            buf = core_get_data(thumb_handle) + lastclip_offset - voicefile_size;
+    if (QUEUE_LEVEL == 0)
+    {
+        if (!is_silence(last_clip) && last_clip->type != THUMB_OFFSET)
+        {   /* add silence clip when queue runs empty playing a voice clip,
+             * only if the previous clip wasn't silence or thumbnail */
+            queue[queue_write] = silence;
+            queue_write = (queue_write + 1) & QUEUE_MASK;
+        }
         else
-            buf = core_get_data(talk_handle) + lastclip_offset;
-        *start = buf;
-        *size = sent;
-        curr_hd[0] = buf[1];
-        curr_hd[1] = buf[2];
-        curr_hd[2] = buf[3];
+        {
+            *size = 0; /* end of data */
+        }
     }
-    else if (silence_offset > 0               /* silence clip available */
-             && lastclip_offset != (unsigned long)silence_offset  /* previous clip wasn't silence */
-             && !(lastclip_offset >= voicefile_size   /* ..or thumbnail */
-                 && lastclip_offset < voicefile_size +size_for_thumbnail))
-    {   /* add silence clip when queue runs empty playing a voice clip */
-        queue[queue_write].offset = silence_offset;
-        queue[queue_write].length = silence_length;
-        queue_write = (queue_write + 1) & QUEUE_MASK;
 
-        goto re_check;
-    }
-    else
-    {
-        *size = 0; /* end of data */
-        talk_handle_locked--;
+    if (QUEUE_LEVEL != 0) /* queue is not empty? */
+    {   /* start next clip */
+        last_clip = &queue[queue_read];
+        *start = commit_transfer(last_clip, size);
+        curr_hd[0] = commit_buffer[1];
+        curr_hd[1] = commit_buffer[2];
+        curr_hd[2] = commit_buffer[3];
     }
+    
     talk_queue_unlock();
 }
 
@@ -672,7 +745,7 @@ void talk_force_shutup(void)
     unsigned char* search;
     unsigned char* end;
     int len;
-    unsigned clip_offset;
+    unsigned offset;
     if (QUEUE_LEVEL == 0) /* has ended anyway */
         return;
 
@@ -681,11 +754,16 @@ void talk_force_shutup(void)
 #endif /* CONFIG_CPU == SH7034 */
     /* search next frame boundary and continue up to there */
     pos = search = mp3_get_pos();
-    clip_offset = queue[queue_read].offset;
-    if (clip_offset >= voicefile_size)
-        end = core_get_data(thumb_handle) + clip_offset - voicefile_size;
-    else
-        end = core_get_data(talk_handle) + clip_offset;
+    offset = queue[queue_read].offset;
+    switch (queue[queue_read].type)
+    { 
+        case TALK_OFFSET:  end = core_get_data(talk_handle) + offset; break;
+        case THUMB_OFFSET: end = core_get_data(thumb_handle) + offset; break;
+#ifdef TALK_PARTIAL_LOAD
+        case TALK_HANDLE:  end = buflib_get_data(&clip_ctx, offset); break;
+#endif
+        default: end = NULL; /* shut up gcc */
+    }
     len = queue[queue_read].length;
 
     if (pos >= end && pos <= (end+len)) /* really our clip? */
@@ -727,7 +805,6 @@ void talk_force_shutup(void)
     mp3_play_stop();
     talk_queue_lock();
     queue_write = queue_read = 0; /* reset the queue */
-    talk_handle_locked = MAX(talk_handle_locked-1, 0);
     thumbnail_buf_used = 0;
     talk_queue_unlock();
     need_shutup = false;
@@ -741,9 +818,9 @@ void talk_shutup(void)
 }
 
 /* schedule a clip, at the end or discard the existing queue */
-static void queue_clip(unsigned long clip_offset, long size, bool enqueue)
+static void queue_clip(struct queue_entry *clip, bool enqueue)
 {
-    unsigned char *buf;
+    struct queue_entry *qe;
     int queue_level;
 
     if (!enqueue)
@@ -752,7 +829,7 @@ static void queue_clip(unsigned long clip_offset, long size, bool enqueue)
        longer in effect. */
     force_enqueue_next = false;
     
-    if (!size)
+    if (!clip->length)
         return; /* safety check */
 #if CONFIG_CPU == SH7034
     /* disable the DMA temporarily, to be safe of race condition */
@@ -760,32 +837,24 @@ static void queue_clip(unsigned long clip_offset, long size, bool enqueue)
 #endif
     talk_queue_lock();
     queue_level = QUEUE_LEVEL; /* check old level */
+    qe = &queue[queue_write];
 
     if (queue_level < QUEUE_SIZE - 1) /* space left? */
     {
-        queue[queue_write].offset = clip_offset; /* populate an entry */
-        queue[queue_write].length = size;
+        queue[queue_write] = *clip;
         queue_write = (queue_write + 1) & QUEUE_MASK;
     }
     talk_queue_unlock();
 
     if (queue_level == 0)
     {   /* queue was empty, we have to do the initial start */
-        lastclip_offset = clip_offset;
-#if CONFIG_CODEC != SWCODEC
-        sent = MIN(size, 0xFFFF); /* DMA can do no more */
-#else
-        sent = size;
-#endif
-        talk_handle_locked++;
-        if (clip_offset >= voicefile_size)
-            buf = core_get_data(thumb_handle) + clip_offset - voicefile_size;
-        else
-            buf = core_get_data(talk_handle) + clip_offset;
-        mp3_play_data(buf, sent, mp3_callback);
-        curr_hd[0] = buf[1];
-        curr_hd[1] = buf[2];
-        curr_hd[2] = buf[3];
+        size_t size;
+        void *buf = commit_transfer(qe, &size);
+        last_clip = qe;
+        mp3_play_data(buf, size, mp3_callback);
+        curr_hd[0] = commit_buffer[1];
+        curr_hd[1] = commit_buffer[2];
+        curr_hd[2] = commit_buffer[3];
         mp3_play_pause(true); /* kickoff audio */
     }
     else
@@ -822,11 +891,13 @@ void talk_init(void)
         return;
     }
 
-#if CONFIG_CODEC == SWCODEC
     if(!talk_initialized)
+    {
+#if CONFIG_CODEC == SWCODEC
         mutex_init(&queue_mutex);
 #endif /* CONFIG_CODEC == SWCODEC */
-
+        mutex_init(&read_buffer_mutex);
+    }
     talk_initialized = true;
     strlcpy((char *)last_lang, (char *)global_settings.lang_file,
         MAX_FILENAME);
@@ -835,12 +906,7 @@ void talk_init(void)
     queue_write = queue_read = 0; /* reset the queue */
     memset(&voicefile, 0, sizeof(voicefile));
 
-#ifdef TALK_PARTIAL_LOAD
-    for(int i=0; i<QUEUE_SIZE; i++)
-        buffered_id[i] = -1;
-#endif
-
-    silence_offset = -1; /* pause clip not accessible */
+    silence.offset = -1; /* pause clip not accessible */
     voicefile_size = has_voicefile = 0;
     /* need to free these as their size depends on the voice file, and
      * this function is called when the talk voice file changes */
@@ -861,34 +927,59 @@ void talk_init(void)
      * at once */
     unsigned num_clips = voicefile.id1_max + voicefile.id2_max;
     struct clip_entry *clips = core_get_data(index_handle);
-    int silence_size = 0;
-
-    for(unsigned i=0; i<num_clips; i++) {
-        int size = clips[i].size;
-        if (size > max_clipsize)
-            max_clipsize = size;
-        if (i == VOICE_PAUSE)
-            silence_size = size;
+    int avg_size = clips[0].size;
+    int real_clips = 1; /* shut up gcc */
+    /* check for the smallest clip size to estimate the max. number of clips
+     * the buffer has to hold */
+    for(unsigned i=1; i<num_clips; i++)
+    {
+        if (clips[i].size)
+        {   /* don't consider empty clips, they are not stored anyway */
+            real_clips += 1;
+            avg_size += clips[i].size;
+        }
     }
-
-    voicefile_size = voicefile.table + num_clips * sizeof(struct clip_entry);
-    voicefile_size += max_clipsize * QUEUE_SIZE + silence_size;
-
-    /* test if we can open and if it fits in the audiobuffer */
-    size_t audiobufsz = audio_buffer_available();
-    has_voicefile = audiobufsz >= voicefile_size;
+    avg_size /= real_clips;
+
+    max_clips = MIN((int)(MAX_CLIP_BUFFER_SIZE/avg_size) + 1, real_clips);
+    voicefile_size = MAX_CLIP_BUFFER_SIZE;
+    /* additionally to the clip we need a table to record the age of the clips
+     * so that, when memory is tight, only the most recently used ones are kept */
+    voicefile_size += sizeof(struct clip_cache_metadata) * max_clips;
+    /* compensate a bit for buflib per-alloc overhead */
+    voicefile_size += BUFLIB_ALLOC_OVERHEAD * max_clips;
+    /* cap to the max. number of clips or the size of the available audio
+     * buffer which we grab. We leave some to the rest of the system.
+     * While that reduces our buffer size it improves the chance that
+     * other allocs succeed without disabling voice which would require
+     * reloading the voice from disk (as we do not shrink our buffer when
+     * other code attempts new allocs these would fail) */
+    ssize_t cap = MIN(MAX_CLIP_BUFFER_SIZE, audio_buffer_available() - (64<<10));
+    if (UNLIKELY(cap < 0))
+    {
+        logf("Not enough memory for voice. Disabling...\n");
+        if (index_handle > 0)
+            index_handle = core_free(index_handle);
+        voicefile_size = 0;
+        goto out;
+    }
+    else if (voicefile_size > (size_t)cap)
+        voicefile_size = cap;
 
 #else
-    /* load the compressed clip data into memory, in its entirety */
-    voicefile_size = filesize(filehandle);
+    size_t clips_size;
+    clips_size = (voicefile.id1_max+voicefile.id2_max) * sizeof(struct clip_entry);
+    voicefile_size = filesize(filehandle) - voicefile.table - clips_size;
+    /* load the compressed clip data into memory */
     if (!load_voicefile_data(filehandle, voicefile_size))
     {
         voicefile_size = 0;
         goto out;
     }
-    has_voicefile = true;
 #endif
 
+    has_voicefile = true;
+
 #if CONFIG_CODEC == SWCODEC
     /* Initialize the actual voice clip playback engine as well */
     if (talk_voice_required())
@@ -915,10 +1006,9 @@ void talk_buffer_set_policy(int policy)
 /* play a voice ID from voicefile */
 int talk_id(int32_t id, bool enqueue)
 {
-    int clip;
-    long clipsize;
     int32_t unit;
     int decimals;
+    struct queue_entry clip;
 
     if (!has_voicefile)
         return 0; /* no voicefile loaded, not an error -> pretent success */
@@ -952,8 +1042,7 @@ int talk_id(int32_t id, bool enqueue)
         return 0; /* and stop, end of special case */
     }
 
-    clip = get_clip(id, &clipsize);
-    if (clip < 0)
+    if (get_clip(id, &clip) < 0)
         return -1; /* not present */
 
 #ifdef LOGF_ENABLE
@@ -963,7 +1052,7 @@ int talk_id(int32_t id, bool enqueue)
         logf("\ntalk_id: Say '%s'\n", str(id));
 #endif
 
-    queue_clip(clip, clipsize, enqueue);
+    queue_clip(&clip, enqueue);
 
     return 0;
 }
@@ -997,11 +1086,11 @@ static int _talk_file(const char* filename,
     int fd;
     int size;
     int thumb_used;
-    char *buf;
 #if CONFIG_CODEC != SWCODEC
     struct mp3entry info;
 #endif
 
+    /* reload needed? */
     if (talk_temp_disable_count > 0)
         return -1;  /* talking has been disabled */
     if (!check_audio_status())
@@ -1038,16 +1127,14 @@ static int _talk_file(const char* filename,
     lseek(fd, info.first_frame_offset, SEEK_SET); /* behind ID data */
 #endif
 
-    talk_handle_locked++;
-    buf = core_get_data(thumb_handle);
-    size = read(fd, buf+thumb_used, size_for_thumbnail - thumb_used);
-    talk_handle_locked--;
+    size = read_to_handle(fd, thumb_handle, thumb_used, size_for_thumbnail - thumb_used);
     close(fd);
 
     /* ToDo: find audio, skip ID headers and trailers */
 
     if (size > 0)    /* Don't play missing clips */
     {
+        struct queue_entry clip;
 #if CONFIG_CODEC != SWCODEC && !defined(SIMULATOR)
         /* bitswap doesnt yield() */
         bitswap(core_get_data(thumb_handle), size);
@@ -1060,7 +1147,8 @@ static int _talk_file(const char* filename,
         talk_queue_lock();
         thumbnail_buf_used = thumb_used + size;
         talk_queue_unlock();
-        queue_clip(voicefile_size + thumb_used, size, true);
+        clip = (struct queue_entry){ .offset = thumb_used, .length = size, .remaining = size, .type = THUMB_OFFSET };
+        queue_clip(&clip, true);
     }
 
     return size;
@@ -1460,7 +1548,6 @@ void talk_time(const struct tm *tm, bool enqueue)
 
 #endif /* CONFIG_RTC */
 
-
 bool talk_get_debug_data(struct talk_debug_data *data)
 {
     char* p_lang = DEFAULT_VOICE_LANG; /* default */
@@ -1500,9 +1587,13 @@ bool talk_get_debug_data(struct talk_debug_data *data)
     }
     data->avg_clipsize /= real_clips;
     data->num_empty_clips = data->num_clips - real_clips;
-    data->memory_allocated = voicefile_size + size_for_thumbnail;
-    data->memory_used = voicefile_size + thumbnail_buf_used;
+    data->memory_allocated = sizeof(commit_buffer) + sizeof(voicefile)
+                         + data->num_clips * sizeof(struct clip_entry)
+                         + voicefile_size + size_for_thumbnail;
+    data->memory_used = data->memory_allocated - size_for_thumbnail + thumbnail_buf_used;
 #ifdef TALK_PARTIAL_LOAD
+    if (talk_handle > 0)
+        data->memory_used -= buflib_available(&clip_ctx);
     data->cached_clips = cached;
     data->cache_hits   = cache_hits;
     data->cache_misses = cache_misses;