1 files changed, 373 insertions, 282 deletions

diff --git a/apps/talk.c b/apps/talk.c
index 0904e41c3b..4c4e6184b8 100644
--- a/apps/talk.c
+++ b/apps/talk.c
@@ -87,6 +87,8 @@ const char* const file_thumbnail_ext = ".talk";
 #define MAX_THUMBNAIL_BUFSIZE 0x10000
 #endif
 
+#define DEFAULT_VOICE_LANG "english"
+
 /***************** Data types *****************/
 
 struct clip_entry /* one entry of the index table */
@@ -114,16 +116,7 @@ struct voicefile_header /* file format of our voice file */
  * The Ondios have slow storage access and loading the entire voice file would
  * take several seconds, so we use the same mechanism. */
 #define TALK_PARTIAL_LOAD
-#endif
-
-#ifdef TALK_PARTIAL_LOAD
-static long           max_clipsize; /* size of the biggest clip */
-static long           buffered_id[QUEUE_SIZE];  /* IDs of the talk clips */
-static uint8_t        clip_age[QUEUE_SIZE];
-#if QUEUE_SIZE > 255
-#   error clip_age[] type too small
-#endif
-static int cache_hits, cache_misses;
+#define MAX_CLIP_BUFFER_SIZE 100000 /* 70+ clips should fit into 100k */
 #endif
 
 /* Multiple thumbnails can be loaded back-to-back in this buffer. */
@@ -147,9 +140,6 @@ static struct mutex queue_mutex SHAREDBSS_ATTR;
 #endif /* CONFIG_CODEC */
 static int sent; /* how many bytes handed over to playback, owned by ISR */
 static unsigned char curr_hd[3]; /* current frame header, for re-sync */
-static int silence_offset; /* VOICE_PAUSE clip, used for termination */
-static long silence_length; /* length of the VOICE_PAUSE clip */
-static unsigned long lastclip_offset; /* address of latest clip, for silence add */
 static unsigned char last_lang[MAX_FILENAME+1]; /* name of last used lang file (in talk_init) */
 static bool talk_initialized; /* true if talk_init has been called */
 static bool give_buffer_away; /* true if we should give the buffers away in shrink_callback if requested */
@@ -161,27 +151,65 @@ static unsigned long voicefile_size;
 
 struct queue_entry /* one entry of the internal queue */
 {
-    int offset, length;
+    int offset; /* actually a buflib handle if type == HANDLE */
+    int length; /* total length of the clip */
+    int remaining; /* bytes that still need to be deoded */
+    /* for large QUEUE_SIZE values it might be worthwhile to merge the type
+     * into the bits of the above members (as a space saver). For small values
+     * the required extra code outweights this so it's not done here */
+    enum offset_type {
+        TALK_OFFSET,
+        THUMB_OFFSET,
+#ifdef TALK_PARTIAL_LOAD
+        TALK_HANDLE,
+#endif
+    } type;
 };
 
-static struct queue_entry queue[QUEUE_SIZE]; /* queue of scheduled clips */
+#ifdef TALK_PARTIAL_LOAD
+static struct buflib_context clip_ctx;
 
-#define DEFAULT_VOICE_LANG "english"
+struct clip_cache_metadata {
+    long tick;
+    int handle, voice_id;
+};
+
+static int metadata_table_handle;
+static unsigned max_clips;
+static int cache_hits, cache_misses;
+#endif
+
+static struct queue_entry queue[QUEUE_SIZE]; /* queue of scheduled clips */
+static struct queue_entry silence, *last_clip;
 
 /***************** Private implementation *****************/
 
-static int thumb_handle;
-static int talk_handle, talk_handle_locked;
+static int index_handle, talk_handle, thumb_handle;
+
+static int move_callback(int handle, void *current, void *new)
+{
+    (void)handle; (void)current; (void)new;
+#ifdef TALK_PARTIAL_LOAD
+    if (handle == talk_handle)
+        if (!buflib_context_relocate(&clip_ctx, new))
+            return BUFLIB_CB_CANNOT_MOVE;
+#endif
+    return BUFLIB_CB_OK;
+}
+
+
+static struct mutex read_buffer_mutex;
 
-#if CONFIG_CODEC != SWCODEC
 
 /* on HWCODEC only voice xor audio can be active at a time */
 static bool check_audio_status(void)
 {
+#if CONFIG_CODEC != SWCODEC
     if (audio_status()) /* busy, buffer in use */
         return false;
     /* ensure playback is given up on the buffer */
     audio_hard_stop();
+#endif
     return true;
 }
 
@@ -190,31 +218,43 @@ static bool check_audio_status(void)
 static void sync_callback(int handle, bool sync_on)
 {
     (void) handle;
-    (void) sync_on;
-#if CONFIG_CPU == SH7034
     if (sync_on)
-        CHCR3 &= ~0x0001; /* disable the DMA (and therefore the interrupt also) */
+        mutex_lock(&read_buffer_mutex);
     else
-        CHCR3 |=  0x0001; /* re-enable the DMA */
-#endif
+        mutex_unlock(&read_buffer_mutex);
 }
-#else
-#define check_audio_status() (true)
-#endif
 
-static int move_callback(int handle, void *current, void *new)
+static ssize_t read_to_handle_ex(int fd, struct buflib_context *ctx, int handle,
+                                 int handle_offset, size_t count)
 {
-    (void)handle;(void)current;(void)new;
-    if (UNLIKELY(talk_handle_locked))
-        return BUFLIB_CB_CANNOT_MOVE;
-    return BUFLIB_CB_OK;
+    unsigned char *buf;
+    ssize_t ret;
+    mutex_lock(&read_buffer_mutex);
+
+    if (!ctx)
+        buf = core_get_data(handle);
+    else
+        buf = buflib_get_data(ctx, handle);
+
+    buf += handle_offset;
+    ret = read(fd, buf, count);
+
+    mutex_unlock(&read_buffer_mutex);
+
+    return ret;
 }
 
-static int clip_shrink_callback(int handle, unsigned hints, void *start, size_t old_size)
+static ssize_t read_to_handle(int fd, int handle, int handle_offset, size_t count)
 {
-    (void)start;(void)old_size;(void)hints;
+    return read_to_handle_ex(fd, NULL, handle, handle_offset, count);
+}
+
 
-#if (!defined(TALK_PARTIAL_LOAD) || (MEMORYSIZE > 2))
+static int shrink_callback(int handle, unsigned hints, void *start, size_t old_size)
+{
+    (void)start;(void)old_size;(void)hints;
+    int *h;
+#if (MAX_CLIP_BUFFER_SIZE < (MEMORYSIZE<<20) || (MEMORYSIZE > 2))
     /* on low-mem and when the voice buffer size is not limited (i.e.
      * on 2MB HWCODEC) we effectively own the entire buffer because
      * the voicefile takes up all RAM. This blocks other Rockbox parts
@@ -222,12 +262,22 @@ static int clip_shrink_callback(int handle, unsigned hints, void *start, size_t
      * up the buffer and reload when clips are played back. On high-mem
      * or when the clip buffer is limited to a few 100K this provision is
      * not necessary. */
-    if (LIKELY(!talk_handle_locked)
-            && give_buffer_away
+    if (give_buffer_away
             && (hints & BUFLIB_SHRINK_POS_MASK) == BUFLIB_SHRINK_POS_MASK)
 #endif
     {
-        talk_handle = core_free(handle);
+        if (handle == talk_handle)
+            h = &talk_handle;
+        else //if (handle == index_handle)
+            h = &index_handle;
+
+        mutex_lock(&read_buffer_mutex);
+        /* the clip buffer isn't usable without index table */
+        if (handle == index_handle && talk_handle > 0)
+            talk_handle = core_free(talk_handle);
+        *h = core_free(handle);
+        mutex_unlock(&read_buffer_mutex);
+
         return BUFLIB_CB_OK;
     }
     return BUFLIB_CB_CANNOT_SHRINK;
@@ -237,62 +287,31 @@ static int thumb_shrink_callback(int handle, unsigned hints, void *start, size_t
 {
     (void)start;(void)old_size;(void)hints;
 
-    /* be generous about the thumbnail buffer unless currently used */
-    if (LIKELY(!talk_handle_locked) && thumbnail_buf_used == 0)
+    if (handle == thumb_handle && thumbnail_buf_used == 0)
     {
+        mutex_lock(&read_buffer_mutex);
         thumb_handle = core_free(handle);
+        mutex_unlock(&read_buffer_mutex);
         return BUFLIB_CB_OK;
     }
+
     return BUFLIB_CB_CANNOT_SHRINK;
 }
 
-static struct buflib_callbacks clip_ops = {
+
+
+static struct buflib_callbacks talk_ops = {
     .move_callback = move_callback,
-#if CONFIG_CODEC != SWCODEC
     .sync_callback = sync_callback,
-#endif
-    .shrink_callback = clip_shrink_callback,
+    .shrink_callback = shrink_callback,
 };
 
 static struct buflib_callbacks thumb_ops = {
     .move_callback = move_callback,
-#if CONFIG_CODEC != SWCODEC
     .sync_callback = sync_callback,
-#endif
     .shrink_callback = thumb_shrink_callback,
 };
 
-
-static int index_handle, index_handle_locked;
-static int index_move_callback(int handle, void *current, void *new)
-{
-    (void)handle;(void)current;(void)new;
-    if (UNLIKELY(index_handle_locked))
-        return BUFLIB_CB_CANNOT_MOVE;
-    return BUFLIB_CB_OK;
-}
-
-static int index_shrink_callback(int handle, unsigned hints, void *start, size_t old_size)
-{
-    (void)start;(void)old_size;
-    if (LIKELY(!index_handle_locked)
-            && give_buffer_away
-            && (hints & BUFLIB_SHRINK_POS_MASK) == BUFLIB_SHRINK_POS_MASK)
-    {
-        index_handle = core_free(handle);
-        /* the clip buffer isn't usable without index table */
-        if (LIKELY(!talk_handle_locked))
-            talk_handle = core_free(talk_handle);
-        return BUFLIB_CB_OK;
-    }
-    return BUFLIB_CB_CANNOT_SHRINK;
-}
-
-static struct buflib_callbacks index_ops = {
-    .move_callback = index_move_callback,
-    .shrink_callback = index_shrink_callback,
-};
-
 static int open_voicefile(void)
 {
     char buf[64];
@@ -310,114 +329,143 @@ static int open_voicefile(void)
 }
 
 
-/* fetch a clip from the voice file */
-static int get_clip(long id, long* p_size)
+static int id2index(int id)
 {
-    int retval = -1;
-    struct clip_entry* clipbuf;
-    size_t clipsize;
-
+    int index = id;
     if (id > VOICEONLY_DELIMITER)
     {   /* voice-only entries use the second part of the table.
            The first string comes after VOICEONLY_DELIMITER so we need to
            substract VOICEONLY_DELIMITER + 1 */
-        id -= VOICEONLY_DELIMITER + 1;
-        if (id >= voicefile.id2_max)
+        index -= VOICEONLY_DELIMITER + 1;
+        if (index >= voicefile.id2_max)
             return -1; /* must be newer than we have */
-        id += voicefile.id1_max; /* table 2 is behind table 1 */
+        index += voicefile.id1_max; /* table 2 is behind table 1 */
     }
     else
     {   /* normal use of the first table */
         if (id >= voicefile.id1_max)
             return -1; /* must be newer than we have */
     }
+    return index;
+}
+
+#ifdef TALK_PARTIAL_LOAD
+static int free_oldest_clip(void)
+{
+    unsigned i;
+    int oldest = 0;
+    long age, now;
+    struct clip_entry* clipbuf;
+    struct clip_cache_metadata *cc = buflib_get_data(&clip_ctx, metadata_table_handle);
+    for(age = i = 0, now = current_tick; i < max_clips; i++)
+    {
+        if (cc[i].handle && (now - cc[i].tick) > age
+                && cc[i].voice_id != VOICE_PAUSE) /* never consider silence */
+        {
+            age = now - cc[i].tick;
+            oldest = i;
+        }
+    }
+    cc = &cc[oldest];
+    cc->handle = buflib_free(&clip_ctx, cc->handle);
+    /* need to clear the LOADED bit too */
+    clipbuf = core_get_data(index_handle);
+    clipbuf[id2index(cc->voice_id)].size &= ~LOADED_MASK;
+
+    return oldest;
+}
+#endif
+/* fetch a clip from the voice file */
+static int get_clip(long id, struct queue_entry *q)
+{
+    int index;
+    int retval = -1;
+    struct clip_entry* clipbuf;
+    size_t clipsize;
+    enum offset_type type;
+
+    index = id2index(id);
+    if (index == -1)
+        return -1;
 
     clipbuf = core_get_data(index_handle);
-    clipsize = clipbuf[id].size;
+    clipsize = clipbuf[index].size;
     if (clipsize == 0) /* clip not included in voicefile */
         return -1;
 
 #ifndef TALK_PARTIAL_LOAD
-    retval = clipbuf[id].offset;
+    retval = clipbuf[index].offset;
+    type   = TALK_OFFSET;
 
 #else
     if (!(clipsize & LOADED_MASK))
     {   /* clip needs loading */
+        struct clip_cache_metadata *cc;
+        int fd, handle, oldest = -1;
+        unsigned i;
         ssize_t ret;
-        int fd, idx = 0;
-        unsigned char *voicebuf;
         cache_misses++;
-        if (id == VOICE_PAUSE) {
-            idx = QUEUE_SIZE;   /* we keep VOICE_PAUSE loaded */
-        } else {
-            int oldest = 0, i;
-            for(i=0; i<QUEUE_SIZE; i++) {
-                if (buffered_id[i] < 0) {
-                    /* found a free entry, that means the buffer isn't
-                     * full yet. */
-                    idx = i;
-                    break;
-                }
-
-                /* find the oldest clip */
-                if(clip_age[i] > oldest) {
-                    idx = i;
-                    oldest = clip_age[i];
-                }
-
-                /* increment age of each loaded clip */
-                clip_age[i]++;
-            }
-            clip_age[idx] = 0; /* reset clip's age */
-        }
-        retval = idx * max_clipsize;
+        /* free clips from cache until this one succeeds to allocate */
+        while ((handle = buflib_alloc(&clip_ctx, clipsize)) < 0)
+            oldest = free_oldest_clip();
+        /* handle should now hold a valid alloc. Load from disk
+         * and insert into cache */
         fd = open_voicefile();
         if (fd < 0)
+        {
+            buflib_free(&clip_ctx, handle);
             return -1; /* open error */
-
-        talk_handle_locked++;
-        voicebuf = core_get_data(talk_handle);
+        }
         clipbuf = core_get_data(index_handle);
-        lseek(fd, clipbuf[id].offset, SEEK_SET);
-        ret = read(fd, &voicebuf[retval], clipsize);
+        lseek(fd, clipbuf[index].offset, SEEK_SET);
+        ret = read_to_handle_ex(fd, &clip_ctx, handle, 0, clipsize);
         close(fd);
-        talk_handle_locked--;
 
         if (ret < 0 || clipsize != (size_t)ret)
+        {
+            buflib_free(&clip_ctx, handle);
             return -1; /* read error */
+        }
 
         clipbuf = core_get_data(index_handle);
-        clipbuf[id].size |= LOADED_MASK; /* mark as loaded */
+        clipbuf[index].size |= LOADED_MASK; /* mark as loaded */
 
-        if (id != VOICE_PAUSE) {
-            if (buffered_id[idx] >= 0) {
-                /* mark previously loaded clip as unloaded */
-                clipbuf[buffered_id[idx]].size &= ~LOADED_MASK;
-            }
-            buffered_id[idx] = id;
+        /* finally insert into metadata table */
+        cc = buflib_get_data(&clip_ctx, metadata_table_handle);
+        if (oldest != -1)
+            /* went through the cache in the above loop already, re-use the slot */
+            cc = &cc[oldest];
+        else
+        {   /* find an empty slot */
+            for(i = 0; cc[i].handle && i < max_clips; i++) ;
+            if (i == max_clips) /* no free slot in the cache table? */
+                i = free_oldest_clip();
+            cc = &cc[i];
         }
+        cc->handle = handle;
+        cc->tick = current_tick;
+        cc->voice_id = id;
+        retval = handle;
     }
     else
-    {   /* clip is in memory already */
-        /* Find where it was loaded */
+    {   /* clip is in memory already; find where it was loaded */
         cache_hits++;
-        if (id == VOICE_PAUSE) {
-            retval = QUEUE_SIZE * max_clipsize;
-        } else {
-            int idx;
-            for (idx=0; idx<QUEUE_SIZE; idx++)
-                if (buffered_id[idx] == id) {
-                    retval = idx * max_clipsize;
-                    clip_age[idx] = 0; /* reset clip's age */
-                    break;
-                }
-        }
+        struct clip_cache_metadata *cc;
+        static int i;
+        cc = buflib_get_data(&clip_ctx, metadata_table_handle);
+        for (i = 0; cc[i].voice_id != id || !cc[i].handle; i++) ;
+        cc[i].tick = current_tick; /* reset age */
         clipsize &= ~LOADED_MASK; /* without the extra bit gives true size */
+        retval = cc[i].handle;
     }
+    type = TALK_HANDLE;
 #endif /* TALK_PARTIAL_LOAD */
 
-    *p_size = clipsize;
-    return retval;
+    q->offset    = retval;
+    q->length    = clipsize;
+    q->remaining = clipsize;
+    q->type      = type;
+    return 0;
 }
 
 static bool load_index_table(int fd, const struct voicefile_header *hdr)
@@ -429,13 +477,11 @@ static bool load_index_table(int fd, const struct voicefile_header *hdr)
         return true;
 
     ssize_t alloc_size = (hdr->id1_max + hdr->id2_max) * sizeof(struct clip_entry);
-    index_handle = core_alloc_ex("voice index", alloc_size, &index_ops);
+    index_handle = core_alloc_ex("voice index", alloc_size, &talk_ops);
     if (index_handle < 0)
         return false;
 
-    index_handle_locked++;
-    buf = core_get_data(index_handle);
-    ret = read(fd, buf, alloc_size);
+    ret = read_to_handle(fd, index_handle, 0, alloc_size);
 
 #ifndef TALK_PARTIAL_LOAD
     int clips_offset, num_clips;
@@ -447,19 +493,20 @@ static bool load_index_table(int fd, const struct voicefile_header *hdr)
     clips_offset += num_clips * sizeof(struct clip_entry); /* skip index */
 #endif
     if (ret == alloc_size)
+    {
+        buf = core_get_data(index_handle);
         for (int i = 0; i < hdr->id1_max + hdr->id2_max; i++)
         {
 #ifdef ROCKBOX_LITTLE_ENDIAN
+            /* doesn't yield() */
             structec_convert(&buf[i], "ll", 1, true);
 #endif
 #ifndef TALK_PARTIAL_LOAD
             buf[i].offset -= clips_offset;
 #endif
         }
-
-    index_handle_locked--;
-
-    if (ret != alloc_size)
+    }
+    else
         index_handle = core_free(index_handle);
 
     return ret == alloc_size;
@@ -481,20 +528,16 @@ static bool load_header(int fd, struct voicefile_header *hdr)
 #ifndef TALK_PARTIAL_LOAD
 static bool load_data(int fd, ssize_t size_to_read)
 {
-    unsigned char *buf;
     ssize_t ret;
 
     if (size_to_read < 0)
         return false;
 
-    talk_handle = core_alloc_ex("voice data", size_to_read, &clip_ops);
+    talk_handle = core_alloc_ex("voice data", size_to_read, &talk_ops);
     if (talk_handle < 0)
         return false;
 
-    talk_handle_locked++;
-    buf = core_get_data(talk_handle);
-    ret = read(fd, buf, size_to_read);
-    talk_handle_locked--;
+    ret = read_to_handle(fd, talk_handle, 0, size_to_read);
 
     if (ret != size_to_read)
         talk_handle = core_free(talk_handle);
@@ -558,23 +601,33 @@ static bool load_voicefile_data(int fd, size_t max_size)
 {
 #ifdef TALK_PARTIAL_LOAD
     (void)fd;
-    /* just allocate, populate on an as-needed basis later */
-    talk_handle = core_alloc_ex("voice data", max_size, &clip_ops);
+    size_t alloc_size;
+    /* just allocate, populate on an as-needed basis later
+     * re-create the clip buffer to ensure clip_ctx is up-to-date */
+    if (talk_handle > 0)
+        talk_handle = core_free(talk_handle);
+    talk_handle = core_alloc_ex("voice data", max_size, &talk_ops);
     if (talk_handle < 0)
         goto load_err_free;
+
+    buflib_init(&clip_ctx, core_get_data(talk_handle), max_size);
+
+    alloc_size = max_clips * sizeof(struct clip_cache_metadata);
+    /* the first alloc is the clip metadata table */
+    metadata_table_handle = buflib_alloc(&clip_ctx, alloc_size);
+    memset(buflib_get_data(&clip_ctx, metadata_table_handle), 0, alloc_size);
+
 #else
-    size_t load_size, clips_size;
     /* load the entire file into memory */
-    clips_size = (voicefile.id1_max+voicefile.id2_max) * sizeof(struct clip_entry);
-    load_size  = max_size - voicefile.table - clips_size;
-    if (!load_data(fd, load_size))
+    if (!load_data(fd, max_size))
         goto load_err_free;
 #endif
 
     /* make sure to have the silence clip, if available
      * return value can be cached globally even for TALK_PARTIAL_LOAD because
      * the VOICE_PAUSE clip is specially handled */
-    silence_offset = get_clip(VOICE_PAUSE, &silence_length);
+    if (get_clip(VOICE_PAUSE, &silence))
+        goto load_err_free;
 
     /* not an error if this fails here, might try again when the
      * actual thumbnails are attempted to be played back */
@@ -587,74 +640,94 @@ load_err_free:
     return false;
 }
 
+/* most, if not all, clips should be well below 32k (largest in english.lang is
+ * 4.5K). Currently there is a problem with voice decoding such that clips
+ * cannot be decoded in chunks. Once that is resolved this buffer could be
+ * smaller and clips be decoded in multiple chunks */
+static unsigned char commit_buffer[32<<10];
+
+static void* commit_transfer(struct queue_entry *qe, size_t *size)
+{
+    void *buf = NULL; /* shut up gcc */
+    static unsigned char *bufpos = commit_buffer;
+    int offset = qe->offset;
+#if CONFIG_CODEC != SWCODEC
+    sent = MIN(qe->remaining, 0xFFFF);
+#else
+    sent = qe->remaining;
+#endif
+    sent = MIN((size_t)sent, sizeof(commit_buffer));
+    switch (qe->type)
+    {
+        case TALK_OFFSET:  buf = core_get_data(talk_handle) + offset; break;
+        case THUMB_OFFSET: buf = core_get_data(thumb_handle) + offset; break;
+#ifdef TALK_PARTIAL_LOAD
+        case TALK_HANDLE:  buf = buflib_get_data(&clip_ctx, offset); break;
+#endif
+    }
+    /* adjust buffer position to what has been played already */
+    buf += (qe->length - qe->remaining);
+    memcpy(bufpos, buf, sent);
+    *size = sent;
+
+
+    return commit_buffer;
+}
+
+static inline bool is_silence(struct queue_entry *qe)
+{
+    if (silence.length > 0) /* silence clip available? */
+        return (qe->offset == silence.offset && qe->type == silence.type);
+    else
+        return false;
+}
+
 /* called in ISR context (on HWCODEC) if mp3 data got consumed */
 static void mp3_callback(const void** start, size_t* size)
 {
-    queue[queue_read].length -= sent; /* we completed this */
-    queue[queue_read].offset += sent;
+    struct queue_entry *qe = &queue[queue_read];
+    qe->remaining -= sent; /* we completed this */
 
-    if (queue[queue_read].length > 0) /* current clip not finished? */
+    if (qe->remaining > 0) /* current clip not finished? */
     {   /* feed the next 64K-1 chunk */
-        int offset;
-#if CONFIG_CODEC != SWCODEC
-        sent = MIN(queue[queue_read].length, 0xFFFF);
-#else
-        sent = queue[queue_read].length;
-#endif
-        offset = queue[queue_read].offset;
-        if ((unsigned long)offset >= voicefile_size)
-            *start = core_get_data(thumb_handle) + offset - voicefile_size;
-        else
-            *start = core_get_data(talk_handle) + offset;
-        *size = sent;
+        *start = commit_transfer(qe, size);
         return;
     }
+
     talk_queue_lock();
-    if(thumb_handle && (unsigned long)queue[queue_read].offset == voicefile_size+thumbnail_buf_used)
-        thumbnail_buf_used = 0;
-    if (sent > 0) /* go to next entry */
+    /* check if thumbnails have been played */
+    if (qe->type == THUMB_OFFSET)
     {
-        queue_read = (queue_read + 1) & QUEUE_MASK;
+        if (qe->remaining == 0 && (qe->length + qe->offset) == thumbnail_buf_used)
+            thumbnail_buf_used = 0;
     }
 
-re_check:
+    /* increment read position for the just played clip */
+    queue_read = (queue_read + 1) & QUEUE_MASK;
 
-    if (QUEUE_LEVEL != 0) /* queue is not empty? */
-    {   /* start next clip */
-        unsigned char *buf;
-#if CONFIG_CODEC != SWCODEC
-        sent = MIN(queue[queue_read].length, 0xFFFF);
-#else
-        sent = queue[queue_read].length;
-#endif
-        lastclip_offset = queue[queue_read].offset;
-        /* offsets larger than voicefile_size denote thumbnail clips */
-        if (lastclip_offset >= voicefile_size)
-            buf = core_get_data(thumb_handle) + lastclip_offset - voicefile_size;
+    if (QUEUE_LEVEL == 0)
+    {
+        if (!is_silence(last_clip) && last_clip->type != THUMB_OFFSET)
+        {   /* add silence clip when queue runs empty playing a voice clip,
+             * only if the previous clip wasn't silence or thumbnail */
+            queue[queue_write] = silence;
+            queue_write = (queue_write + 1) & QUEUE_MASK;
+        }
         else
-            buf = core_get_data(talk_handle) + lastclip_offset;
-        *start = buf;
-        *size = sent;
-        curr_hd[0] = buf[1];
-        curr_hd[1] = buf[2];
-        curr_hd[2] = buf[3];
+        {
+            *size = 0; /* end of data */
+        }
     }
-    else if (silence_offset > 0               /* silence clip available */
-             && lastclip_offset != (unsigned long)silence_offset  /* previous clip wasn't silence */
-             && !(lastclip_offset >= voicefile_size   /* ..or thumbnail */
-                 && lastclip_offset < voicefile_size +size_for_thumbnail))
-    {   /* add silence clip when queue runs empty playing a voice clip */
-        queue[queue_write].offset = silence_offset;
-        queue[queue_write].length = silence_length;
-        queue_write = (queue_write + 1) & QUEUE_MASK;
 
-        goto re_check;
-    }
-    else
-    {
-        *size = 0; /* end of data */
-        talk_handle_locked--;
+    if (QUEUE_LEVEL != 0) /* queue is not empty? */
+    {   /* start next clip */
+        last_clip = &queue[queue_read];
+        *start = commit_transfer(last_clip, size);
+        curr_hd[0] = commit_buffer[1];
+        curr_hd[1] = commit_buffer[2];
+        curr_hd[2] = commit_buffer[3];
     }
+    
     talk_queue_unlock();
 }
 
@@ -672,7 +745,7 @@ void talk_force_shutup(void)
     unsigned char* search;
     unsigned char* end;
     int len;
-    unsigned clip_offset;
+    unsigned offset;
     if (QUEUE_LEVEL == 0) /* has ended anyway */
         return;
 
@@ -681,11 +754,16 @@ void talk_force_shutup(void)
 #endif /* CONFIG_CPU == SH7034 */
     /* search next frame boundary and continue up to there */
     pos = search = mp3_get_pos();
-    clip_offset = queue[queue_read].offset;
-    if (clip_offset >= voicefile_size)
-        end = core_get_data(thumb_handle) + clip_offset - voicefile_size;
-    else
-        end = core_get_data(talk_handle) + clip_offset;
+    offset = queue[queue_read].offset;
+    switch (queue[queue_read].type)
+    { 
+        case TALK_OFFSET:  end = core_get_data(talk_handle) + offset; break;
+        case THUMB_OFFSET: end = core_get_data(thumb_handle) + offset; break;
+#ifdef TALK_PARTIAL_LOAD
+        case TALK_HANDLE:  end = buflib_get_data(&clip_ctx, offset); break;
+#endif
+        default: end = NULL; /* shut up gcc */
+    }
     len = queue[queue_read].length;
 
     if (pos >= end && pos <= (end+len)) /* really our clip? */
@@ -727,7 +805,6 @@ void talk_force_shutup(void)
     mp3_play_stop();
     talk_queue_lock();
     queue_write = queue_read = 0; /* reset the queue */
-    talk_handle_locked = MAX(talk_handle_locked-1, 0);
     thumbnail_buf_used = 0;
     talk_queue_unlock();
     need_shutup = false;
@@ -741,9 +818,9 @@ void talk_shutup(void)
 }
 
 /* schedule a clip, at the end or discard the existing queue */
-static void queue_clip(unsigned long clip_offset, long size, bool enqueue)
+static void queue_clip(struct queue_entry *clip, bool enqueue)
 {
-    unsigned char *buf;
+    struct queue_entry *qe;
     int queue_level;
 
     if (!enqueue)
@@ -752,7 +829,7 @@ static void queue_clip(unsigned long clip_offset, long size, bool enqueue)
        longer in effect. */
     force_enqueue_next = false;
     
-    if (!size)
+    if (!clip->length)
         return; /* safety check */
 #if CONFIG_CPU == SH7034
     /* disable the DMA temporarily, to be safe of race condition */
@@ -760,32 +837,24 @@ static void queue_clip(unsigned long clip_offset, long size, bool enqueue)
 #endif
     talk_queue_lock();
     queue_level = QUEUE_LEVEL; /* check old level */
+    qe = &queue[queue_write];
 
     if (queue_level < QUEUE_SIZE - 1) /* space left? */
     {
-        queue[queue_write].offset = clip_offset; /* populate an entry */
-        queue[queue_write].length = size;
+        queue[queue_write] = *clip;
         queue_write = (queue_write + 1) & QUEUE_MASK;
     }
     talk_queue_unlock();
 
     if (queue_level == 0)
     {   /* queue was empty, we have to do the initial start */
-        lastclip_offset = clip_offset;
-#if CONFIG_CODEC != SWCODEC
-        sent = MIN(size, 0xFFFF); /* DMA can do no more */
-#else
-        sent = size;
-#endif
-        talk_handle_locked++;
-        if (clip_offset >= voicefile_size)
-            buf = core_get_data(thumb_handle) + clip_offset - voicefile_size;
-        else
-            buf = core_get_data(talk_handle) + clip_offset;
-        mp3_play_data(buf, sent, mp3_callback);
-        curr_hd[0] = buf[1];
-        curr_hd[1] = buf[2];
-        curr_hd[2] = buf[3];
+        size_t size;
+        void *buf = commit_transfer(qe, &size);
+        last_clip = qe;
+        mp3_play_data(buf, size, mp3_callback);
+        curr_hd[0] = commit_buffer[1];
+        curr_hd[1] = commit_buffer[2];
+        curr_hd[2] = commit_buffer[3];
         mp3_play_pause(true); /* kickoff audio */
     }
     else
@@ -822,11 +891,13 @@ void talk_init(void)
         return;
     }
 
-#if CONFIG_CODEC == SWCODEC
     if(!talk_initialized)
+    {
+#if CONFIG_CODEC == SWCODEC
         mutex_init(&queue_mutex);
 #endif /* CONFIG_CODEC == SWCODEC */
-
+        mutex_init(&read_buffer_mutex);
+    }
     talk_initialized = true;
     strlcpy((char *)last_lang, (char *)global_settings.lang_file,
         MAX_FILENAME);
@@ -835,12 +906,7 @@ void talk_init(void)
     queue_write = queue_read = 0; /* reset the queue */
     memset(&voicefile, 0, sizeof(voicefile));
 
-#ifdef TALK_PARTIAL_LOAD
-    for(int i=0; i<QUEUE_SIZE; i++)
-        buffered_id[i] = -1;
-#endif
-
-    silence_offset = -1; /* pause clip not accessible */
+    silence.offset = -1; /* pause clip not accessible */
     voicefile_size = has_voicefile = 0;
     /* need to free these as their size depends on the voice file, and
      * this function is called when the talk voice file changes */
@@ -861,34 +927,59 @@ void talk_init(void)
      * at once */
     unsigned num_clips = voicefile.id1_max + voicefile.id2_max;
     struct clip_entry *clips = core_get_data(index_handle);
-    int silence_size = 0;
-
-    for(unsigned i=0; i<num_clips; i++) {
-        int size = clips[i].size;
-        if (size > max_clipsize)
-            max_clipsize = size;
-        if (i == VOICE_PAUSE)
-            silence_size = size;
+    int avg_size = clips[0].size;
+    int real_clips = 1; /* shut up gcc */
+    /* check for the smallest clip size to estimate the max. number of clips
+     * the buffer has to hold */
+    for(unsigned i=1; i<num_clips; i++)
+    {
+        if (clips[i].size)
+        {   /* don't consider empty clips, they are not stored anyway */
+            real_clips += 1;
+            avg_size += clips[i].size;
+        }
     }
-
-    voicefile_size = voicefile.table + num_clips * sizeof(struct clip_entry);
-    voicefile_size += max_clipsize * QUEUE_SIZE + silence_size;
-
-    /* test if we can open and if it fits in the audiobuffer */
-    size_t audiobufsz = audio_buffer_available();
-    has_voicefile = audiobufsz >= voicefile_size;
+    avg_size /= real_clips;
+
+    max_clips = MIN((int)(MAX_CLIP_BUFFER_SIZE/avg_size) + 1, real_clips);
+    voicefile_size = MAX_CLIP_BUFFER_SIZE;
+    /* additionally to the clip we need a table to record the age of the clips
+     * so that, when memory is tight, only the most recently used ones are kept */
+    voicefile_size += sizeof(struct clip_cache_metadata) * max_clips;
+    /* compensate a bit for buflib per-alloc overhead */
+    voicefile_size += BUFLIB_ALLOC_OVERHEAD * max_clips;
+    /* cap to the max. number of clips or the size of the available audio
+     * buffer which we grab. We leave some to the rest of the system.
+     * While that reduces our buffer size it improves the chance that
+     * other allocs succeed without disabling voice which would require
+     * reloading the voice from disk (as we do not shrink our buffer when
+     * other code attempts new allocs these would fail) */
+    ssize_t cap = MIN(MAX_CLIP_BUFFER_SIZE, audio_buffer_available() - (64<<10));
+    if (UNLIKELY(cap < 0))
+    {
+        logf("Not enough memory for voice. Disabling...\n");
+        if (index_handle > 0)
+            index_handle = core_free(index_handle);
+        voicefile_size = 0;
+        goto out;
+    }
+    else if (voicefile_size > (size_t)cap)
+        voicefile_size = cap;
 
 #else
-    /* load the compressed clip data into memory, in its entirety */
-    voicefile_size = filesize(filehandle);
+    size_t clips_size;
+    clips_size = (voicefile.id1_max+voicefile.id2_max) * sizeof(struct clip_entry);
+    voicefile_size = filesize(filehandle) - voicefile.table - clips_size;
+    /* load the compressed clip data into memory */
     if (!load_voicefile_data(filehandle, voicefile_size))
     {
         voicefile_size = 0;
         goto out;
     }
-    has_voicefile = true;
 #endif
 
+    has_voicefile = true;
+
 #if CONFIG_CODEC == SWCODEC
     /* Initialize the actual voice clip playback engine as well */
     if (talk_voice_required())
@@ -915,10 +1006,9 @@ void talk_buffer_set_policy(int policy)
 /* play a voice ID from voicefile */
 int talk_id(int32_t id, bool enqueue)
 {
-    int clip;
-    long clipsize;
     int32_t unit;
     int decimals;
+    struct queue_entry clip;
 
     if (!has_voicefile)
         return 0; /* no voicefile loaded, not an error -> pretent success */
@@ -952,8 +1042,7 @@ int talk_id(int32_t id, bool enqueue)
         return 0; /* and stop, end of special case */
     }
 
-    clip = get_clip(id, &clipsize);
-    if (clip < 0)
+    if (get_clip(id, &clip) < 0)
         return -1; /* not present */
 
 #ifdef LOGF_ENABLE
@@ -963,7 +1052,7 @@ int talk_id(int32_t id, bool enqueue)
         logf("\ntalk_id: Say '%s'\n", str(id));
 #endif
 
-    queue_clip(clip, clipsize, enqueue);
+    queue_clip(&clip, enqueue);
 
     return 0;
 }
@@ -997,11 +1086,11 @@ static int _talk_file(const char* filename,
     int fd;
     int size;
     int thumb_used;
-    char *buf;
 #if CONFIG_CODEC != SWCODEC
     struct mp3entry info;
 #endif
 
+    /* reload needed? */
     if (talk_temp_disable_count > 0)
         return -1;  /* talking has been disabled */
     if (!check_audio_status())
@@ -1038,16 +1127,14 @@ static int _talk_file(const char* filename,
     lseek(fd, info.first_frame_offset, SEEK_SET); /* behind ID data */
 #endif
 
-    talk_handle_locked++;
-    buf = core_get_data(thumb_handle);
-    size = read(fd, buf+thumb_used, size_for_thumbnail - thumb_used);
-    talk_handle_locked--;
+    size = read_to_handle(fd, thumb_handle, thumb_used, size_for_thumbnail - thumb_used);
     close(fd);
 
     /* ToDo: find audio, skip ID headers and trailers */
 
     if (size > 0)    /* Don't play missing clips */
     {
+        struct queue_entry clip;
 #if CONFIG_CODEC != SWCODEC && !defined(SIMULATOR)
         /* bitswap doesnt yield() */
         bitswap(core_get_data(thumb_handle), size);
@@ -1060,7 +1147,8 @@ static int _talk_file(const char* filename,
         talk_queue_lock();
         thumbnail_buf_used = thumb_used + size;
         talk_queue_unlock();
-        queue_clip(voicefile_size + thumb_used, size, true);
+        clip = (struct queue_entry){ .offset = thumb_used, .length = size, .remaining = size, .type = THUMB_OFFSET };
+        queue_clip(&clip, true);
     }
 
     return size;
@@ -1460,7 +1548,6 @@ void talk_time(const struct tm *tm, bool enqueue)
 
 #endif /* CONFIG_RTC */
 
-
 bool talk_get_debug_data(struct talk_debug_data *data)
 {
     char* p_lang = DEFAULT_VOICE_LANG; /* default */
@@ -1500,9 +1587,13 @@ bool talk_get_debug_data(struct talk_debug_data *data)
     }
     data->avg_clipsize /= real_clips;
     data->num_empty_clips = data->num_clips - real_clips;
-    data->memory_allocated = voicefile_size + size_for_thumbnail;
-    data->memory_used = voicefile_size + thumbnail_buf_used;
+    data->memory_allocated = sizeof(commit_buffer) + sizeof(voicefile)
+                         + data->num_clips * sizeof(struct clip_entry)
+                         + voicefile_size + size_for_thumbnail;
+    data->memory_used = data->memory_allocated - size_for_thumbnail + thumbnail_buf_used;
 #ifdef TALK_PARTIAL_LOAD
+    if (talk_handle > 0)
+        data->memory_used -= buflib_available(&clip_ctx);
     data->cached_clips = cached;
     data->cache_hits   = cache_hits;
     data->cache_misses = cache_misses;