tagcache reduce stack usage in tmpbuf_insert and build_index functions

move some functions around for later ifdef for tagcache commit plugin

used fixed width variables in struct temp_file_entry

Change-Id: Idf9c37d67fc048550366e3d3504601a19c31f31e

1 files changed, 141 insertions, 135 deletions

diff --git a/apps/tagcache.c b/apps/tagcache.c
index f7746f1ec3..16f48a7828 100644
--- a/apps/tagcache.c
+++ b/apps/tagcache.c
@@ -55,6 +55,7 @@
  *
  */
 
+
 /*#define LOGF_ENABLE*/
 /*#define LOGF_CLAUSES define to enable logf clause matching (LOGF_ENABLE req'd) */
 
@@ -174,6 +175,8 @@ static const char tagcache_thread_name[] = "tagcache";
 
 /* Previous path when scanning directory tree recursively. */
 static char curpath[TAGCACHE_BUFSZ];
+/* Shared buffer for several build_index fns to reduce stack usage */
+static char tmp_buf[TAGCACHE_BUFSZ];
 
 /* Used when removing duplicates. */
 static char *tempbuf;     /* Allocated when needed. */
@@ -355,11 +358,10 @@ static inline void tcrc_buffer_unlock(void)
  * Full tag entries stored in a temporary file waiting
  * for commit to the cache. */
 struct temp_file_entry {
-    long tag_offset[TAG_COUNT];
-    short tag_length[TAG_COUNT];
-    long flag;
-
-    long data_length;
+    int32_t tag_offset[TAG_COUNT];
+    int16_t tag_length[TAG_COUNT];
+    int32_t flag;
+    int32_t data_length;
 };
 
 struct tempbuf_id_list {
@@ -396,43 +398,6 @@ static inline void str_setlen(char *buf, size_t len)
     buf[len] = '\0';
 }
 
-static void allocate_tempbuf(void)
-{
-    /* Yeah, malloc would be really nice now :) */
-    size_t size;
-    tempbuf_size = 0;
-
-#ifdef __PCTOOL__
-    size = 32*1024*1024;
-    tempbuf = malloc(size);
-    if (tempbuf)
-        tempbuf_size = size;
-#else /* !__PCTOOL__ */
-    /* Need to pass dummy ops to prevent the buffer being moved
-     * out from under us, since we yield during the tagcache commit. */
-    tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked);
-    if (tempbuf_handle > 0)
-    {
-        tempbuf = core_get_data(tempbuf_handle);
-        tempbuf_size = size;
-    }
-#endif /* __PCTOOL__ */
-}
-
-static void free_tempbuf(void)
-{
-    if (tempbuf_size == 0)
-        return ;
-
-#ifdef __PCTOOL__
-    free(tempbuf);
-#else
-    tempbuf_handle = core_free(tempbuf_handle);
-#endif
-    tempbuf = NULL;
-    tempbuf_size = 0;
-}
-
 const char* tagcache_tag_to_str(int tag)
 {
     return tags_str[tag];
@@ -631,8 +596,8 @@ static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
                       tc_stat.db_path, tag);
     if (fd < 0)
     {
-        logf("tag file open failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX,
-             tag, write, tag);
+        logf("%s failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX,
+             __func__, tag, write, tag);
         tc_stat.ready = false;
         return fd;
     }
@@ -693,6 +658,85 @@ static int open_master_fd(struct master_header *hdr, bool write)
     return fd;
 }
 
+static void remove_files(void)
+{
+    int i;
+    char buf[MAX_PATH];
+    const int bufsz = sizeof(buf);
+    logf("%s", __func__);
+
+    tc_stat.ready = false;
+    tc_stat.ramcache = false;
+    tc_stat.econ = false;
+    remove_db_file(TAGCACHE_FILE_MASTER);
+    for (i = 0; i < TAG_COUNT; i++)
+    {
+        if (TAGCACHE_IS_NUMERIC(i))
+            continue;
+
+        snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX,
+                 tc_stat.db_path, i);
+        remove(buf);
+    }
+}
+
+static bool check_all_headers(void)
+{
+    struct master_header myhdr;
+    struct tagcache_header tch;
+    int tag;
+    int fd;
+
+    if ( (fd = open_master_fd(&myhdr, false)) < 0)
+        return false;
+
+    close(fd);
+    if (myhdr.dirty)
+    {
+        logf("tagcache is dirty!");
+        return false;
+    }
+
+    memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
+
+    for (tag = 0; tag < TAG_COUNT; tag++)
+    {
+        if (TAGCACHE_IS_NUMERIC(tag))
+            continue;
+
+        if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
+            return false;
+
+        close(fd);
+    }
+
+    return true;
+}
+
+static bool update_master_header(void)
+{
+    struct master_header myhdr;
+    int fd;
+
+    if (!tc_stat.ready)
+        return false;
+
+    if ( (fd = open_master_fd(&myhdr, true)) < 0)
+        return false;
+
+    myhdr.serial = current_tcmh.serial;
+    myhdr.commitid = current_tcmh.commitid;
+    myhdr.dirty = current_tcmh.dirty;
+
+    /* Write it back */
+    lseek(fd, 0, SEEK_SET);
+    write_master_header(fd, &myhdr);
+    close(fd);
+
+    return true;
+}
+
+
 #ifndef __PCTOOL__
 static bool do_timed_yield(void)
 {
@@ -708,6 +752,44 @@ static bool do_timed_yield(void)
 }
 #endif /* __PCTOOL__ */
 
+static void allocate_tempbuf(void)
+{
+    /* Yeah, malloc would be really nice now :) */
+    size_t size;
+    tempbuf_size = 0;
+
+#ifdef __PCTOOL__
+    size = 32*1024*1024;
+    tempbuf = malloc(size);
+    if (tempbuf)
+        tempbuf_size = size;
+#else /* !__PCTOOL__ */
+    /* Need to pass dummy ops to prevent the buffer being moved
+     * out from under us, since we yield during the tagcache commit. */
+    tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked);
+    if (tempbuf_handle > 0)
+    {
+        tempbuf = core_get_data(tempbuf_handle);
+        tempbuf_size = size;
+    }
+#endif /* __PCTOOL__ */
+
+}
+
+static void free_tempbuf(void)
+{
+    if (tempbuf_size == 0)
+        return ;
+
+#ifdef __PCTOOL__
+    free(tempbuf);
+#else
+    tempbuf_handle = core_free(tempbuf_handle);
+#endif
+    tempbuf = NULL;
+    tempbuf_size = 0;
+}
+
 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 /* find the ramcache entry corresponding to the file indicated by
  * filename and dc (it's corresponding dircache id). */
@@ -1636,61 +1718,6 @@ static bool build_lookup_list(struct tagcache_search *tcs)
 }
 
 
-static void remove_files(void)
-{
-    int i;
-    char buf[MAX_PATH];
-    const int bufsz = sizeof(buf);
-    
-    tc_stat.ready = false;
-    tc_stat.ramcache = false;
-    tc_stat.econ = false;
-    remove_db_file(TAGCACHE_FILE_MASTER);
-    for (i = 0; i < TAG_COUNT; i++)
-    {
-        if (TAGCACHE_IS_NUMERIC(i))
-            continue;
-
-        snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX,
-                 tc_stat.db_path, i);
-        remove(buf);
-    }
-}
-
-
-static bool check_all_headers(void)
-{
-    struct master_header myhdr;
-    struct tagcache_header tch;
-    int tag;
-    int fd;
-
-    if ( (fd = open_master_fd(&myhdr, false)) < 0)
-        return false;
-
-    close(fd);
-    if (myhdr.dirty)
-    {
-        logf("tagcache is dirty!");
-        return false;
-    }
-
-    memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
-
-    for (tag = 0; tag < TAG_COUNT; tag++)
-    {
-        if (TAGCACHE_IS_NUMERIC(tag))
-            continue;
-
-        if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
-            return false;
-
-        close(fd);
-    }
-
-    return true;
-}
-
 bool tagcache_search(struct tagcache_search *tcs, int tag)
 {
     struct tagcache_header tag_hdr;
@@ -2005,29 +2032,6 @@ bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
     return retrieve(tcs, IF_DIRCACHE(idxid,) &idx, tag, buf, size);
 }
 
-static bool update_master_header(void)
-{
-    struct master_header myhdr;
-    int fd;
-
-    if (!tc_stat.ready)
-        return false;
-
-    if ( (fd = open_master_fd(&myhdr, true)) < 0)
-        return false;
-
-    myhdr.serial = current_tcmh.serial;
-    myhdr.commitid = current_tcmh.commitid;
-    myhdr.dirty = current_tcmh.dirty;
-
-    /* Write it back */
-    lseek(fd, 0, SEEK_SET);
-    write_master_header(fd, &myhdr);
-    close(fd);
-
-    return true;
-}
-
 void tagcache_search_finish(struct tagcache_search *tcs)
 {
     int i;
@@ -2366,15 +2370,14 @@ static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
     int len = strlen(str)+1;
     int i;
-    unsigned crc32;
     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
-    char buf[TAGCACHE_BUFSZ];
-    const int bufsz = sizeof(buf);
-    
-    for (i = 0; str[i] != '\0' && i < bufsz-1; i++)
-        buf[i] = tolower(str[i]);
-
-    crc32 = crc_32(buf, i, 0xffffffff);
+    unsigned crc32 = 0xffffffff;
+    char chr_lower;
+    for (i = 0; str[i] != '\0' && i < len; i++)
+    {
+        chr_lower = tolower(str[i]);
+        crc32 = crc_32(&chr_lower, 1, crc32);
+    }
 
     if (unique)
     {
@@ -2568,8 +2571,8 @@ static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
     int entries_processed = 0;
     int i, j;
 
-    char buf[TAGCACHE_BUFSZ];
-    const int bufsz = sizeof(buf);
+    char *buf = tmp_buf;
+    const long bufsz = sizeof(tmp_buf);
 
     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
 
@@ -2808,8 +2811,8 @@ static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
     struct master_header   tcmh;
     struct index_entry idxbuf[IDX_BUF_DEPTH];
     int idxbuf_pos;
-    char buf[TAGCACHE_BUFSZ];
-    const long bufsz = sizeof(buf);
+    char *buf = tmp_buf;
+    const long bufsz = sizeof(tmp_buf);
     int fd = -1, masterfd;
     bool error = false;
     int init;
@@ -2947,6 +2950,7 @@ static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
     }
     else
     {
+        logf("Create New Index: %d", index_type);
         /**
          * Creating new index file to store the tags. No need to preload
          * anything whether the index type is sorted or not.
@@ -3408,6 +3412,7 @@ static bool commit(void)
             tc_stat.commit_step = 0;
             goto commit_error;
         }
+        do_timed_yield();
     }
 
     if (!build_numeric_indices(&tch, tmpfd))
@@ -3443,7 +3448,7 @@ static bool commit(void)
     tc_stat.ready = check_all_headers();
     tc_stat.readyvalid = true;
 
-#ifdef HAVE_TC_RAMCACHE
+#if defined(HAVE_TC_RAMCACHE)
     if (ramcache_buffer_stolen)
     {
         tempbuf = NULL;
@@ -3483,6 +3488,7 @@ commit_error:
     return rc;
 }
 
+
 #ifndef __PCTOOL__
 
 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)