From 3bb3e3d1a6297c7bc9015d008389266e529758c1 Mon Sep 17 00:00:00 2001 From: William Wilgus Date: Tue, 3 Oct 2023 09:57:23 -0400 Subject: tagcache reduce stack usage in tmpbuf_insert and build_index functions move some functions around for later ifdef for tagcache commit plugin used fixed width variables in struct temp_file_entry Change-Id: Idf9c37d67fc048550366e3d3504601a19c31f31e --- apps/tagcache.c | 276 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 141 insertions(+), 135 deletions(-) (limited to 'apps/tagcache.c') diff --git a/apps/tagcache.c b/apps/tagcache.c index f7746f1ec3..16f48a7828 100644 --- a/apps/tagcache.c +++ b/apps/tagcache.c @@ -55,6 +55,7 @@ * */ + /*#define LOGF_ENABLE*/ /*#define LOGF_CLAUSES define to enable logf clause matching (LOGF_ENABLE req'd) */ @@ -174,6 +175,8 @@ static const char tagcache_thread_name[] = "tagcache"; /* Previous path when scanning directory tree recursively. */ static char curpath[TAGCACHE_BUFSZ]; +/* Shared buffer for several build_index fns to reduce stack usage */ +static char tmp_buf[TAGCACHE_BUFSZ]; /* Used when removing duplicates. */ static char *tempbuf; /* Allocated when needed. */ @@ -355,11 +358,10 @@ static inline void tcrc_buffer_unlock(void) * Full tag entries stored in a temporary file waiting * for commit to the cache. */ struct temp_file_entry { - long tag_offset[TAG_COUNT]; - short tag_length[TAG_COUNT]; - long flag; - - long data_length; + int32_t tag_offset[TAG_COUNT]; + int16_t tag_length[TAG_COUNT]; + int32_t flag; + int32_t data_length; }; struct tempbuf_id_list { @@ -396,43 +398,6 @@ static inline void str_setlen(char *buf, size_t len) buf[len] = '\0'; } -static void allocate_tempbuf(void) -{ - /* Yeah, malloc would be really nice now :) */ - size_t size; - tempbuf_size = 0; - -#ifdef __PCTOOL__ - size = 32*1024*1024; - tempbuf = malloc(size); - if (tempbuf) - tempbuf_size = size; -#else /* !__PCTOOL__ */ - /* Need to pass dummy ops to prevent the buffer being moved - * out from under us, since we yield during the tagcache commit. */ - tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked); - if (tempbuf_handle > 0) - { - tempbuf = core_get_data(tempbuf_handle); - tempbuf_size = size; - } -#endif /* __PCTOOL__ */ -} - -static void free_tempbuf(void) -{ - if (tempbuf_size == 0) - return ; - -#ifdef __PCTOOL__ - free(tempbuf); -#else - tempbuf_handle = core_free(tempbuf_handle); -#endif - tempbuf = NULL; - tempbuf_size = 0; -} - const char* tagcache_tag_to_str(int tag) { return tags_str[tag]; @@ -631,8 +596,8 @@ static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write) tc_stat.db_path, tag); if (fd < 0) { - logf("tag file open failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX, - tag, write, tag); + logf("%s failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX, + __func__, tag, write, tag); tc_stat.ready = false; return fd; } @@ -693,6 +658,85 @@ static int open_master_fd(struct master_header *hdr, bool write) return fd; } +static void remove_files(void) +{ + int i; + char buf[MAX_PATH]; + const int bufsz = sizeof(buf); + logf("%s", __func__); + + tc_stat.ready = false; + tc_stat.ramcache = false; + tc_stat.econ = false; + remove_db_file(TAGCACHE_FILE_MASTER); + for (i = 0; i < TAG_COUNT; i++) + { + if (TAGCACHE_IS_NUMERIC(i)) + continue; + + snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX, + tc_stat.db_path, i); + remove(buf); + } +} + +static bool check_all_headers(void) +{ + struct master_header myhdr; + struct tagcache_header tch; + int tag; + int fd; + + if ( (fd = open_master_fd(&myhdr, false)) < 0) + return false; + + close(fd); + if (myhdr.dirty) + { + logf("tagcache is dirty!"); + return false; + } + + memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header)); + + for (tag = 0; tag < TAG_COUNT; tag++) + { + if (TAGCACHE_IS_NUMERIC(tag)) + continue; + + if ( (fd = open_tag_fd(&tch, tag, false)) < 0) + return false; + + close(fd); + } + + return true; +} + +static bool update_master_header(void) +{ + struct master_header myhdr; + int fd; + + if (!tc_stat.ready) + return false; + + if ( (fd = open_master_fd(&myhdr, true)) < 0) + return false; + + myhdr.serial = current_tcmh.serial; + myhdr.commitid = current_tcmh.commitid; + myhdr.dirty = current_tcmh.dirty; + + /* Write it back */ + lseek(fd, 0, SEEK_SET); + write_master_header(fd, &myhdr); + close(fd); + + return true; +} + + #ifndef __PCTOOL__ static bool do_timed_yield(void) { @@ -708,6 +752,44 @@ static bool do_timed_yield(void) } #endif /* __PCTOOL__ */ +static void allocate_tempbuf(void) +{ + /* Yeah, malloc would be really nice now :) */ + size_t size; + tempbuf_size = 0; + +#ifdef __PCTOOL__ + size = 32*1024*1024; + tempbuf = malloc(size); + if (tempbuf) + tempbuf_size = size; +#else /* !__PCTOOL__ */ + /* Need to pass dummy ops to prevent the buffer being moved + * out from under us, since we yield during the tagcache commit. */ + tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked); + if (tempbuf_handle > 0) + { + tempbuf = core_get_data(tempbuf_handle); + tempbuf_size = size; + } +#endif /* __PCTOOL__ */ + +} + +static void free_tempbuf(void) +{ + if (tempbuf_size == 0) + return ; + +#ifdef __PCTOOL__ + free(tempbuf); +#else + tempbuf_handle = core_free(tempbuf_handle); +#endif + tempbuf = NULL; + tempbuf_size = 0; +} + #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) /* find the ramcache entry corresponding to the file indicated by * filename and dc (it's corresponding dircache id). */ @@ -1636,61 +1718,6 @@ static bool build_lookup_list(struct tagcache_search *tcs) } -static void remove_files(void) -{ - int i; - char buf[MAX_PATH]; - const int bufsz = sizeof(buf); - - tc_stat.ready = false; - tc_stat.ramcache = false; - tc_stat.econ = false; - remove_db_file(TAGCACHE_FILE_MASTER); - for (i = 0; i < TAG_COUNT; i++) - { - if (TAGCACHE_IS_NUMERIC(i)) - continue; - - snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX, - tc_stat.db_path, i); - remove(buf); - } -} - - -static bool check_all_headers(void) -{ - struct master_header myhdr; - struct tagcache_header tch; - int tag; - int fd; - - if ( (fd = open_master_fd(&myhdr, false)) < 0) - return false; - - close(fd); - if (myhdr.dirty) - { - logf("tagcache is dirty!"); - return false; - } - - memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header)); - - for (tag = 0; tag < TAG_COUNT; tag++) - { - if (TAGCACHE_IS_NUMERIC(tag)) - continue; - - if ( (fd = open_tag_fd(&tch, tag, false)) < 0) - return false; - - close(fd); - } - - return true; -} - bool tagcache_search(struct tagcache_search *tcs, int tag) { struct tagcache_header tag_hdr; @@ -2005,29 +2032,6 @@ bool tagcache_retrieve(struct tagcache_search *tcs, int idxid, return retrieve(tcs, IF_DIRCACHE(idxid,) &idx, tag, buf, size); } -static bool update_master_header(void) -{ - struct master_header myhdr; - int fd; - - if (!tc_stat.ready) - return false; - - if ( (fd = open_master_fd(&myhdr, true)) < 0) - return false; - - myhdr.serial = current_tcmh.serial; - myhdr.commitid = current_tcmh.commitid; - myhdr.dirty = current_tcmh.dirty; - - /* Write it back */ - lseek(fd, 0, SEEK_SET); - write_master_header(fd, &myhdr); - close(fd); - - return true; -} - void tagcache_search_finish(struct tagcache_search *tcs) { int i; @@ -2366,15 +2370,14 @@ static bool tempbuf_insert(char *str, int id, int idx_id, bool unique) struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; int len = strlen(str)+1; int i; - unsigned crc32; unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4]; - char buf[TAGCACHE_BUFSZ]; - const int bufsz = sizeof(buf); - - for (i = 0; str[i] != '\0' && i < bufsz-1; i++) - buf[i] = tolower(str[i]); - - crc32 = crc_32(buf, i, 0xffffffff); + unsigned crc32 = 0xffffffff; + char chr_lower; + for (i = 0; str[i] != '\0' && i < len; i++) + { + chr_lower = tolower(str[i]); + crc32 = crc_32(&chr_lower, 1, crc32); + } if (unique) { @@ -2568,8 +2571,8 @@ static bool build_numeric_indices(struct tagcache_header *h, int tmpfd) int entries_processed = 0; int i, j; - char buf[TAGCACHE_BUFSZ]; - const int bufsz = sizeof(buf); + char *buf = tmp_buf; + const long bufsz = sizeof(tmp_buf); max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1; @@ -2808,8 +2811,8 @@ static int build_index(int index_type, struct tagcache_header *h, int tmpfd) struct master_header tcmh; struct index_entry idxbuf[IDX_BUF_DEPTH]; int idxbuf_pos; - char buf[TAGCACHE_BUFSZ]; - const long bufsz = sizeof(buf); + char *buf = tmp_buf; + const long bufsz = sizeof(tmp_buf); int fd = -1, masterfd; bool error = false; int init; @@ -2947,6 +2950,7 @@ static int build_index(int index_type, struct tagcache_header *h, int tmpfd) } else { + logf("Create New Index: %d", index_type); /** * Creating new index file to store the tags. No need to preload * anything whether the index type is sorted or not. @@ -3408,6 +3412,7 @@ static bool commit(void) tc_stat.commit_step = 0; goto commit_error; } + do_timed_yield(); } if (!build_numeric_indices(&tch, tmpfd)) @@ -3443,7 +3448,7 @@ static bool commit(void) tc_stat.ready = check_all_headers(); tc_stat.readyvalid = true; -#ifdef HAVE_TC_RAMCACHE +#if defined(HAVE_TC_RAMCACHE) if (ramcache_buffer_stolen) { tempbuf = NULL; @@ -3483,6 +3488,7 @@ commit_error: return rc; } + #ifndef __PCTOOL__ static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data) -- cgit v1.2.3