From b64d46d9d450588e00555a5c12c2fc119448657e Mon Sep 17 00:00:00 2001 From: William Wilgus Date: Tue, 3 Oct 2023 11:07:49 -0400 Subject: Revert "tagcache reduce stack usage in tmpbuf_insert and build_index functions" This reverts commit 3bb3e3d1a6297c7bc9015d008389266e529758c1. Reason for revert: lifetimes do not overlap but patch has issues Change-Id: I3f831599dd1523c156205aa6565ff8afc2f4d8f6 --- apps/tagcache.c | 276 +++++++++++++++++++++++++++----------------------------- 1 file changed, 135 insertions(+), 141 deletions(-) diff --git a/apps/tagcache.c b/apps/tagcache.c index 16f48a7828..f7746f1ec3 100644 --- a/apps/tagcache.c +++ b/apps/tagcache.c @@ -55,7 +55,6 @@ * */ - /*#define LOGF_ENABLE*/ /*#define LOGF_CLAUSES define to enable logf clause matching (LOGF_ENABLE req'd) */ @@ -175,8 +174,6 @@ static const char tagcache_thread_name[] = "tagcache"; /* Previous path when scanning directory tree recursively. */ static char curpath[TAGCACHE_BUFSZ]; -/* Shared buffer for several build_index fns to reduce stack usage */ -static char tmp_buf[TAGCACHE_BUFSZ]; /* Used when removing duplicates. */ static char *tempbuf; /* Allocated when needed. */ @@ -358,10 +355,11 @@ static inline void tcrc_buffer_unlock(void) * Full tag entries stored in a temporary file waiting * for commit to the cache. */ struct temp_file_entry { - int32_t tag_offset[TAG_COUNT]; - int16_t tag_length[TAG_COUNT]; - int32_t flag; - int32_t data_length; + long tag_offset[TAG_COUNT]; + short tag_length[TAG_COUNT]; + long flag; + + long data_length; }; struct tempbuf_id_list { @@ -398,6 +396,43 @@ static inline void str_setlen(char *buf, size_t len) buf[len] = '\0'; } +static void allocate_tempbuf(void) +{ + /* Yeah, malloc would be really nice now :) */ + size_t size; + tempbuf_size = 0; + +#ifdef __PCTOOL__ + size = 32*1024*1024; + tempbuf = malloc(size); + if (tempbuf) + tempbuf_size = size; +#else /* !__PCTOOL__ */ + /* Need to pass dummy ops to prevent the buffer being moved + * out from under us, since we yield during the tagcache commit. */ + tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked); + if (tempbuf_handle > 0) + { + tempbuf = core_get_data(tempbuf_handle); + tempbuf_size = size; + } +#endif /* __PCTOOL__ */ +} + +static void free_tempbuf(void) +{ + if (tempbuf_size == 0) + return ; + +#ifdef __PCTOOL__ + free(tempbuf); +#else + tempbuf_handle = core_free(tempbuf_handle); +#endif + tempbuf = NULL; + tempbuf_size = 0; +} + const char* tagcache_tag_to_str(int tag) { return tags_str[tag]; @@ -596,8 +631,8 @@ static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write) tc_stat.db_path, tag); if (fd < 0) { - logf("%s failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX, - __func__, tag, write, tag); + logf("tag file open failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX, + tag, write, tag); tc_stat.ready = false; return fd; } @@ -658,85 +693,6 @@ static int open_master_fd(struct master_header *hdr, bool write) return fd; } -static void remove_files(void) -{ - int i; - char buf[MAX_PATH]; - const int bufsz = sizeof(buf); - logf("%s", __func__); - - tc_stat.ready = false; - tc_stat.ramcache = false; - tc_stat.econ = false; - remove_db_file(TAGCACHE_FILE_MASTER); - for (i = 0; i < TAG_COUNT; i++) - { - if (TAGCACHE_IS_NUMERIC(i)) - continue; - - snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX, - tc_stat.db_path, i); - remove(buf); - } -} - -static bool check_all_headers(void) -{ - struct master_header myhdr; - struct tagcache_header tch; - int tag; - int fd; - - if ( (fd = open_master_fd(&myhdr, false)) < 0) - return false; - - close(fd); - if (myhdr.dirty) - { - logf("tagcache is dirty!"); - return false; - } - - memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header)); - - for (tag = 0; tag < TAG_COUNT; tag++) - { - if (TAGCACHE_IS_NUMERIC(tag)) - continue; - - if ( (fd = open_tag_fd(&tch, tag, false)) < 0) - return false; - - close(fd); - } - - return true; -} - -static bool update_master_header(void) -{ - struct master_header myhdr; - int fd; - - if (!tc_stat.ready) - return false; - - if ( (fd = open_master_fd(&myhdr, true)) < 0) - return false; - - myhdr.serial = current_tcmh.serial; - myhdr.commitid = current_tcmh.commitid; - myhdr.dirty = current_tcmh.dirty; - - /* Write it back */ - lseek(fd, 0, SEEK_SET); - write_master_header(fd, &myhdr); - close(fd); - - return true; -} - - #ifndef __PCTOOL__ static bool do_timed_yield(void) { @@ -752,44 +708,6 @@ static bool do_timed_yield(void) } #endif /* __PCTOOL__ */ -static void allocate_tempbuf(void) -{ - /* Yeah, malloc would be really nice now :) */ - size_t size; - tempbuf_size = 0; - -#ifdef __PCTOOL__ - size = 32*1024*1024; - tempbuf = malloc(size); - if (tempbuf) - tempbuf_size = size; -#else /* !__PCTOOL__ */ - /* Need to pass dummy ops to prevent the buffer being moved - * out from under us, since we yield during the tagcache commit. */ - tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked); - if (tempbuf_handle > 0) - { - tempbuf = core_get_data(tempbuf_handle); - tempbuf_size = size; - } -#endif /* __PCTOOL__ */ - -} - -static void free_tempbuf(void) -{ - if (tempbuf_size == 0) - return ; - -#ifdef __PCTOOL__ - free(tempbuf); -#else - tempbuf_handle = core_free(tempbuf_handle); -#endif - tempbuf = NULL; - tempbuf_size = 0; -} - #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) /* find the ramcache entry corresponding to the file indicated by * filename and dc (it's corresponding dircache id). */ @@ -1718,6 +1636,61 @@ static bool build_lookup_list(struct tagcache_search *tcs) } +static void remove_files(void) +{ + int i; + char buf[MAX_PATH]; + const int bufsz = sizeof(buf); + + tc_stat.ready = false; + tc_stat.ramcache = false; + tc_stat.econ = false; + remove_db_file(TAGCACHE_FILE_MASTER); + for (i = 0; i < TAG_COUNT; i++) + { + if (TAGCACHE_IS_NUMERIC(i)) + continue; + + snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX, + tc_stat.db_path, i); + remove(buf); + } +} + + +static bool check_all_headers(void) +{ + struct master_header myhdr; + struct tagcache_header tch; + int tag; + int fd; + + if ( (fd = open_master_fd(&myhdr, false)) < 0) + return false; + + close(fd); + if (myhdr.dirty) + { + logf("tagcache is dirty!"); + return false; + } + + memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header)); + + for (tag = 0; tag < TAG_COUNT; tag++) + { + if (TAGCACHE_IS_NUMERIC(tag)) + continue; + + if ( (fd = open_tag_fd(&tch, tag, false)) < 0) + return false; + + close(fd); + } + + return true; +} + bool tagcache_search(struct tagcache_search *tcs, int tag) { struct tagcache_header tag_hdr; @@ -2032,6 +2005,29 @@ bool tagcache_retrieve(struct tagcache_search *tcs, int idxid, return retrieve(tcs, IF_DIRCACHE(idxid,) &idx, tag, buf, size); } +static bool update_master_header(void) +{ + struct master_header myhdr; + int fd; + + if (!tc_stat.ready) + return false; + + if ( (fd = open_master_fd(&myhdr, true)) < 0) + return false; + + myhdr.serial = current_tcmh.serial; + myhdr.commitid = current_tcmh.commitid; + myhdr.dirty = current_tcmh.dirty; + + /* Write it back */ + lseek(fd, 0, SEEK_SET); + write_master_header(fd, &myhdr); + close(fd); + + return true; +} + void tagcache_search_finish(struct tagcache_search *tcs) { int i; @@ -2370,14 +2366,15 @@ static bool tempbuf_insert(char *str, int id, int idx_id, bool unique) struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; int len = strlen(str)+1; int i; + unsigned crc32; unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4]; - unsigned crc32 = 0xffffffff; - char chr_lower; - for (i = 0; str[i] != '\0' && i < len; i++) - { - chr_lower = tolower(str[i]); - crc32 = crc_32(&chr_lower, 1, crc32); - } + char buf[TAGCACHE_BUFSZ]; + const int bufsz = sizeof(buf); + + for (i = 0; str[i] != '\0' && i < bufsz-1; i++) + buf[i] = tolower(str[i]); + + crc32 = crc_32(buf, i, 0xffffffff); if (unique) { @@ -2571,8 +2568,8 @@ static bool build_numeric_indices(struct tagcache_header *h, int tmpfd) int entries_processed = 0; int i, j; - char *buf = tmp_buf; - const long bufsz = sizeof(tmp_buf); + char buf[TAGCACHE_BUFSZ]; + const int bufsz = sizeof(buf); max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1; @@ -2811,8 +2808,8 @@ static int build_index(int index_type, struct tagcache_header *h, int tmpfd) struct master_header tcmh; struct index_entry idxbuf[IDX_BUF_DEPTH]; int idxbuf_pos; - char *buf = tmp_buf; - const long bufsz = sizeof(tmp_buf); + char buf[TAGCACHE_BUFSZ]; + const long bufsz = sizeof(buf); int fd = -1, masterfd; bool error = false; int init; @@ -2950,7 +2947,6 @@ static int build_index(int index_type, struct tagcache_header *h, int tmpfd) } else { - logf("Create New Index: %d", index_type); /** * Creating new index file to store the tags. No need to preload * anything whether the index type is sorted or not. @@ -3412,7 +3408,6 @@ static bool commit(void) tc_stat.commit_step = 0; goto commit_error; } - do_timed_yield(); } if (!build_numeric_indices(&tch, tmpfd)) @@ -3448,7 +3443,7 @@ static bool commit(void) tc_stat.ready = check_all_headers(); tc_stat.readyvalid = true; -#if defined(HAVE_TC_RAMCACHE) +#ifdef HAVE_TC_RAMCACHE if (ramcache_buffer_stolen) { tempbuf = NULL; @@ -3488,7 +3483,6 @@ commit_error: return rc; } - #ifndef __PCTOOL__ static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data) -- cgit v1.2.3