diff options
author | William Wilgus <wilgus.william@gmail.com> | 2023-10-03 09:57:23 -0400 |
---|---|---|
committer | William Wilgus <wilgus.william@gmail.com> | 2023-10-03 10:08:13 -0400 |
commit | 3bb3e3d1a6297c7bc9015d008389266e529758c1 (patch) | |
tree | cfabbc4554ea7dd66d9f1b3e55a62c893ec41d3e /apps/tagcache.c | |
parent | 7616822fbb0bbe9762fe2b6271982743ac8d114e (diff) | |
download | rockbox-3bb3e3d1a6297c7bc9015d008389266e529758c1.tar.gz rockbox-3bb3e3d1a6297c7bc9015d008389266e529758c1.zip |
tagcache reduce stack usage in tmpbuf_insert and build_index functions
move some functions around for later ifdef for tagcache commit plugin
used fixed width variables in struct temp_file_entry
Change-Id: Idf9c37d67fc048550366e3d3504601a19c31f31e
Diffstat (limited to 'apps/tagcache.c')
-rw-r--r-- | apps/tagcache.c | 276 |
1 files changed, 141 insertions, 135 deletions
diff --git a/apps/tagcache.c b/apps/tagcache.c index f7746f1ec3..16f48a7828 100644 --- a/apps/tagcache.c +++ b/apps/tagcache.c | |||
@@ -55,6 +55,7 @@ | |||
55 | * | 55 | * |
56 | */ | 56 | */ |
57 | 57 | ||
58 | |||
58 | /*#define LOGF_ENABLE*/ | 59 | /*#define LOGF_ENABLE*/ |
59 | /*#define LOGF_CLAUSES define to enable logf clause matching (LOGF_ENABLE req'd) */ | 60 | /*#define LOGF_CLAUSES define to enable logf clause matching (LOGF_ENABLE req'd) */ |
60 | 61 | ||
@@ -174,6 +175,8 @@ static const char tagcache_thread_name[] = "tagcache"; | |||
174 | 175 | ||
175 | /* Previous path when scanning directory tree recursively. */ | 176 | /* Previous path when scanning directory tree recursively. */ |
176 | static char curpath[TAGCACHE_BUFSZ]; | 177 | static char curpath[TAGCACHE_BUFSZ]; |
178 | /* Shared buffer for several build_index fns to reduce stack usage */ | ||
179 | static char tmp_buf[TAGCACHE_BUFSZ]; | ||
177 | 180 | ||
178 | /* Used when removing duplicates. */ | 181 | /* Used when removing duplicates. */ |
179 | static char *tempbuf; /* Allocated when needed. */ | 182 | static char *tempbuf; /* Allocated when needed. */ |
@@ -355,11 +358,10 @@ static inline void tcrc_buffer_unlock(void) | |||
355 | * Full tag entries stored in a temporary file waiting | 358 | * Full tag entries stored in a temporary file waiting |
356 | * for commit to the cache. */ | 359 | * for commit to the cache. */ |
357 | struct temp_file_entry { | 360 | struct temp_file_entry { |
358 | long tag_offset[TAG_COUNT]; | 361 | int32_t tag_offset[TAG_COUNT]; |
359 | short tag_length[TAG_COUNT]; | 362 | int16_t tag_length[TAG_COUNT]; |
360 | long flag; | 363 | int32_t flag; |
361 | 364 | int32_t data_length; | |
362 | long data_length; | ||
363 | }; | 365 | }; |
364 | 366 | ||
365 | struct tempbuf_id_list { | 367 | struct tempbuf_id_list { |
@@ -396,43 +398,6 @@ static inline void str_setlen(char *buf, size_t len) | |||
396 | buf[len] = '\0'; | 398 | buf[len] = '\0'; |
397 | } | 399 | } |
398 | 400 | ||
399 | static void allocate_tempbuf(void) | ||
400 | { | ||
401 | /* Yeah, malloc would be really nice now :) */ | ||
402 | size_t size; | ||
403 | tempbuf_size = 0; | ||
404 | |||
405 | #ifdef __PCTOOL__ | ||
406 | size = 32*1024*1024; | ||
407 | tempbuf = malloc(size); | ||
408 | if (tempbuf) | ||
409 | tempbuf_size = size; | ||
410 | #else /* !__PCTOOL__ */ | ||
411 | /* Need to pass dummy ops to prevent the buffer being moved | ||
412 | * out from under us, since we yield during the tagcache commit. */ | ||
413 | tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked); | ||
414 | if (tempbuf_handle > 0) | ||
415 | { | ||
416 | tempbuf = core_get_data(tempbuf_handle); | ||
417 | tempbuf_size = size; | ||
418 | } | ||
419 | #endif /* __PCTOOL__ */ | ||
420 | } | ||
421 | |||
422 | static void free_tempbuf(void) | ||
423 | { | ||
424 | if (tempbuf_size == 0) | ||
425 | return ; | ||
426 | |||
427 | #ifdef __PCTOOL__ | ||
428 | free(tempbuf); | ||
429 | #else | ||
430 | tempbuf_handle = core_free(tempbuf_handle); | ||
431 | #endif | ||
432 | tempbuf = NULL; | ||
433 | tempbuf_size = 0; | ||
434 | } | ||
435 | |||
436 | const char* tagcache_tag_to_str(int tag) | 401 | const char* tagcache_tag_to_str(int tag) |
437 | { | 402 | { |
438 | return tags_str[tag]; | 403 | return tags_str[tag]; |
@@ -631,8 +596,8 @@ static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write) | |||
631 | tc_stat.db_path, tag); | 596 | tc_stat.db_path, tag); |
632 | if (fd < 0) | 597 | if (fd < 0) |
633 | { | 598 | { |
634 | logf("tag file open failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX, | 599 | logf("%s failed: tag=%d write=%d file= " TAGCACHE_FILE_INDEX, |
635 | tag, write, tag); | 600 | __func__, tag, write, tag); |
636 | tc_stat.ready = false; | 601 | tc_stat.ready = false; |
637 | return fd; | 602 | return fd; |
638 | } | 603 | } |
@@ -693,6 +658,85 @@ static int open_master_fd(struct master_header *hdr, bool write) | |||
693 | return fd; | 658 | return fd; |
694 | } | 659 | } |
695 | 660 | ||
661 | static void remove_files(void) | ||
662 | { | ||
663 | int i; | ||
664 | char buf[MAX_PATH]; | ||
665 | const int bufsz = sizeof(buf); | ||
666 | logf("%s", __func__); | ||
667 | |||
668 | tc_stat.ready = false; | ||
669 | tc_stat.ramcache = false; | ||
670 | tc_stat.econ = false; | ||
671 | remove_db_file(TAGCACHE_FILE_MASTER); | ||
672 | for (i = 0; i < TAG_COUNT; i++) | ||
673 | { | ||
674 | if (TAGCACHE_IS_NUMERIC(i)) | ||
675 | continue; | ||
676 | |||
677 | snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX, | ||
678 | tc_stat.db_path, i); | ||
679 | remove(buf); | ||
680 | } | ||
681 | } | ||
682 | |||
683 | static bool check_all_headers(void) | ||
684 | { | ||
685 | struct master_header myhdr; | ||
686 | struct tagcache_header tch; | ||
687 | int tag; | ||
688 | int fd; | ||
689 | |||
690 | if ( (fd = open_master_fd(&myhdr, false)) < 0) | ||
691 | return false; | ||
692 | |||
693 | close(fd); | ||
694 | if (myhdr.dirty) | ||
695 | { | ||
696 | logf("tagcache is dirty!"); | ||
697 | return false; | ||
698 | } | ||
699 | |||
700 | memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header)); | ||
701 | |||
702 | for (tag = 0; tag < TAG_COUNT; tag++) | ||
703 | { | ||
704 | if (TAGCACHE_IS_NUMERIC(tag)) | ||
705 | continue; | ||
706 | |||
707 | if ( (fd = open_tag_fd(&tch, tag, false)) < 0) | ||
708 | return false; | ||
709 | |||
710 | close(fd); | ||
711 | } | ||
712 | |||
713 | return true; | ||
714 | } | ||
715 | |||
716 | static bool update_master_header(void) | ||
717 | { | ||
718 | struct master_header myhdr; | ||
719 | int fd; | ||
720 | |||
721 | if (!tc_stat.ready) | ||
722 | return false; | ||
723 | |||
724 | if ( (fd = open_master_fd(&myhdr, true)) < 0) | ||
725 | return false; | ||
726 | |||
727 | myhdr.serial = current_tcmh.serial; | ||
728 | myhdr.commitid = current_tcmh.commitid; | ||
729 | myhdr.dirty = current_tcmh.dirty; | ||
730 | |||
731 | /* Write it back */ | ||
732 | lseek(fd, 0, SEEK_SET); | ||
733 | write_master_header(fd, &myhdr); | ||
734 | close(fd); | ||
735 | |||
736 | return true; | ||
737 | } | ||
738 | |||
739 | |||
696 | #ifndef __PCTOOL__ | 740 | #ifndef __PCTOOL__ |
697 | static bool do_timed_yield(void) | 741 | static bool do_timed_yield(void) |
698 | { | 742 | { |
@@ -708,6 +752,44 @@ static bool do_timed_yield(void) | |||
708 | } | 752 | } |
709 | #endif /* __PCTOOL__ */ | 753 | #endif /* __PCTOOL__ */ |
710 | 754 | ||
755 | static void allocate_tempbuf(void) | ||
756 | { | ||
757 | /* Yeah, malloc would be really nice now :) */ | ||
758 | size_t size; | ||
759 | tempbuf_size = 0; | ||
760 | |||
761 | #ifdef __PCTOOL__ | ||
762 | size = 32*1024*1024; | ||
763 | tempbuf = malloc(size); | ||
764 | if (tempbuf) | ||
765 | tempbuf_size = size; | ||
766 | #else /* !__PCTOOL__ */ | ||
767 | /* Need to pass dummy ops to prevent the buffer being moved | ||
768 | * out from under us, since we yield during the tagcache commit. */ | ||
769 | tempbuf_handle = core_alloc_maximum(&size, &buflib_ops_locked); | ||
770 | if (tempbuf_handle > 0) | ||
771 | { | ||
772 | tempbuf = core_get_data(tempbuf_handle); | ||
773 | tempbuf_size = size; | ||
774 | } | ||
775 | #endif /* __PCTOOL__ */ | ||
776 | |||
777 | } | ||
778 | |||
779 | static void free_tempbuf(void) | ||
780 | { | ||
781 | if (tempbuf_size == 0) | ||
782 | return ; | ||
783 | |||
784 | #ifdef __PCTOOL__ | ||
785 | free(tempbuf); | ||
786 | #else | ||
787 | tempbuf_handle = core_free(tempbuf_handle); | ||
788 | #endif | ||
789 | tempbuf = NULL; | ||
790 | tempbuf_size = 0; | ||
791 | } | ||
792 | |||
711 | #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) | 793 | #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
712 | /* find the ramcache entry corresponding to the file indicated by | 794 | /* find the ramcache entry corresponding to the file indicated by |
713 | * filename and dc (it's corresponding dircache id). */ | 795 | * filename and dc (it's corresponding dircache id). */ |
@@ -1636,61 +1718,6 @@ static bool build_lookup_list(struct tagcache_search *tcs) | |||
1636 | } | 1718 | } |
1637 | 1719 | ||
1638 | 1720 | ||
1639 | static void remove_files(void) | ||
1640 | { | ||
1641 | int i; | ||
1642 | char buf[MAX_PATH]; | ||
1643 | const int bufsz = sizeof(buf); | ||
1644 | |||
1645 | tc_stat.ready = false; | ||
1646 | tc_stat.ramcache = false; | ||
1647 | tc_stat.econ = false; | ||
1648 | remove_db_file(TAGCACHE_FILE_MASTER); | ||
1649 | for (i = 0; i < TAG_COUNT; i++) | ||
1650 | { | ||
1651 | if (TAGCACHE_IS_NUMERIC(i)) | ||
1652 | continue; | ||
1653 | |||
1654 | snprintf(buf, bufsz, "%s/" TAGCACHE_FILE_INDEX, | ||
1655 | tc_stat.db_path, i); | ||
1656 | remove(buf); | ||
1657 | } | ||
1658 | } | ||
1659 | |||
1660 | |||
1661 | static bool check_all_headers(void) | ||
1662 | { | ||
1663 | struct master_header myhdr; | ||
1664 | struct tagcache_header tch; | ||
1665 | int tag; | ||
1666 | int fd; | ||
1667 | |||
1668 | if ( (fd = open_master_fd(&myhdr, false)) < 0) | ||
1669 | return false; | ||
1670 | |||
1671 | close(fd); | ||
1672 | if (myhdr.dirty) | ||
1673 | { | ||
1674 | logf("tagcache is dirty!"); | ||
1675 | return false; | ||
1676 | } | ||
1677 | |||
1678 | memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header)); | ||
1679 | |||
1680 | for (tag = 0; tag < TAG_COUNT; tag++) | ||
1681 | { | ||
1682 | if (TAGCACHE_IS_NUMERIC(tag)) | ||
1683 | continue; | ||
1684 | |||
1685 | if ( (fd = open_tag_fd(&tch, tag, false)) < 0) | ||
1686 | return false; | ||
1687 | |||
1688 | close(fd); | ||
1689 | } | ||
1690 | |||
1691 | return true; | ||
1692 | } | ||
1693 | |||
1694 | bool tagcache_search(struct tagcache_search *tcs, int tag) | 1721 | bool tagcache_search(struct tagcache_search *tcs, int tag) |
1695 | { | 1722 | { |
1696 | struct tagcache_header tag_hdr; | 1723 | struct tagcache_header tag_hdr; |
@@ -2005,29 +2032,6 @@ bool tagcache_retrieve(struct tagcache_search *tcs, int idxid, | |||
2005 | return retrieve(tcs, IF_DIRCACHE(idxid,) &idx, tag, buf, size); | 2032 | return retrieve(tcs, IF_DIRCACHE(idxid,) &idx, tag, buf, size); |
2006 | } | 2033 | } |
2007 | 2034 | ||
2008 | static bool update_master_header(void) | ||
2009 | { | ||
2010 | struct master_header myhdr; | ||
2011 | int fd; | ||
2012 | |||
2013 | if (!tc_stat.ready) | ||
2014 | return false; | ||
2015 | |||
2016 | if ( (fd = open_master_fd(&myhdr, true)) < 0) | ||
2017 | return false; | ||
2018 | |||
2019 | myhdr.serial = current_tcmh.serial; | ||
2020 | myhdr.commitid = current_tcmh.commitid; | ||
2021 | myhdr.dirty = current_tcmh.dirty; | ||
2022 | |||
2023 | /* Write it back */ | ||
2024 | lseek(fd, 0, SEEK_SET); | ||
2025 | write_master_header(fd, &myhdr); | ||
2026 | close(fd); | ||
2027 | |||
2028 | return true; | ||
2029 | } | ||
2030 | |||
2031 | void tagcache_search_finish(struct tagcache_search *tcs) | 2035 | void tagcache_search_finish(struct tagcache_search *tcs) |
2032 | { | 2036 | { |
2033 | int i; | 2037 | int i; |
@@ -2366,15 +2370,14 @@ static bool tempbuf_insert(char *str, int id, int idx_id, bool unique) | |||
2366 | struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; | 2370 | struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; |
2367 | int len = strlen(str)+1; | 2371 | int len = strlen(str)+1; |
2368 | int i; | 2372 | int i; |
2369 | unsigned crc32; | ||
2370 | unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4]; | 2373 | unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4]; |
2371 | char buf[TAGCACHE_BUFSZ]; | 2374 | unsigned crc32 = 0xffffffff; |
2372 | const int bufsz = sizeof(buf); | 2375 | char chr_lower; |
2373 | 2376 | for (i = 0; str[i] != '\0' && i < len; i++) | |
2374 | for (i = 0; str[i] != '\0' && i < bufsz-1; i++) | 2377 | { |
2375 | buf[i] = tolower(str[i]); | 2378 | chr_lower = tolower(str[i]); |
2376 | 2379 | crc32 = crc_32(&chr_lower, 1, crc32); | |
2377 | crc32 = crc_32(buf, i, 0xffffffff); | 2380 | } |
2378 | 2381 | ||
2379 | if (unique) | 2382 | if (unique) |
2380 | { | 2383 | { |
@@ -2568,8 +2571,8 @@ static bool build_numeric_indices(struct tagcache_header *h, int tmpfd) | |||
2568 | int entries_processed = 0; | 2571 | int entries_processed = 0; |
2569 | int i, j; | 2572 | int i, j; |
2570 | 2573 | ||
2571 | char buf[TAGCACHE_BUFSZ]; | 2574 | char *buf = tmp_buf; |
2572 | const int bufsz = sizeof(buf); | 2575 | const long bufsz = sizeof(tmp_buf); |
2573 | 2576 | ||
2574 | max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1; | 2577 | max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1; |
2575 | 2578 | ||
@@ -2808,8 +2811,8 @@ static int build_index(int index_type, struct tagcache_header *h, int tmpfd) | |||
2808 | struct master_header tcmh; | 2811 | struct master_header tcmh; |
2809 | struct index_entry idxbuf[IDX_BUF_DEPTH]; | 2812 | struct index_entry idxbuf[IDX_BUF_DEPTH]; |
2810 | int idxbuf_pos; | 2813 | int idxbuf_pos; |
2811 | char buf[TAGCACHE_BUFSZ]; | 2814 | char *buf = tmp_buf; |
2812 | const long bufsz = sizeof(buf); | 2815 | const long bufsz = sizeof(tmp_buf); |
2813 | int fd = -1, masterfd; | 2816 | int fd = -1, masterfd; |
2814 | bool error = false; | 2817 | bool error = false; |
2815 | int init; | 2818 | int init; |
@@ -2947,6 +2950,7 @@ static int build_index(int index_type, struct tagcache_header *h, int tmpfd) | |||
2947 | } | 2950 | } |
2948 | else | 2951 | else |
2949 | { | 2952 | { |
2953 | logf("Create New Index: %d", index_type); | ||
2950 | /** | 2954 | /** |
2951 | * Creating new index file to store the tags. No need to preload | 2955 | * Creating new index file to store the tags. No need to preload |
2952 | * anything whether the index type is sorted or not. | 2956 | * anything whether the index type is sorted or not. |
@@ -3408,6 +3412,7 @@ static bool commit(void) | |||
3408 | tc_stat.commit_step = 0; | 3412 | tc_stat.commit_step = 0; |
3409 | goto commit_error; | 3413 | goto commit_error; |
3410 | } | 3414 | } |
3415 | do_timed_yield(); | ||
3411 | } | 3416 | } |
3412 | 3417 | ||
3413 | if (!build_numeric_indices(&tch, tmpfd)) | 3418 | if (!build_numeric_indices(&tch, tmpfd)) |
@@ -3443,7 +3448,7 @@ static bool commit(void) | |||
3443 | tc_stat.ready = check_all_headers(); | 3448 | tc_stat.ready = check_all_headers(); |
3444 | tc_stat.readyvalid = true; | 3449 | tc_stat.readyvalid = true; |
3445 | 3450 | ||
3446 | #ifdef HAVE_TC_RAMCACHE | 3451 | #if defined(HAVE_TC_RAMCACHE) |
3447 | if (ramcache_buffer_stolen) | 3452 | if (ramcache_buffer_stolen) |
3448 | { | 3453 | { |
3449 | tempbuf = NULL; | 3454 | tempbuf = NULL; |
@@ -3483,6 +3488,7 @@ commit_error: | |||
3483 | return rc; | 3488 | return rc; |
3484 | } | 3489 | } |
3485 | 3490 | ||
3491 | |||
3486 | #ifndef __PCTOOL__ | 3492 | #ifndef __PCTOOL__ |
3487 | 3493 | ||
3488 | static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data) | 3494 | static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data) |