diff --git a/src/lib/libc/stdlib/malloc.c b/src/lib/libc/stdlib/malloc.c index aedb4c8f..661265bb 100644 --- a/src/lib/libc/stdlib/malloc.c +++ b/src/lib/libc/stdlib/malloc.c @@ -8,7 +8,7 @@ */ #if defined(LIBC_SCCS) && !defined(lint) -static char rcsid[] = "$OpenBSD: malloc.c,v 1.70 2004/08/05 21:55:21 tdeval Exp $"; +static char rcsid[] = "$OpenBSD: malloc.c,v 1.71 2004/08/11 06:22:45 tdeval Exp $"; #endif /* LIBC_SCCS and not lint */ /* @@ -38,8 +38,6 @@ static char rcsid[] = "$OpenBSD: malloc.c,v 1.70 2004/08/05 21:55:21 tdeval Exp #define SOME_JUNK 0xd0 /* as in "Duh" :-) */ #include -#include -#include #include #include #include @@ -81,11 +79,11 @@ static char rcsid[] = "$OpenBSD: malloc.c,v 1.70 2004/08/05 21:55:21 tdeval Exp struct pginfo { struct pginfo *next; /* next on the free list */ void *page; /* Pointer to the page */ - u_short size; /* size of this page's chunks */ - u_short shift; /* How far to shift for this size chunks */ - u_short free; /* How many free chunks */ - u_short total; /* How many chunk */ - u_long bits[1]; /* Which chunks are free */ + u_short size; /* size of this page's chunks */ + u_short shift; /* How far to shift for this size chunks */ + u_short free; /* How many free chunks */ + u_short total; /* How many chunk */ + u_long bits[1]; /* Which chunks are free */ }; /* @@ -96,8 +94,8 @@ struct pgfree { struct pgfree *next; /* next run of free pages */ struct pgfree *prev; /* prev run of free pages */ void *page; /* pointer to free pages */ - void *pdir; /* pointer to the base page's dir */ - size_t size; /* number of bytes free */ + void *end; /* pointer to end of free pages */ + u_long size; /* number of bytes free */ }; /* @@ -142,8 +140,8 @@ struct pgfree { /* A mask for the offset inside a page. */ #define malloc_pagemask ((malloc_pagesize)-1) -#define pageround(foo) (((foo) + (malloc_pagemask)) & ~malloc_pagemask) -#define ptr2index(foo) (((u_long)(foo) >> malloc_pageshift)+malloc_pageshift) +#define pageround(foo) (((foo) + (malloc_pagemask))&(~(malloc_pagemask))) +#define ptr2index(foo) (((u_long)(foo) >> malloc_pageshift)-malloc_origo) /* fd of /dev/zero */ #ifdef USE_DEV_ZERO @@ -163,22 +161,8 @@ static unsigned int malloc_started; /* Number of free pages we cache */ static unsigned int malloc_cache = 16; -/* Structure used for linking discrete directory pages. */ -struct pdinfo { - struct pginfo **base; - struct pdinfo *prev; - struct pdinfo *next; - u_long dirnum; -}; -static struct pdinfo *last_dir; /* Caches to the last and previous */ -static struct pdinfo *prev_dir; /* referenced directory pages. */ - -static size_t pdi_off; -static u_long pdi_mod; -#define PD_IDX(num) ((num) / (malloc_pagesize/sizeof(struct pginfo *))) -#define PD_OFF(num) ((num) & ((malloc_pagesize/sizeof(struct pginfo *))-1)) -#define PI_IDX(index) ((index) / pdi_mod) -#define PI_OFF(index) ((index) % pdi_mod) +/* The offset from pagenumber to index into the page directory */ +static u_long malloc_origo; /* The last index in the page directory we care about */ static u_long last_index; @@ -247,144 +231,72 @@ void utrace(struct ut *, int); /* Status of malloc. */ static int malloc_active; -/* Allocated memory. */ -static size_t malloc_used; - -/* My last break. */ +/* my last break. */ static void *malloc_brk; -/* One location cache for free-list holders. */ +/* one location cache for free-list holders */ static struct pgfree *px; -/* Compile-time options. */ +/* compile-time options */ char *malloc_options; -/* Name of the current public function. */ +/* Name of the current public function */ static char *malloc_func; -/* Macro for mmap. */ +/* Macro for mmap */ #define MMAP(size) \ mmap((void *)0, (size), PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, \ MMAP_FD, (off_t)0) /* - * Necessary function declarations. + * Necessary function declarations */ +static int extend_pgdir(u_long index); static void *imalloc(size_t size); static void ifree(void *ptr); static void *irealloc(void *ptr, size_t size); static void *malloc_bytes(size_t size); - -/* - * Function for page directory lookup. - */ -static int -pdir_lookup(u_long index, struct pdinfo **pdi) -{ - struct pdinfo *spi; - u_long pidx = PI_IDX(index); - - if (last_dir != NULL && PD_IDX(last_dir->dirnum) == pidx) - *pdi = last_dir; - else if (prev_dir != NULL && PD_IDX(prev_dir->dirnum) == pidx) - *pdi = prev_dir; - else if (last_dir != NULL && prev_dir != NULL) { - if ((PD_IDX(last_dir->dirnum) > pidx) ? - (PD_IDX(last_dir->dirnum) - pidx):(pidx - PD_IDX(last_dir->dirnum)) - < (PD_IDX(prev_dir->dirnum) > pidx) ? - (PD_IDX(prev_dir->dirnum) - pidx):(pidx - PD_IDX(prev_dir->dirnum))) - *pdi = last_dir; - else - *pdi = prev_dir; - - if (PD_IDX((*pdi)->dirnum) > pidx) { - for (spi=(*pdi)->prev;spi!=NULL && PD_IDX(spi->dirnum)>pidx; - spi=spi->prev) - *pdi = spi; - if (spi != NULL) - *pdi = spi; - } else - for (spi=(*pdi)->next;spi!=NULL && PD_IDX(spi->dirnum)<=pidx; - spi=spi->next) - *pdi = spi; - } else { - *pdi = (struct pdinfo *)((caddr_t)page_dir + pdi_off); - for (spi=*pdi;spi!=NULL && PD_IDX(spi->dirnum)<=pidx;spi=spi->next) - *pdi = spi; - } - - return ((PD_IDX((*pdi)->dirnum) == pidx)?0:(PD_IDX((*pdi)->dirnum) > pidx)?1:-1); -} - - #ifdef MALLOC_STATS void malloc_dump(FILE *fd) { struct pginfo **pd; struct pgfree *pf; - struct pdinfo *pi; int j; pd = page_dir; - pi = (struct pdinfo *)((caddr_t)pd + pdi_off); /* print out all the pages */ - for(j=0;j<=last_index;) { - fprintf(fd, "%08lx %5d ", j << malloc_pageshift, j); - if (pd[PI_OFF(j)] == MALLOC_NOT_MINE) { - for(j++;j<=last_index && pd[PI_OFF(j)] == MALLOC_NOT_MINE;) { - if (!PI_OFF(++j)) { - if ((pi = pi->next) == NULL || - PD_IDX(pi->dirnum) != PI_IDX(j)) break; - pd = pi->base; - j += pdi_mod; - } - } + for(j=0;j<=last_index;j++) { + fprintf(fd, "%08lx %5d ", (j+malloc_origo) << malloc_pageshift, j); + if (pd[j] == MALLOC_NOT_MINE) { + for(j++;j<=last_index && pd[j] == MALLOC_NOT_MINE;j++) + ; j--; fprintf(fd, ".. %5d not mine\n", j); - } else if (pd[PI_OFF(j)] == MALLOC_FREE) { - for(j++;j<=last_index && pd[PI_OFF(j)] == MALLOC_FREE;) { - if (!PI_OFF(++j)) { - if ((pi = pi->next) == NULL || - PD_IDX(pi->dirnum) != PI_IDX(j)) break; - pd = pi->base; - j += pdi_mod; - } - } + } else if (pd[j] == MALLOC_FREE) { + for(j++;j<=last_index && pd[j] == MALLOC_FREE;j++) + ; j--; fprintf(fd, ".. %5d free\n", j); - } else if (pd[PI_OFF(j)] == MALLOC_FIRST) { - for(j++;j<=last_index && pd[PI_OFF(j)] == MALLOC_FOLLOW;) { - if (!PI_OFF(++j)) { - if ((pi = pi->next) == NULL || - PD_IDX(pi->dirnum) != PI_IDX(j)) break; - pd = pi->base; - j += pdi_mod; - } - } + } else if (pd[j] == MALLOC_FIRST) { + for(j++;j<=last_index && pd[j] == MALLOC_FOLLOW;j++) + ; j--; fprintf(fd, ".. %5d in use\n", j); - } else if (pd[PI_OFF(j)] < MALLOC_MAGIC) { - fprintf(fd, "(%p)\n", pd[PI_OFF(j)]); + } else if (pd[j] < MALLOC_MAGIC) { + fprintf(fd, "(%p)\n", pd[j]); } else { fprintf(fd, "%p %d (of %d) x %d @ %p --> %p\n", - pd[PI_OFF(j)], pd[PI_OFF(j)]->free, pd[PI_OFF(j)]->total, - pd[PI_OFF(j)]->size, pd[PI_OFF(j)]->page, pd[PI_OFF(j)]->next); - } - if (!PI_OFF(++j)) { - if ((pi = pi->next) == NULL) - break; - pd = pi->base; - j += (1 + PD_IDX(pi->dirnum) - PI_IDX(j)) * pdi_mod; + pd[j], pd[j]->free, pd[j]->total, + pd[j]->size, pd[j]->page, pd[j]->next); } } for(pf=free_list.next; pf; pf=pf->next) { fprintf(fd, "Free: @%p [%p...%p[ %ld ->%p <-%p\n", - pf, pf->page, pf->page + pf->size, pf->size, - pf->prev, pf->next); + pf, pf->page, pf->end, pf->size, pf->prev, pf->next); if (pf == pf->next) { fprintf(fd, "Free_list loops\n"); break; @@ -396,7 +308,10 @@ malloc_dump(FILE *fd) fprintf(fd, "Maxsize\t%d\n", malloc_maxsize); fprintf(fd, "Pagesize\t%lu\n", (u_long)malloc_pagesize); fprintf(fd, "Pageshift\t%d\n", malloc_pageshift); - fprintf(fd, "In use\t%lu\n", (u_long)malloc_used); + fprintf(fd, "FirstPage\t%ld\n", malloc_origo); + fprintf(fd, "LastPage\t%ld %lx\n", last_index+malloc_pageshift, + (last_index + malloc_pageshift) << malloc_pageshift); + fprintf(fd, "Break\t%ld\n", (u_long)sbrk(0) >> malloc_pageshift); } #endif /* MALLOC_STATS */ @@ -424,8 +339,7 @@ wrterror(char *p) malloc_dump(stderr); #endif /* MALLOC_STATS */ malloc_active--; - if (malloc_abort) - abort(); + abort(); } static void @@ -471,86 +385,89 @@ malloc_exit(void) static void * map_pages(size_t pages) { - struct pdinfo *pi, *spi; - struct pginfo **pd; - u_long pidx,lidx; - void *result, *tail; - u_long index; + caddr_t result, tail; + result = (caddr_t)pageround((u_long)sbrk(0)); pages <<= malloc_pageshift; - result = MMAP(pages + malloc_guard); - if (result == MAP_FAILED) { - errno = ENOMEM; + if (pages > SIZE_T_MAX - (size_t)result) { #ifdef MALLOC_EXTRA_SANITY - wrtwarning("(ES): map_pages fails\n"); + wrtwarning("(ES): overflow in map_pages fails\n"); #endif /* MALLOC_EXTRA_SANITY */ + errno = ENOMEM; return (NULL); } tail = result + pages + malloc_guard; - if (malloc_guard) - mprotect(result + pages, malloc_guard, PROT_NONE); - - if (tail > malloc_brk) - malloc_brk = tail; - if ((index = ptr2index(tail) - 1) > last_index) - last_index = index; - - /* Insert directory pages, if needed. */ - pidx = PI_IDX(ptr2index(result)); - lidx = PI_IDX(index); - pdir_lookup(ptr2index(result), &pi); - - for (index=pidx,spi=pi;index<=lidx;index++) { - if (pi == NULL || PD_IDX(pi->dirnum) != index) { - if ((pd = MMAP(malloc_pagesize)) == MAP_FAILED) { - errno = ENOMEM; - munmap(result, tail - result); + if (brk(tail) == (char *)-1) { #ifdef MALLOC_EXTRA_SANITY - wrtwarning("(ES): map_pages fails\n"); -#endif /* MALLOC_EXTRA_SANITY */ - return (NULL); - } - memset(pd, 0, malloc_pagesize); - pi = (struct pdinfo *)((caddr_t)pd + pdi_off); - pi->base = pd; - pi->prev = spi; - pi->next = spi->next; - pi->dirnum = index * (malloc_pagesize/sizeof(struct pginfo *)); - - if (spi->next != NULL) - spi->next->prev = pi; - spi->next = pi; - } - if (index > pidx && index < lidx) { - pi->dirnum += pdi_mod; - } else if (index == pidx) { - if (pidx == lidx) { - pi->dirnum += (tail - result) >> malloc_pageshift; - } else { - pi->dirnum += pdi_mod - PI_OFF(ptr2index(result)); - } - } else { - pi->dirnum += PI_OFF(ptr2index(tail - 1)) + 1; - } -#ifdef MALLOC_EXTRA_SANITY - if (PD_OFF(pi->dirnum) > pdi_mod || PD_IDX(pi->dirnum) > index) { - wrterror("(ES): pages directory overflow\n"); - errno = EFAULT; - return (NULL); - } + wrtwarning("(ES): map_pages fails\n"); #endif /* MALLOC_EXTRA_SANITY */ - if (index == pidx && pi != last_dir) { - prev_dir = last_dir; - last_dir = pi; - } - spi = pi; - pi = spi->next; + return (NULL); } + if (malloc_guard) + mprotect(result + pages, malloc_pagesize, PROT_NONE); + + last_index = ptr2index(tail) - 1; + malloc_brk = tail; + + if ((last_index+1) >= malloc_ninfo && !extend_pgdir(last_index)) + return (NULL); return (result); } +/* + * Extend page directory + */ +static int +extend_pgdir(u_long index) +{ + struct pginfo **new, **old; + size_t i, oldlen; + + /* Make it this many pages */ + i = index * sizeof *page_dir; + i /= malloc_pagesize; + i += 2; + + /* remember the old mapping size */ + oldlen = malloc_ninfo * sizeof *page_dir; + + /* + * NOTE: we allocate new pages and copy the directory rather than tempt + * fate by trying to "grow" the region.. There is nothing to prevent + * us from accidently re-mapping space that's been allocated by our caller + * via dlopen() or other mmap(). + * + * The copy problem is not too bad, as there is 4K of page index per + * 4MB of malloc arena. + * + * We can totally avoid the copy if we open a file descriptor to associate + * the anon mappings with. Then, when we remap the pages at the new + * address, the old pages will be "magically" remapped.. But this means + * keeping open a "secret" file descriptor..... + */ + + /* Get new pages */ + new = (struct pginfo**) MMAP(i * malloc_pagesize); + if (new == MAP_FAILED) + return (0); + + /* Copy the old stuff */ + memcpy(new, page_dir, + malloc_ninfo * sizeof *page_dir); + + /* register the new size */ + malloc_ninfo = i * malloc_pagesize / sizeof *page_dir; + + /* swap the pointers */ + old = page_dir; + page_dir = new; + + /* Now free the old stuff */ + munmap(old, oldlen); + return (1); +} /* * Initialize the world @@ -603,12 +520,12 @@ malloc_init(void) case 'h': malloc_hint = 0; break; case 'H': malloc_hint = 1; break; #endif /* __FreeBSD__ */ + case 'r': malloc_realloc = 0; break; + case 'R': malloc_realloc = 1; break; case 'j': malloc_junk = 0; break; case 'J': malloc_junk = 1; break; case 'n': malloc_silent = 0; break; case 'N': malloc_silent = 1; break; - case 'r': malloc_realloc = 0; break; - case 'R': malloc_realloc = 1; break; #ifdef __FreeBSD__ case 'u': malloc_utrace = 0; break; case 'U': malloc_utrace = 1; break; @@ -644,21 +561,17 @@ malloc_init(void) /* Allocate one page for the page directory */ page_dir = (struct pginfo **) MMAP(malloc_pagesize); - if (page_dir == MAP_FAILED) { + if (page_dir == MAP_FAILED) wrterror("mmap(2) failed, check limits\n"); - errno = ENOMEM; - return; - } - pdi_off = (malloc_pagesize - sizeof(struct pdinfo)) & ~(malloc_minsize - 1); - pdi_mod = pdi_off / sizeof(struct pginfo *); - - last_dir = (struct pdinfo *)((caddr_t)page_dir + pdi_off); - last_dir->base = page_dir; - last_dir->prev = last_dir->next = NULL; - last_dir->dirnum = malloc_pageshift; + /* + * We need a maximum of malloc_pageshift buckets, steal these from the + * front of the page_directory; + */ + malloc_origo = ((u_long)pageround((u_long)sbrk(0))) >> malloc_pageshift; + malloc_origo -= malloc_pageshift; - malloc_ninfo = pdi_mod; + malloc_ninfo = malloc_pagesize / sizeof *page_dir; /* Been here, done that */ malloc_started++; @@ -670,6 +583,11 @@ malloc_init(void) malloc_cache <<= malloc_pageshift; + /* + * This is a nice hack from Kaleb Keithly (kaleb@x.org). + * We can sbrk(2) further back when we keep this on a low address. + */ + px = (struct pgfree *) imalloc (sizeof *px); errno = save_errno; } @@ -681,66 +599,30 @@ malloc_pages(size_t size) { void *p, *delay_free = NULL; int i; - struct rlimit rl; - struct pginfo **pd; - struct pdinfo *pi; - u_long pidx; - void *tp; struct pgfree *pf; u_long index; - int m; size = pageround(size) + malloc_guard; p = NULL; /* Look for free pages before asking for more */ - for (pf = free_list.next; pf; pf = pf->next) { + for(pf = free_list.next; pf; pf = pf->next) { #ifdef MALLOC_EXTRA_SANITY - if (pf->size & malloc_pagemask) { + if (pf->size & malloc_pagemask) wrterror("(ES): junk length entry on free_list\n"); - errno = EFAULT; - return (NULL); - } - if (!pf->size) { + if (!pf->size) wrterror("(ES): zero length entry on free_list\n"); - errno = EFAULT; - return (NULL); - } - if (pf->page > (pf->page + pf->size)) { + if (pf->page == pf->end) + wrterror("(ES): zero entry on free_list\n"); + if (pf->page > pf->end) wrterror("(ES): sick entry on free_list\n"); - errno = EFAULT; - return (NULL); - } - if ((pi = pf->pdir) == NULL) { - wrterror("(ES): invalid page directory on free-list\n"); - errno = EFAULT; - return (NULL); - } - if ((pidx = PI_IDX(ptr2index(pf->page))) != PD_IDX(pi->dirnum)) { - wrterror("(ES): directory index mismatch on free-list\n"); - errno = EFAULT; - return (NULL); - } - pd = pi->base; - if (pd[PI_OFF(ptr2index(pf->page))] != MALLOC_FREE) { + if ((void*)pf->page >= (void*)sbrk(0)) + wrterror("(ES): entry on free_list past brk\n"); + if (page_dir[ptr2index(pf->page)] != MALLOC_FREE) wrterror("(ES): non-free first page on free-list\n"); - errno = EFAULT; - return (NULL); - } - pidx = PI_IDX(ptr2index((pf->page)+(pf->size))-1); - for (pi=pf->pdir; pi!=NULL && PD_IDX(pi->dirnum)next); - if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { - wrterror("(ES): last page not referenced in page directory\n"); - errno = EFAULT; - return (NULL); - } - pd = pi->base; - if (pd[PI_OFF(ptr2index((pf->page)+(pf->size))-1)] != MALLOC_FREE) { + if (page_dir[ptr2index(pf->end)-1] != MALLOC_FREE) wrterror("(ES): non-free last page on free-list\n"); - errno = EFAULT; - return (NULL); - } #endif /* MALLOC_EXTRA_SANITY */ if (pf->size < size) @@ -748,7 +630,6 @@ malloc_pages(size_t size) if (pf->size == size) { p = pf->page; - pi = pf->pdir; if (pf->next != NULL) pf->next->prev = pf->prev; pf->prev->next = pf->next; @@ -759,34 +640,17 @@ malloc_pages(size_t size) p = pf->page; pf->page = (char *)pf->page + size; pf->size -= size; - pidx = PI_IDX(ptr2index(pf->page)); - for (pi=pf->pdir; pi!=NULL && PD_IDX(pi->dirnum)next); - if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { - wrterror("(ES): hole in directories\n"); - errno = EFAULT; - return (NULL); - } - tp = pf->pdir; - pf->pdir = pi; - pi = tp; break; } size -= malloc_guard; -#ifdef MALLOC_EXTRA_SANITY - if (p != NULL && pi != NULL) { - pidx = PD_IDX(pi->dirnum); - pd = pi->base; - } - if (p != NULL && pd[PI_OFF(ptr2index(p))] != MALLOC_FREE) { +#ifdef MALLOC_EXTRA_SANITY + if (p != NULL && page_dir[ptr2index(p)] != MALLOC_FREE) wrterror("(ES): allocated non-free page on free-list\n"); - errno = EFAULT; - return (NULL); - } #endif /* MALLOC_EXTRA_SANITY */ - if (p != NULL && (malloc_guard || malloc_freeprot)) + if ((malloc_guard || malloc_freeprot) && p != NULL) mprotect(p, size, PROT_READ|PROT_WRITE); size >>= malloc_pageshift; @@ -798,53 +662,9 @@ malloc_pages(size_t size) if (p != NULL) { index = ptr2index(p); - pidx = PI_IDX(index); - pdir_lookup(index, &pi); -#ifdef MALLOC_EXTRA_SANITY - if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { - wrterror("(ES): mapped pages not found in directory\n"); - errno = EFAULT; - return (NULL); - } -#endif /* MALLOC_EXTRA_SANITY */ - if (pi != last_dir) { - prev_dir = last_dir; - last_dir = pi; - } - pd = pi->base; - pd[PI_OFF(index)] = MALLOC_FIRST; - for (i=1;inext; -#ifdef MALLOC_EXTRA_SANITY - if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { - wrterror("(ES): hole in mapped pages directory\n"); - errno = EFAULT; - return (NULL); - } -#endif /* MALLOC_EXTRA_SANITY */ - pd = pi->base; - } - pd[PI_OFF(index+i)] = MALLOC_FOLLOW; - } - if (malloc_guard) { - if (!PI_OFF(index+i)) { - pidx++; - pi = pi->next; -#ifdef MALLOC_EXTRA_SANITY - if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { - wrterror("(ES): hole in mapped pages directory\n"); - errno = EFAULT; - return (NULL); - } -#endif /* MALLOC_EXTRA_SANITY */ - pd = pi->base; - } - pd[PI_OFF(index+i)] = MALLOC_FIRST; - } - - malloc_used += size << malloc_pageshift; + page_dir[index] = MALLOC_FIRST; + for (i=1;idirnum) != pidx) { - wrterror("(ES): mapped pages not found in directory\n"); - errno = EFAULT; - return (0); - } -#endif /* MALLOC_EXTRA_SANITY */ - if (pi != last_dir) { - prev_dir = last_dir; - last_dir = pi; - } - pd = pi->base; - pd[PI_OFF(ptr2index(pp))] = bp; + page_dir[ptr2index(pp)] = bp; bp->next = page_dir[bits]; page_dir[bits] = bp; @@ -1016,7 +819,7 @@ malloc_bytes(size_t size) u += u; k++; } - + if (malloc_guard) { /* Walk to a random position. */ i = arc4random() % bp->free; @@ -1029,14 +832,11 @@ malloc_bytes(size_t size) k = 0; } #ifdef MALLOC_EXTRA_SANITY - if (lp - bp->bits > (bp->total - 1) / MALLOC_BITS) { + if (lp - bp->bits > (bp->total - 1) / MALLOC_BITS) wrterror("chunk overflow\n"); - errno = EFAULT; - return (NULL); - } #endif /* MALLOC_EXTRA_SANITY */ - if (*lp & u) - i--; + if (*lp & u) + i--; } } *lp ^= u; @@ -1096,11 +896,9 @@ static void * irealloc(void *ptr, size_t size) { void *p; - u_long osize, index, i; + u_long osize, index; struct pginfo **mp; - struct pginfo **pd; - struct pdinfo *pi; - u_long pidx; + int i; if (suicide) abort(); @@ -1122,22 +920,7 @@ irealloc(void *ptr, size_t size) return (NULL); } - pidx = PI_IDX(index); - pdir_lookup(index, &pi); -#ifdef MALLOC_EXTRA_SANITY - if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { - wrterror("(ES): mapped pages not found in directory\n"); - errno = EFAULT; - return (NULL); - } -#endif /* MALLOC_EXTRA_SANITY */ - if (pi != last_dir) { - prev_dir = last_dir; - last_dir = pi; - } - - pd = pi->base; - mp = &pd[PI_OFF(index)]; + mp = &page_dir[index]; if (*mp == MALLOC_FIRST) { /* Page allocation */ @@ -1148,25 +931,8 @@ irealloc(void *ptr, size_t size) } /* Find the size in bytes */ - i = index; - if (!PI_OFF(++i)) { - pi = pi->next; - if (pi != NULL && PD_IDX(pi->dirnum) != PI_IDX(i)) - pi = NULL; - if (pi != NULL) - pd = pi->base; - } - for (osize = malloc_pagesize; - pi != NULL && pd[PI_OFF(i)] == MALLOC_FOLLOW;) { + for (osize = malloc_pagesize; *(++mp) == MALLOC_FOLLOW;) osize += malloc_pagesize; - if (!PI_OFF(++i)) { - pi = pi->next; - if (pi != NULL && PD_IDX(pi->dirnum) != PI_IDX(i)) - pi = NULL; - if (pi != NULL) - pd = pi->base; - } - } if (!malloc_realloc && /* Unless we have to, */ size <= osize && /* .. or are too small, */ @@ -1222,7 +988,6 @@ irealloc(void *ptr, size_t size) } ifree(ptr); } - return (p); } @@ -1234,9 +999,6 @@ static __inline__ void free_pages(void *ptr, u_long index, struct pginfo *info) { u_long i, l; - struct pginfo **pd; - struct pdinfo *pi, *spi; - u_long pidx, lidx; struct pgfree *pf, *pt=NULL; void *tail; @@ -1256,77 +1018,40 @@ free_pages(void *ptr, u_long index, struct pginfo *info) } /* Count how many pages and mark them free at the same time */ - pidx = PI_IDX(index); - pdir_lookup(index, &pi); -#ifdef MALLOC_EXTRA_SANITY - if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { - wrterror("(ES): mapped pages not found in directory\n"); - errno = EFAULT; - return; - } -#endif /* MALLOC_EXTRA_SANITY */ - - spi = pi; /* Save page index for start of region. */ - - pd = pi->base; - pd[PI_OFF(index)] = MALLOC_FREE; - i = 1; - if (!PI_OFF(index+i)) { - pi = pi->next; - if (pi == NULL || PD_IDX(pi->dirnum) != PI_IDX(index+i)) - pi = NULL; - else - pd = pi->base; - } - while (pi != NULL && pd[PI_OFF(index+i)] == MALLOC_FOLLOW) { - pd[PI_OFF(index+i)] = MALLOC_FREE; - i++; - if (!PI_OFF(index+i)) { - if ((pi=pi->next) == NULL || PD_IDX(pi->dirnum) != PI_IDX(index+i)) - pi = NULL; - else - pd = pi->base; - } - } + page_dir[index] = MALLOC_FREE; + for (i = 1; page_dir[index+i] == MALLOC_FOLLOW; i++) + page_dir[index + i] = MALLOC_FREE; l = i << malloc_pageshift; if (malloc_junk) memset(ptr, SOME_JUNK, l); - malloc_used -= l; - if (malloc_guard) { -#ifdef MALLOC_EXTRA_SANITY - if (pi == NULL || PD_IDX(pi->dirnum) != PI_IDX(index+i)) { - wrterror("(ES): hole in mapped pages directory\n"); - errno = EFAULT; - return; - } -#endif /* MALLOC_EXTRA_SANITY */ - pd[PI_OFF(index+i)] = MALLOC_FREE; - l += malloc_guard; - } - tail = (char *)ptr + l; - #if defined(__FreeBSD__) || (defined(__OpenBSD__) && defined(MADV_FREE)) if (malloc_hint) madvise(ptr, l, MADV_FREE); #endif + if (malloc_guard) { + page_dir[index + i] = MALLOC_FREE; + l += malloc_guard; + } + tail = (char *)ptr+l; + if (malloc_freeprot) - mprotect(ptr, l, PROT_NONE); + mprotect(ptr, tail - ptr, PROT_NONE); - /* Add to free-list. */ + /* add to free-list */ if (px == NULL) px = imalloc(sizeof *px); /* This cannot fail... */ px->page = ptr; - px->pdir = spi; + px->end = tail; px->size = l; if (free_list.next == NULL) { - /* Nothing on free list, put this at head. */ - px->next = NULL; + /* Nothing on free list, put this at head */ + px->next = free_list.next; px->prev = &free_list; free_list.next = px; pf = px; @@ -1336,9 +1061,9 @@ free_pages(void *ptr, u_long index, struct pginfo *info) /* Find the right spot, leave pf pointing to the modified entry. */ - for(pf = free_list.next; (pf->page+pf->size) < ptr && pf->next != NULL; + for(pf = free_list.next; pf->end < ptr && pf->next != NULL; pf = pf->next) - ; /* Race ahead here. */ + ; /* Race ahead here */ if (pf->page > tail) { /* Insert before entry */ @@ -1348,24 +1073,25 @@ free_pages(void *ptr, u_long index, struct pginfo *info) px->prev->next = px; pf = px; px = NULL; - } else if ((pf->page + pf->size) == ptr ) { - /* Append to the previous entry. */ + } else if (pf->end == ptr ) { + /* Append to the previous entry */ + pf->end = (char *)pf->end + l; pf->size += l; - if (pf->next != NULL && (pf->page + pf->size) == pf->next->page ) { + if (pf->next != NULL && pf->end == pf->next->page ) { /* And collapse the next too. */ pt = pf->next; + pf->end = pt->end; pf->size += pt->size; pf->next = pt->next; if (pf->next != NULL) pf->next->prev = pf; } } else if (pf->page == tail) { - /* Prepend to entry. */ + /* Prepend to entry */ pf->size += l; pf->page = ptr; - pf->pdir = spi; } else if (pf->next == NULL) { - /* Append at tail of chain. */ + /* Append at tail of chain */ px->next = NULL; px->prev = pf; pf->next = px; @@ -1373,80 +1099,34 @@ free_pages(void *ptr, u_long index, struct pginfo *info) px = NULL; } else { wrterror("freelist is destroyed\n"); - errno = EFAULT; - return; } } - if (pf->pdir != last_dir) { - prev_dir = last_dir; - last_dir = pf->pdir; - } - /* Return something to OS ? */ if (pf->next == NULL && /* If we're the last one, */ pf->size > malloc_cache && /* ..and the cache is full, */ - (pf->page + pf->size) == malloc_brk) { /* ..and none behind us, */ + pf->end == malloc_brk && /* ..and none behind us, */ + malloc_brk == sbrk(0)) { /* ..and it's OK to do... */ /* * Keep the cache intact. Notice that the '>' above guarantees that * the pf will always have at least one page afterwards. */ - if (munmap((char *)pf->page + malloc_cache, pf->size - malloc_cache)!=0) - goto not_return; - tail = pf->page + pf->size; - lidx = ptr2index(tail) - 1; + pf->end = (char *)pf->page + malloc_cache; pf->size = malloc_cache; - malloc_brk = pf->page + malloc_cache; - - index = ptr2index(malloc_brk); - - pidx = PI_IDX(index); - if (prev_dir != NULL && PD_IDX(prev_dir->dirnum) >= pidx) - prev_dir = NULL; /* Will be wiped out below ! */ + brk(pf->end); + malloc_brk = pf->end; - for (pi=pf->pdir; pi!=NULL && PD_IDX(pi->dirnum)next); + index = ptr2index(pf->end); - if (pi != NULL && PD_IDX(pi->dirnum) == pidx) { - pd = pi->base; - - for(i=index;i <= last_index;) { - if (pd[PI_OFF(i)] != MALLOC_NOT_MINE) { - pd[PI_OFF(i)] = MALLOC_NOT_MINE; -#ifdef MALLOC_EXTRA_SANITY - if (!PD_OFF(pi->dirnum)) { - wrterror("(ES): pages directory underflow\n"); - errno = EFAULT; - return; - } -#endif /* MALLOC_EXTRA_SANITY */ - pi->dirnum--; - } - i++; - if (!PI_OFF(i)) { - /* If no page in that dir, free directory page. */ - if (!PD_OFF(pi->dirnum)) { - /* Remove from list. */ - pi->prev->next = pi->next; - if (pi->next != NULL) - pi->next->prev = pi->prev; - pi = pi->next; - munmap(pd, malloc_pagesize); - } else - pi = pi->next; - if (pi == NULL || PD_IDX(pi->dirnum) != PI_IDX(i)) - break; - pd = pi->base; - } - } - } + for(i=index;i <= last_index;) + page_dir[i++] = MALLOC_NOT_MINE; last_index = index - 1; /* XXX: We could realloc/shrink the pagedir here I guess. */ } -not_return: if (pt != NULL) ifree(pt); } @@ -1461,9 +1141,6 @@ free_bytes(void *ptr, int index, struct pginfo *info) { int i; struct pginfo **mp; - struct pginfo **pd; - struct pdinfo *pi; - u_long pidx; void *vp; /* Find the chunk number on the page */ @@ -1495,8 +1172,7 @@ free_bytes(void *ptr, int index, struct pginfo *info) /* Page became non-full */ /* Insert in address order */ - while (*mp != NULL && (*mp)->next != NULL && - (*mp)->next->page < info->page) + while (*mp && (*mp)->next && (*mp)->next->page < info->page) mp = &(*mp)->next; info->next = *mp; *mp = info; @@ -1510,32 +1186,14 @@ free_bytes(void *ptr, int index, struct pginfo *info) while (*mp != info) { mp = &((*mp)->next); #ifdef MALLOC_EXTRA_SANITY - if (!*mp) { - wrterror("(ES): Not on queue\n"); - errno = EFAULT; - return; - } + if (!*mp) + wrterror("(ES): Not on queue\n"); #endif /* MALLOC_EXTRA_SANITY */ } *mp = info->next; /* Free the page & the info structure if need be */ - pidx = PI_IDX(ptr2index(info->page)); - pdir_lookup(ptr2index(info->page), &pi); -#ifdef MALLOC_EXTRA_SANITY - if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { - wrterror("(ES): mapped pages not found in directory\n"); - errno = EFAULT; - return; - } -#endif /* MALLOC_EXTRA_SANITY */ - if (pi != last_dir) { - prev_dir = last_dir; - last_dir = pi; - } - - pd = pi->base; - pd[PI_OFF(ptr2index(info->page))] = MALLOC_FIRST; + page_dir[ptr2index(info->page)] = MALLOC_FIRST; /* If the page was mprotected, unprotect it before releasing it */ if (info->size == 0) { @@ -1553,9 +1211,6 @@ static void ifree(void *ptr) { struct pginfo *info; - struct pginfo **pd; - struct pdinfo *pi; - u_long pidx; u_long index; /* This is legal */ @@ -1583,22 +1238,7 @@ ifree(void *ptr) return; } - pidx = PI_IDX(index); - pdir_lookup(index, &pi); -#ifdef MALLOC_EXTRA_SANITY - if (pi == NULL || PD_IDX(pi->dirnum) != pidx) { - wrterror("(ES): mapped pages not found in directory\n"); - errno = EFAULT; - return; - } -#endif /* MALLOC_EXTRA_SANITY */ - if (pi != last_dir) { - prev_dir = last_dir; - last_dir = pi; - } - - pd = pi->base; - info = pd[PI_OFF(index)]; + info = page_dir[index]; if (info < MALLOC_MAGIC) free_pages(ptr, index, info); @@ -1617,10 +1257,9 @@ malloc_recurse(void) { static int noprint; - if (noprint == 0) { - noprint = 1; + if (noprint == 0) wrtwarning("recursive call\n"); - } + noprint = 1; malloc_active--; _MALLOC_UNLOCK(); errno = EDEADLK; @@ -1644,10 +1283,8 @@ malloc(size_t size) UTRACE(0, size, r); malloc_active--; _MALLOC_UNLOCK(); - if (malloc_xmalloc && r == NULL) { + if (malloc_xmalloc && r == NULL) wrterror("out of memory\n"); - errno = ENOMEM; - } return (r); } @@ -1686,9 +1323,7 @@ realloc(void *ptr, size_t size) UTRACE(ptr, size, r); malloc_active--; _MALLOC_UNLOCK(); - if (malloc_xmalloc && r == NULL) { + if (malloc_xmalloc && r == NULL) wrterror("out of memory\n"); - errno = ENOMEM; - } return (r); }