/* 
 * Copyright (C) 2020,  Paul Sheer. All rights reserved.
 * 
 * This software is provided without warranty. The author(s) disclaim liability for damages
 * related to this software. All risk is assumed by you.
 * 
 */


/* 
 * 
 * Dallas Forth-Worth Malloc - DFWMalloc
 * 
 * Copyright (C) 2020, Paul Sheer. All rights reserved.
 * 
 * This software is provided without warranty. The author(s) disclaim liability for
 * damages related to this software. All risk is assumed by you.
 * 
 * 
 * General purpose malloc library to replace the core functions of the C
 * malloc library, malloc/free/new/delete/realloc/calloc/memalign/posix_memalign/malloc_trim.
 * 
 * 
 * 
 * USAGE:
 * 
 * To compile for maximum performance use:
 * 
 *         -DUSE_TLS -DHAVE_PTHREAD -DNO_ASSERT_CHECKS
 * 
 * To compile for backtracing support and verification do as follows. Note that your
 * log function must NOT call malloc() nor any functions that call malloc() (like printf):
 * 
 *         -DUSE_TLS -DWITH_BTANDCHECK -DWITH_ACCOUNTS -DNO_CB_HEAPS
 *         -no-pie -g       # ==> gcc opt for all code or else addr2line prints ??:0
 *         -Og              # ==> desirable
 *         Add dfwmalloc_set_log(...) to your code.
 *         sysctl -w kernel.randomize_va_space=0   # ==> or else addr2line prints ??:0
 * 
 * Inspect back-traces with, for example:
 * 
 *         addr2line -e my_exe   0x419b54 0x419bb3 0x419bda 0x419c1a 0x7ffff75e2b97 0x40101a
 * 
 * Check that a pointer is valid, returns the allocated size, checks the bounds for corruption:
 * 
 *         dfwmalloc_check()
 * 
 * To dump leaks use:
 * 
 *         dfwmalloc_dump_leaks(...)  or
 *         dfwmalloc_log_leaks()
 * 
 * Similar to malloc_trim(), to free mapped pages do this call once every second:
 * 
 *         dfwmalloc_truncate_heap()
 *         dfwmalloc_dfwtruncate_heap()
 * 
 * If a foreign thread free'd a lot of stuff on your behalf, then catch up with:
 * 
 *         dfwmalloc_free_foreign()
 * 
 * To create your own arena in which to malloc and free, do:
 * 
 *         dfw = dfwmalloc_new ();
 *         p = dfwmalloc_malloc (dfw, c);
 *         dfwmalloc_free (dfw, p);
 *         if (0) free (p); <== you can also free from a different arena but you
 *                              incur a performance penalty.
 *         leaks = dfwmalloc_destroy (dfw);
 *         assert (leaks == 0);
 *         dfw = NULL;
 * 
 * These functions are for getting usage stats:
 * 
 *         dfwmalloc_stats(...)
 *         dfwmalloc_dfwstats(...)
 *         dfwmalloc_heap_size()        <-- simpler version of dfwmalloc_stats
 *         dfwmalloc_alloced_bytes()    <-- simpler version of dfwmalloc_stats
 * 
 * To dump all control blocks to the log for fragmentation assessment:
 * 
 *         dfwmalloc_efficiency()
 *         dfwmalloc_dfwefficiency(...)
 * 
 * Get the version as (major << 20) | (minor << 10) | patch
 * 
 *         dfwmalloc_version()
 *         DFWMALLOC_VERSION
 * 
 * Also note:
 * 
 *         strings lib*.so | grep VERSION
 *         DFWMALLOC_VERSION_MAJOR
 *         DFWMALLOC_VERSION_MINOR
 *         DFWMALLOC_VERSION_PATCH
 * 
 * See also as follows for disabling built-in malloc libraries,
 * 
 *         https://fuzzing-project.org/tutorial-malloc.html
 *         https://web.archive.org/web/20191008001924/https://fuzzing-project.org/tutorial-malloc.html
 * 
 * 
 * PERFORMANCE:
 * 
 * Appears to beat all other malloc libraries. DFWMalloc operates lock free for
 * mallocs and free's within the same thread.
 * 
 * 
 * BEHAVIOR:
 * 
 * Calling realloc() at the tail of the heap efficiently extends the heap
 * without a memcpy() such that a program that wishes to do a large number of
 * reallocs as its primary function can keep extending the space efficiently.
 * (Such as loading a large file into memory.)
 * 
 * Calling free() on a region at the tail of the heap releases mapped pages
 * on the next malloc() that requires any pages.
 * 
 * Calling malloc() for small blocks (<32k) properly reuses pages starting
 * with the pages closest to the head of the heap -- this uses a rank-pairing
 * binary heap. For large blocks (>32k) a linked list of free blocks is used
 * which tends to reuse blocks near the tail.
 * 
 * Allocating in one thread and free'ing in another is extremely inefficient.
 * DFWMalloc will acquire locks to do this. If a thread exits with blocks
 * unfree'd then the management structure will not be released until the
 * last block is free'd.
 * 
 * The number of arena bits defaults to 10. This limits the number of
 * concurrent threads you can run. The actual limit of arenas is slightly
 * less than 1<<10 after which DFWMalloc prints "out of arenas, aborting"
 * and calls abort().
 * 
 * 
 * BUGS:
 * 
 * 1. No testing has been done for 32-bit systems. This software is currently
 * 64-bit only, although much effort has been made to ensure portability.
 * A 32-bit port should be low-effort, but functionality will be reduced.
 * 
 * 2. Repeated calls to malloc(0) return the identical pointer. Duplicate
 * free's of this pointer are allowed at all times.
 * 
 * 3. No mallopt()/malloc_info() support.
 * 
 * 4. malloc_trim() ignores its argument and does a full trim, and always
 * returns 1.
 * 
 * 5. If N is 1<<WANTED_ALIGNMENT (the alignment) then malloc(n<N) returns
 * a pointer with alignment of the highest power of 2 less than n. This
 * does not seem to create any problems.
 * 
 * 6. Not a bug, but one should note the discrepancy between POSIX and C++
 * specifications. POSIX does not require 16-byte alignment, but C++ does.
 * 16-byte alignment is the default.
 * 
 * 7. With the default compile options, a single process can create no more
 * than 1000 concurrent threads.
 * 
 * 8. The inline new/delete mechanism has not been tested within a real
 * application.
 * 
 */


/* Configuration: */

/* Basic configuration: */

/* Guaranteed alignment of all malloc() return values (power of 2).
 * Example alignment requirements for 64-bit architectures:
 *   Intel CPU: 1<<0 = 1
 *   Most RISC: 1<<3 = 8
 *   C: 1<<3 = 8
 *   C++: 1<<4 = 16
 *   Chrome: 1<<5 = 32 */
/* #undef WANTED_ALIGNMENT */
/* #define WANTED_ALIGNMENT 4 */

/* Add backtrace support and bounds checking. Call dfwmalloc_enable_backtrace() to enable: */
/* #undef WITH_BTANDCHECK */
/* #define WITH_BTANDCHECK */

/* Add accounting: */
/* #undef WITH_ACCOUNTS */
/* #define WITH_ACCOUNTS */

/* For multithreading support: */
/* #undef HAVE_PTHREAD */
/* #define HAVE_PTHREAD */

/* For thread-local-storage support: */
/* #undef USE_TLS */
/* #define USE_TLS */

/* Do not even do basic sanity checks. Applicable only when WITH_BTANDCHECK is undefined: */
/* #undef NO_ASSERT_CHECKS */
/* #define NO_ASSERT_CHECKS */

/* Never truncate the heap: */
/* #undef NO_HEAP_TRUNCATION */
/* #define NO_HEAP_TRUNCATION */

/* malloc size is 32 bits: */
/* #undef LIGHT */
/* #define LIGHT */

/* User can define the maximum expected allocation: */
/* #undef USER_MAX_MALLOC */
/* #define USER_MAX_MALLOC (? * 1024 * 1024) */

/* User can define the granularity of increasing bucket sizes for regular allocations: */
/* #undef BUCKET_SMOOTHNESS */
/* #define BUCKET_SMOOTHNESS 6 */

/* Do not use control blocks: all allocations are Regular allocations: */
/* #undef NO_CB_HEAPS */
/* #define NO_CB_HEAPS */

/* Do not use the lock-free queue mechanism for allocating in one thread and free'ing in another: */
/* #undef NO_LOCKFREEVISITOR */
/* #define NO_LOCKFREEVISITOR */

/* Do not trap the pthread_exit() function. This means you have to call
 * dfwmalloc_pthread_exit() on thread exit. Saves you have to link with -ldl: */
/* #undef NO_PTHREAD_EXIT_OVERRIDE */
/* #define NO_PTHREAD_EXIT_OVERRIDE */



/* Less common configuration: */

/* This is used to not do certain operations too often. Interpreted as 1<<N MHz (2048 for N=11). It doesn't matter if it off by a small factor: */
/* #undef ROUGH_CPU_SPEED_MHZ */
/* #define ROUGH_CPU_SPEED_MHZ 11 */

/* Roughly the number of threads you can create, 1024 = 1<<10: */
/* #undef N_ARENAS_BITS */
/* #define N_ARENAS_BITS 10 */

/* Do not provide the exported interface functions like dfwmalloc_new()/dfwmalloc_destroy() for creating user dfw-arenas: */
/* #undef NO_INTERNAL_EXPOSE */
/* #define NO_INTERNAL_EXPOSE */

/* Utility function: */
/* #undef WANT_PRINTFSIMPLE */
/* #define WANT_PRINTFSIMPLE */

/* Fidelity checks: */
/* #undef WITH_UNITTESTS */
/* #define WITH_UNITTESTS */

/* Disable inlining optimizations: */
/* #undef DISABLE_INLINE */
/* #define DISABLE_INLINE */

/* Minimum alignment for small areas. This allows dfwmalloc to use a shift to increase the potential size of a chunk: */
/* #undef MIN_ALIGN */
/* #define MIN_ALIGN 3 */

/* Number of bits in pointers reserved for the heap identifier: */
/* #undef CB_SIZE_BITS */
/* #define CB_SIZE_BITS 3 */

/* Steps increase of chunk sizes. 1 means 1024,2048,4096,... and 2 means 1024,4096,16384,...  */
/* #undef CHUNK_POWER */
/* #define CHUNK_POWER 2 */

/* Number of blocks in the largest control block. Used as 1<<n: */
/* #undef ESTIMATED_MIN_BLOCKS */
/* #define ESTIMATED_MIN_BLOCKS 4 */

/* How to estimate the number of blocks per control block for a particular size of malloc: */
/* #undef BLOCKS_FORMULA */
/* #define BLOCKS_FORMULA          (1000000 / (n + 1000) + 32) */

/* Do not order small blocks within their control chunk. This has a slight performance advantage,
 * but hurts cache usage. It will also limit the usable size of control block for packing reasons: */
/* #undef NO_BLOCK_ORDERING */
/* #define NO_BLOCK_ORDERING */

/* Use bitwise packing when possible: */
/* #undef COMPACT_MODE */
/* #define COMPACT_MODE */

/* Ratio of successive allocation sizes for Control Block allocations: */
/* #undef ALLOCATION_SIZE_RATIO */
/* #define ALLOCATION_SIZE_RATIO 1100 */

/* Size of control blocks of the heap with index 0. Used as 1<<n: */
/* #undef CB_START_BITS */
/* #define CB_START_BITS 10 */

/* Needed for Chrome or any application that doesn't like its exit/_exit/_Exit functions
 * being overridden. This does not affect atexit() which is still used: */
/* #undef NO_OVERRIDE_EXIT_FUNCTIONS */
/* #define NO_OVERRIDE_EXIT_FUNCTIONS */

/* 
 * 
 * Extra address ranges to avoid. The "END" parameter must be defined is the last byte plus 1,
 * for example define the range as 0x38000000000-0x40000000000 and not 0x38000000000-0x3FFFFFFFFFF.
 * 
 * #define AVOID_ADDRESS_RANGE_1_START     0x000000000000UL
 * #define AVOID_ADDRESS_RANGE_1_END       0x000000000000UL
 * #define AVOID_ADDRESS_RANGE_2_START     0x000000000000UL
 * #define AVOID_ADDRESS_RANGE_2_END       0x000000000000UL
 * #define AVOID_ADDRESS_RANGE_3_START     0x000000000000UL
 * #define AVOID_ADDRESS_RANGE_3_END       0x000000000000UL
 * #define AVOID_ADDRESS_RANGE_4_START     0x000000000000UL
 * #define AVOID_ADDRESS_RANGE_4_END       0x000000000000UL
 * #define AVOID_ADDRESS_RANGE_5_START     0x000000000000UL
 * #define AVOID_ADDRESS_RANGE_5_END       0x000000000000UL
 * 
 */

/* End configuration. */


/* Header: */

#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif

#include <stdint.h>
#include <stdlib.h>



#ifdef __cplusplus
extern "C" {
#endif

#define DEF_VERS_S(major,minor,patch)           "DFWMALLOC VERSION " #major "." #minor "." #patch
#define DEF_VERS_L(major,minor,patch)           (((LONG) major << 20) | ((LONG) minor << 10) | ((LONG) patch << 0))

/* Edit here: */
#define DFWMALLOC_VERSION_MAJOR                  1
#define DFWMALLOC_VERSION_MINOR                  0
#define DFWMALLOC_VERSION_PATCH                  2
#define DFWMALLOC_VERSION                        DEF_VERS_L(1,0,2)
#define DFWMALLOC_VERSION_STR                    DEF_VERS_S(1,0,2)

struct dfwmalloc;

typedef unsigned long long ULONG64;
typedef long long LONG64;
typedef intptr_t LONG;
typedef uintptr_t ULONG;

struct dfwmalloc_stats {
    LONG alloced;
    LONG blocks;
    LONG alloced_bt;            /* alloced while backtracing was enabled */
    LONG blocks_bt;             /* alloced while backtracing was enabled */
    LONG alloced_small;         /* alloced while backtracing was enabled */
    LONG blocks_small;          /* alloced while backtracing was enabled */
    LONG heap;
    LONG heap_small;            /* not used if backtracing enabled */
    LONG free_blocks;
    LONG free_blocks_small;     /* not used if backtracing enabled */
    ULONG max_heap;             /* maximum size of the heap for large allocations (>3104) */
    ULONG max_small_area;       /* maximum size of the heap for small allocations (<=3104) */
    LONG max_cb_size;           /* maximum size of block that will be handled by control blocks arenas */
};

struct dfwmalloc_leak {
    LONG backtrace_id;
    LONG alloced;
    LONG blocks;
    int arena_id;
    int n;
    void **addr;
};

struct dfwmalloc_leak_info {
    void *dummy;        /* future */
};

typedef void (*dfwmalloc_log_t) (void *, const char *, int);

void dfwmalloc_set_log (dfwmalloc_log_t dfw, void *user_data);
void dfwmalloc_dump_leaks (struct dfwmalloc_leak_info *info, void (*cb) (void *, struct dfwmalloc_leak *), void *hook);
void dfwmalloc_log_leaks (void);
LONG dfwmalloc_efficiency (void);
/* dfwmalloc_heap_size() == st.heap + st.heap_small */
LONG dfwmalloc_heap_size (void);
/* dfwmalloc_alloced_bytes() == st.alloced + st.alloced_small + st.alloced_bt */
LONG dfwmalloc_alloced_bytes (void);
LONG dfwmalloc_alloced_blocks (void);
void dfwmalloc_stats (struct dfwmalloc_stats *s);
void dfwmalloc_enable_backtrace (void);
int dfwmalloc_get_backtrace_id (void);
void dfwmalloc_pthread_exit (void);
void dfwmalloc_truncate_heap (void);
void dfwmalloc_set_lfv_enablement (int enabled);
void dfwmalloc_free_foreign (void);
LONG dfwmalloc_check (void *u);
void dfwmalloc_verify (void);
LONG dfwmalloc_version (void);

#if defined(USE_TLS) && !defined(NO_INTERNAL_EXPOSE)
LONG dfwmalloc_dfwefficiency (struct dfwmalloc *dfw);
void dfwmalloc_dfwstats (struct dfwmalloc *dfw, struct dfwmalloc_stats *s);
int dfwmalloc_dfwget_backtrace_id (struct dfwmalloc *dfw);
void dfwmalloc_dfwtruncate_heap (struct dfwmalloc *dfw);
LONG dfwmalloc_dfwcheck (struct dfwmalloc *, void *u);

struct dfwmalloc *dfwmalloc_new (void);
int dfwmalloc_destroy (struct dfwmalloc *dfw);
void *dfwmalloc_malloc (struct dfwmalloc *dfw, size_t n);
void *dfwmalloc_realloc (struct dfwmalloc *dfw, void *u, size_t n);
void dfwmalloc_free (struct dfwmalloc *dfw, void *u);
#endif


#ifdef __cplusplus
}
#endif

/* End header. */


#ifdef __STRICT_ANSI__
#define inline                  __inline__
#define asm                     __asm__
#define volatile                __volatile__
#endif

#ifdef __GNUC__
#pragma GCC diagnostic ignored "-Wattributes"
#if __GNUC__ >= 9
#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
#endif
#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)
#pragma GCC diagnostic ignored "-Warray-bounds"
#endif
#endif


#ifdef DISABLE_INLINE
#define STATICFN                
#define STATICFNINLINE
#define STATICFNNOINLINE
#else
/* optionally inlined functions for the compiler's discretion: */
#define STATICFN                static inline
/* forcibly inlined functions for performance reasons, ignoring code size considerations: */
#define STATICFNINLINE          static __attribute__((always_inline))
/* seldom-called functions forcibly not inlined due to code size considerations: */
#define STATICFNNOINLINE        static __attribute__((noinline))
#endif


#ifdef PRECOMPILE_STATIC_DATA
#undef NO_ASSERT_CHECKS
#endif


#ifndef WANTED_ALIGNMENT
#define WANTED_ALIGNMENT        4
#endif

#ifndef BUCKET_SMOOTHNESS
#define BUCKET_SMOOTHNESS       6
#endif

#ifndef ALLOCATION_SIZE_RATIO
#define ALLOCATION_SIZE_RATIO   1100
#endif

#ifdef NO_CB_HEAPS
#define CB_SIZE_BITS            0
#else
#ifndef CB_SIZE_BITS
#define CB_SIZE_BITS            3
#endif
#endif

#ifndef MIN_ALIGN
#define MIN_ALIGN               3
#endif

#ifndef CHUNK_POWER
#define CHUNK_POWER             2
#endif

#ifndef ESTIMATED_MIN_BLOCKS
#define ESTIMATED_MIN_BLOCKS    4
#endif

#ifndef BLOCKS_FORMULA
#define BLOCKS_FORMULA          (500000 / (n + 1000) + 32)
#endif

#ifndef CB_START_BITS
#if defined(FEATHERWEIGHT) || defined(FOR_VALGRIND)
#define CB_START_BITS           6
#else
#define CB_START_BITS           10
#endif
#endif

#if defined(USE_TLS) && !defined(HAVE_PTHREAD)
#error defined(USE_TLS) && !defined(HAVE_PTHREAD)
#endif

#ifndef ROUGH_CPU_SPEED_MHZ
#define ROUGH_CPU_SPEED_MHZ     11
#endif



#ifdef HAVE_PTHREAD
#include <setjmp.h>
#include <pthread.h>
#endif

#include <string.h>
#include <stdarg.h>
#include <errno.h>
#include <unistd.h>
#ifndef __FreeBSD__
#include <malloc.h>
#endif
#include <errno.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <unistd.h>
#include <execinfo.h>
#if !defined(NO_PTHREAD_EXIT_OVERRIDE) || defined(WITH_BTANDCHECK)
#include <dlfcn.h>
#endif





#ifdef FEATHERWEIGHT

typedef unsigned short malloc_size_t;

#elif defined(LIGHT)

typedef unsigned int malloc_size_t;

#else

typedef ULONG malloc_size_t;

#endif


#ifdef __cplusplus
#include <new>
extern "C" void dfwmalloc_do_throw (void)
{
    throw std::bad_alloc ();
}
#else
void dfwmalloc_do_throw (void)
{
    int r;
    r = write (2, "malloc returned a null pointer, aborting\n", 41);
    (void) r;
    abort ();
}
#endif


#ifdef __cplusplus
extern "C" {
#endif

#ifdef __linux__
#define dfwmalloc_mutex_t                pthread_mutex_t
#else
typedef long dfwmalloc_mutex_t;
#endif

#define MALLOC_MAX                      ((malloc_size_t) (~0))

#define SOME_32_BIT_LARGE               (1ULL << 28)
#define SOME_64_BIT_LARGE               (1ULL << 44)
#define MALLOC_MAX_64_BIT               (SOME_64_BIT_LARGE - 1ULL)

#define POINTER_ALIGNMENT               (sizeof(void *))
#define NO_ALIGNMENT                    0
#define ALIGNMENT_MASK                  ((1 << WANTED_ALIGNMENT) - 1)
#define ALIGN_ADJUST(n,mask,offset)     ((mask) - (((n) + (mask) + (offset)) & (mask)))

#define LEN_MASK                        (MALLOC_MAX >> 1)
#define IN_USE_MASK                     (MALLOC_MAX - (MALLOC_MAX >> 1))
#define LEN(p)                          ((p)->len & LEN_MASK)
#define IS_IN_USE(p)                    (((p)->len & IN_USE_MASK) == 0)
#define IS_FREE(p)                      (((p)->len & IN_USE_MASK) != 0)
#define SET_FREE(p)                     do { (p)->len |= IN_USE_MASK; } while (0)

STATICFN void strlimcat (char *s, int n, const char *p)
{
    while (*s) {
        s++;
        n--;
    }
    while (n > 1 && *p) {
        *s++ = *p++;
        n--;
    }
    *s = '\0';
}

STATICFN void dfwmalloc_write2 (const void *s1, int l1, const void *s2, int l2)
{
    int r;
    struct iovec iov[2];
    iov[0].iov_base = (void *) s1;
    iov[0].iov_len = l1;
    iov[1].iov_base = (void *) s2;
    iov[1].iov_len = l2;
    r = writev (2, iov, l2 ? 2 : 1);
    (void) r;
}

STATICFN void dfwmalloc_writestr (const char *s)
{
    dfwmalloc_write2 (s, strlen (s), "", 0);
}

STATICFN void dfwmalloc_log (void *user_data, const char *s, int l)
{
    (void) user_data;
    dfwmalloc_write2 (s, l, "\n", 1);
}

static dfwmalloc_log_t dfwmalloc_log_ = dfwmalloc_log;
static void *dfwmalloc_log_user_data_ = NULL;

#define DFWMALLOC_LOG(s,l)       (*dfwmalloc_log_) (dfwmalloc_log_user_data_, (s), (l))

void dfwmalloc_set_log (dfwmalloc_log_t fn, void *user_data)
{
    dfwmalloc_log_user_data_ = user_data;
    dfwmalloc_log_ = fn;
}

__thread char itoa_base_ret__[32];

STATICFN char *itoa_base (ULONG64 i, int issigned, int base)
{
    int neg = 0;
    char *num;
    if (issigned && ((LONG64) i) < 0) {
        neg = 1;
        i = (ULONG64) (-((LONG64) i));
    }
    num = &itoa_base_ret__[31];
    *num = '\0';
    do {
        num--;
        *num = "0123456789abcdef"[i % base];
        i /= base;
    } while (i);
    if (neg) {
        num--;
        *num = '-';
    }
    return num;
}

#if defined(WITH_BTANDCHECK) || defined(WANT_PRINTFSIMPLE)

int snprintf_simple (char *out, int outlen, const char *s, ...) __attribute__ ((__format__ (__printf__, 3, 4)));
void printf_simple (const char *s, ...) __attribute__ ((__format__ (__printf__, 1, 2)));

STATICFN int vsnprintf_simple (char *out, int outlen, const char *s, va_list ap)
{
    int n = 0;
    int islong = 0, isshort = 0, base = 10;
    if (!outlen)
        return 0;

#define CHR(c) \
    do { \
        if (outlen > 0) { \
            *out++ = (c); \
            outlen--; \
            n++; \
        } \
    } while (0)

#define STR(s) \
    do { \
        while (*s && outlen > 0) { \
            *out++ = *s++; \
            outlen--; \
            n++; \
        } \
    } while (0)

    while (*s && outlen > 0) {
        if (*s == '%') {
            const char *t;
            s++;
            islong = 0;
            isshort = 0;
            base = 8;
          more:
            switch (*s) {
            case '%':
                s++;
                CHR ('%');
                break;
            case 'c':
                s++;
                CHR ((int) va_arg(ap, int));
                break;
            case 's':
                s++;
                t = (char *) va_arg(ap, char *);
                if (!t)
                    t = "(null)";
                STR (t);
                break;
            case 'l':
                s++;
                islong++;
                goto more;
            case 'h':
                s++;
                isshort++;
                goto more;
            case 'f':
                s++;
                {
                    double u;
                    ULONG64 d;
                    int l, trail = 0;
                    int sign = 0;
                    u = (double) va_arg (ap, double);
                    if (u < 0.0) {
                        sign = 1;
                        u = -u;
                    }
                    while (u >= 1.84e19) {
                        u /= 10.0L;
                        trail++;
                    }
                    d = (ULONG64) u;
                    t = itoa_base (d, 0, 10);
                    l = strlen (t);
                    if (sign)
                        CHR('-');
                    STR (t);
                    if (l < 15) {
                        u -= (double) d;
                        CHR ('.');
                        l = 15 - l;
                        while (l--) {
                            u *= 10.0;
                            d = (ULONG64) u;
                            CHR ('0' + (int) d);
                            u -= d;
                        }
                    }
                    while (trail--)
                        CHR ('0');
                }
                break;
            case 'd':
                s++;
                if (islong >= 2)
                    t = itoa_base ((ULONG64) va_arg (ap, long long), 1, 10);
                else if (islong)
                    t = itoa_base ((ULONG64) va_arg (ap, long), 1, 10);
                else if (isshort >= 2)
                    t = itoa_base ((ULONG64) (char) va_arg (ap, int), 1, 10);
                else if (isshort)
                    t = itoa_base ((ULONG64) (short) va_arg (ap, int), 1, 10);
                else
                    t = itoa_base ((ULONG64) va_arg (ap, int), 1, 10);
                STR (t);
                break;
            case 'x':
            case 'X':
                base += 6;
                /* Falls through. */
            case 'u':
                base += 2;
                /* Falls through. */
            case 'o':
                s++;
                if (islong >= 2)
                    t = itoa_base ((ULONG64) va_arg (ap, unsigned long long), 0, base);
                else if (islong)
                    t = itoa_base ((ULONG64) va_arg (ap, unsigned long), 0, base);
                else if (isshort >= 2)
                    t = itoa_base ((ULONG64) (unsigned char) va_arg (ap, unsigned int), 1, base);
                else if (isshort)
                    t = itoa_base ((ULONG64) (unsigned short) va_arg (ap, unsigned int), 1, base);
                else
                    t = itoa_base ((ULONG64) va_arg (ap, unsigned int), 0, base);
                STR (t);
                break;
            case 'p':
                s++;
                t = itoa_base ((ULONG64) va_arg (ap, void *), 0, 16);
                CHR ('0');
                CHR ('x');
                STR (t);
                break;
            case '\0':
                break;
            default:
                CHR (*s);
                s++;
                break;
            }
        } else {
            CHR (*s);
            s++;
        }
    }
    if (outlen) {
        *out = '\0';
    } else {
        out--;
        *out = '\0';
        n--;
    }
    return n;
}

int snprintf_simple (char *out, int outlen, const char *fmt, ...)
{
    int r;
    va_list ap;
    va_start (ap, fmt);
    r = vsnprintf_simple (out, outlen, fmt, ap);
    va_end (ap);
    return r;
}

void printf_simple (const char *fmt, ...)
{
    char s[256];
    int l;
    va_list ap;
    va_start (ap, fmt);
    l = vsnprintf_simple (s, sizeof (s), fmt, ap);
    dfwmalloc_write2 (s, l, "", 0);
    va_end (ap);
}

#endif

STATICFN void dfwmalloc_assert_ (const char *file, int line, const char *msg);
#define dfwmalloc_always_assert(x)       (void) (!(x) && (dfwmalloc_assert_(__FILE__, __LINE__, #x), (*((int *) ~((uintptr_t) 0)) = -1)))

#if defined(WITH_BTANDCHECK) || !defined(NO_ASSERT_CHECKS)

/* this forces a segfault instead of calling abort() because this allows
 * us to easily inspect the local variables from the core: */
#define dfwmalloc_assert(x)       (void) (!(x) && (dfwmalloc_assert_(__FILE__, __LINE__, #x), (*((int *) ~((uintptr_t) 0)) = -1)))

#else

#define dfwmalloc_assert(x)       do {  } while (0)

#endif

#if (!defined(NO_CB_HEAPS) && !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)) || (!defined(NO_HEAP_TRUNCATION) && (!defined(NO_CB_HEAPS) || defined(USE_TLS)))

STATICFNINLINE unsigned long long rdtsc (void)
{
    unsigned hi, lo;
    asm volatile ("rdtsc":"=a" (lo), "=d" (hi));
    return ((unsigned long long) lo) | (((unsigned long long) hi) << 32);
}

#endif


#ifdef USE_TLS

#ifndef N_ARENAS_BITS
#define N_ARENAS_BITS           10
#endif

#else

#undef N_ARENAS_BITS
#define N_ARENAS_BITS           1

#endif


#ifdef USE_TLS
#ifdef NO_CB_HEAPS
#define MAX_TLS_HEAP            (((ULONG) 1UL << (47 - N_ARENAS_BITS)) - 1)
#else
#define MAX_TLS_HEAP            ((((ULONG) 1UL << (47 - N_ARENAS_BITS - CB_SIZE_BITS)) * ((1 << CB_SIZE_BITS) - CB_SIZE_MAX)) - 1)
#endif
#endif

#ifndef NO_CB_HEAPS
#define MAX_CB_HEAP             (((ULONG) 1UL << (47 - N_ARENAS_BITS - CB_SIZE_BITS)) - 1)
#endif


#if defined(FEATHERWEIGHT) || defined(LIGHT)

/* Since we are counting on alignment, and since all pointers are offset
 * from the start of the heap, the following bit manipulation trick allows
 * us to store a 64-bit pointer in 32 bits */
#define MAX_HEAP                ((ULONG) MALLOC_MAX << WANTED_ALIGNMENT)

#define GETP_(dfw,p)              ((struct free_item *) (void *) ((dfw)->heap->start_of_heap + (((ULONG) (p)) << WANTED_ALIGNMENT)))
#define SETP_(dfw,p)              ((malloc_size_t) (((ULONG) (((char *) p) - (dfw)->heap->start_of_heap)) >> WANTED_ALIGNMENT))
#define GETP(dfw,p)               ((p) ? GETP_(dfw,p) : NULL)
#define SETP(dfw,p)               ((p) ? SETP_(dfw,p) : (malloc_size_t) 0)

#else

#define MAX_HEAP                ((ULONG) MALLOC_MAX)

#define GETP(dfw,p)               ((struct free_item *) (p))
#define SETP(dfw,p)               ((malloc_size_t) (p))

#endif

#define INVALID_BACKTRACE_ID            0

#ifdef WITH_BTANDCHECK
typedef unsigned int backtrace_id_t;

struct backtrace_id_compressed_s {
    unsigned char d[3];
} __attribute__((packed));
typedef struct backtrace_id_compressed_s backtrace_id_compressed_t;

STATICFNINLINE ULONG backtrace_id_to_long (const backtrace_id_compressed_t * id)
{
    return ((ULONG) id->d[0] << 16) | ((ULONG) id->d[1] << 8) | ((ULONG) id->d[2] << 0);
}

STATICFNINLINE void backtrace_id_from_long (backtrace_id_compressed_t * id, const LONG l)
{
    id->d[0] = ((l >> 16) & 0xff);
    id->d[1] = ((l >> 8) & 0xff);
    id->d[2] = ((l >> 0) & 0xff);
}

#endif

struct free_item {
/* This points to next block on free list. A block cannot be on the
 * shared list as well as the free list, so these are combined into one: */
    malloc_size_t _fl_sh_next;
#define sh_next _fl_sh_next
#define fl_next _fl_sh_next
#ifndef NO_HEAP_TRUNCATION
    malloc_size_t fl_prev;      /* points to prev block on free list */
    malloc_size_t hp_prev;      /* points to previous adjacent block on heap */
#endif
    malloc_size_t len;
#ifdef WITH_BTANDCHECK
    backtrace_id_compressed_t malloc_trace;
    backtrace_id_compressed_t free_trace;
    char magic[1];
#endif
} __attribute__((packed));


#ifdef WITH_BTANDCHECK

struct free_tail {
    char magic[2];
} __attribute__((packed));

#define TAIL_SIZE      sizeof(struct free_tail)

#else

#define TAIL_SIZE      0

#endif

#if defined(FEATHERWEIGHT) || defined(LIGHT)
/* The shift in GETP/SETP means the header has to be aligned, otherwise we get failures
 * with -DWANTED_ALIGNMENT=4 -DWITH_BTANDCHECK -DWITH_ACCOUNTS -DHAVE_PTHREAD -DUSE_TLS -DLIGHT' : */
#define HEAD_SIZE      (sizeof(struct free_item) + ALIGN_ADJUST(sizeof(struct free_item), ALIGNMENT_MASK, 0))
#else
#define HEAD_SIZE      (sizeof(struct free_item) + ALIGN_ADJUST(sizeof(struct free_item), sizeof(malloc_size_t) - 1, 0))
#endif

#define BUCKET_SIZE(dfw,l)       (global->bucket_size[((l) - (dfw)->free_list)])


struct free_list {
#if defined(HAVE_PTHREAD) && !defined(USE_TLS)
    dfwmalloc_mutex_t bucket_mutex;
#endif
#ifdef WITH_ACCOUNTS
    int count;
#endif
    struct free_item *fl_first;
};

#ifdef WITH_BTANDCHECK
struct backtrace_item {
    struct backtrace_item *bt_next;
    backtrace_id_t backtrace_id;
    LONG bt_alloced;
    LONG bt_blocks;
#ifdef USE_TLS
    LONG bt_alloced_otherarena;
    LONG bt_blocks_otherarena;
#endif
    unsigned short hash2;
    char n;
    void *addr[1];
};
#endif

struct backtrace_head {
    struct backtrace_item *bt_first;
};

#ifdef WITH_BTANDCHECK
#ifndef CALLTRACE_DEPTH
#define CALLTRACE_DEPTH         40
#endif

struct calltrace {
    void *addr[CALLTRACE_DEPTH];
    int n;
};

/* This allows for (1 << (11 + 11)) backtace IDs. It seems that projects, even as large as Chrome, do not exceed 500,000 */
#define BACKTRACE_ID_HIGH       11
#define BACKTRACE_ID_LOW        11
#define BACKTRACE_HEADS         32768

struct backtrace {
    struct backtrace_head *backtrace_head;
    LONG backtrace_id;
    struct backtrace_item ***backtrace_array;
#ifdef HAVE_PTHREAD
    dfwmalloc_mutex_t backtrace_mutex;
#endif
};

#else

struct calltrace {
    char dummy;
};

#endif

/* matches CB_FLAG_FLUSH for convenience */
#define HEAP_FLAG_FLUSH         (1<<1)

struct heap {
    LONG mmap_maplength;
    LONG mmap_chunkincr;
    char *start_of_heap;
    char *end_of_heap;
    char *nonzero_heap;
#ifndef NO_HEAP_TRUNCATION
    unsigned int flush;
#endif
    ULONG64 last_timer;
#if defined(HAVE_PTHREAD) && !defined(USE_TLS)
    dfwmalloc_mutex_t heap_mutex;
#endif
};

#ifdef USE_TLS
#define N_ARENAS               ((1L << N_ARENAS_BITS) - 1L)
#endif


#ifndef NO_CB_HEAPS

#define HEAP_INDEX_TO_CHUNKSIZE_(i)      (((i) * CHUNK_POWER) + CB_START_BITS)

/* 
 * Example with CB_SIZE_BITS=3, CHUNK_POWER=2, MIN_ALIGN=3:
 * 
 *   000b <== CB heap index 0
 *   001b <== CB heap index 1
 *   010b <== CB heap index 2
 *   011b <== CB heap index 3
 *   100b <== CB heap index 4
 *   101b <== CB heap index 5
 *   11?b <== regular heap >= CB_SIZE_MAX
 *   11?b <== regular heap >= CB_SIZE_MAX
 * 
 */
#if ESTIMATED_MIN_BLOCKS > 13
#error  ESTIMATED_MIN_BLOCKS is used as  1<<ESTIMATED_MIN_BLOCKS
#endif

#if defined(NO_BLOCK_ORDERING) && defined(COMPACT_MODE)
#error you cannot define NO_BLOCK_ORDERING and COMPACT_MODE
#endif

#define CB_SIZE_MASK            ((1 << CB_SIZE_BITS) - 1)
#ifdef NO_BLOCK_ORDERING
/* A 16-bit pointer has a certain maximum offset after CHUNK_POWER has been applied as well as MIN_ALIGN.
 * This means we can back calculate the maximum feasible Control Block size: */
#define CB_SIZE_MAX_            (((int) sizeof(cb_int_t) * 8 + MIN_ALIGN - CB_START_BITS) / CHUNK_POWER + 1)
#define CB_SIZE_MAX             (CB_SIZE_MAX_ <= CB_SIZE_MASK ? CB_SIZE_MAX_ : CB_SIZE_MASK)
#else
/* ... on the other hand, in compact mode we use only bit masks for the free list. So the limit is large.
 * We set the maximum control block at 16 MB (1<<24): */
#define CB_SIZE_MAX             (HEAP_INDEX_TO_CHUNKSIZE_(CB_SIZE_MASK - 1) <= (ESTIMATED_MIN_BLOCKS + (int) sizeof(cb_int_t) * 8) ? CB_SIZE_MASK : (((ESTIMATED_MIN_BLOCKS + (int) sizeof(cb_int_t) * 8) - CB_START_BITS) / CHUNK_POWER + 1))
#endif


/* @INLINE4START@ */

typedef unsigned short cb_int_t;

/* @INLINE4END@ */

/* Maximum reach of a 16-bit pointer on the free list: */
#define MAX_CB_INT_T            ((1L << (sizeof(cb_int_t) * 8 + MIN_ALIGN)) - 1)

/* @INLINE7START@ */

#define CB_POINTER(cb, pi, cnt) ((void *) (((char *) (void *) (cb)) + (pi)->first_block + (cnt) * (pi)->round_up))

/* @INLINE7END@ */

#define CB_MAGIC                0x76e0
#define CB_MAGIC_MASK           0xfff0
#define CB_MAGIC_CHK(p)         (((p)->cb_magic_flags & CB_MAGIC_MASK) == CB_MAGIC)

/* @INLINE3START@ */

#define CB_FLAG_MASK            0x000f
#define CB_FLAG_PARTIAL_LINKED  (1<<0)
#define CB_FLAG_FLUSH           HEAP_FLAG_FLUSH

#ifndef NO_BLOCK_ORDERING
#define CB_FLAG_PACKING         (3<<2)
/* 
 * Four ordering styles: List(L) (not ordered), Heap(H), Shift(S) (for block sizes
 * that are a power of two), and Multiply(M) (for block sizes that are not a power
 * of two). H packing style is not used since it is slower than the M. (*9*)
 */
#define CB_FLAG_PACKING_L       (0<<2)
#define CB_FLAG_PACKING_H       (1<<2)
#define CB_FLAG_PACKING_S       (2<<2)
#define CB_FLAG_PACKING_M       (3<<2)
#endif

/* @INLINE3END@ */


#define CB_TAIL_MAGIC           0xac4a

#ifdef USE_TLS
/* Pointers free'd in another thread get stored in the free'd block. But
 * we don't need 8-bytes to do this. 6 is usually enough and sometimes 4
 * works too: */
#define SIZEOF_CB_SIZE          ((47 - N_ARENAS_BITS - CB_SIZE_BITS + 7) / 8)
struct cb_size_s {
    unsigned char d[SIZEOF_CB_SIZE];
} __attribute__((packed));

typedef struct cb_size_s cb_size_t;
#endif

/* @INLINE1START@ */

struct bitmaskset_s {
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
#else
    cb_int_t pad6;              /* 6 */
#endif
    cb_int_t first_block_;      /* 7 */
    cb_int_t sz_shift_mul_;     /* 8 */
    ULONG64 cb_bit_mask_[0];
} __attribute__((packed));

typedef unsigned char cb_headint_t;
#define CB_N_HEADS              256

struct control_block {
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
    struct control_block *rh_prev;
    struct control_block *rh_next;
    struct control_block *rh_child;
#endif
    struct control_block *partial_blocks_next;
#ifndef NO_HEAP_TRUNCATION
    struct control_block *partial_blocks_prev;
#endif

/*                                                      */
/*                                 ,--  alignment count */
/*                                 |                    */
/*                                 V                    */
/*                                                      */
    cb_int_t cb_magic_flags;    /* 1 */
    cb_headint_t head_index;    /* 2 */
    unsigned char n_masks;      /* 2 */
    cb_int_t free_count;        /* 3 */
    cb_int_t total_count;       /* 4 */
    cb_int_t used_count;        /* 5 */
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
    cb_int_t create_time;       /* 6 */
#endif

    union {
        cb_int_t cb_free_list_[0];
        struct bitmaskset_s bitmaskset;
    } free_set[0];
#define cb_free_list    free_set[0].cb_free_list_
#define cb_bit_mask     free_set[0].bitmaskset.cb_bit_mask_
#define sz_shift_mul    free_set[0].bitmaskset.sz_shift_mul_
#define cb_first_block  free_set[0].bitmaskset.first_block_
} __attribute__((packed));

/* @INLINE1END@ */

#define CB_FIRST_OFFSET_L(total_count)          (sizeof(struct control_block) + sizeof(cb_int_t) * (total_count))
#define CB_MASK_N(total_count)                  (((total_count) + 63) >> 6)
#define CB_FIRST_OFFSET_M(total_count)          (sizeof(struct control_block) + sizeof(struct bitmaskset_s) + 8 * CB_MASK_N(total_count))

#define CB_FIRST_OFFSET(m,total_count,align)    ((m) ? (CB_FIRST_OFFSET_M(total_count) + ALIGN_ADJUST(CB_FIRST_OFFSET_M(total_count), (align) - 1, 0)) : \
                                                       (CB_FIRST_OFFSET_L(total_count) + ALIGN_ADJUST(CB_FIRST_OFFSET_L(total_count), (align) - 1, 0)))
#define CB_SIZE(m,len,align,total_count)        ((LONG) CB_FIRST_OFFSET(m, total_count, align) + (LONG) (len) * (total_count) + /* tail magic ==> */ sizeof(cb_int_t))

#if defined(FEATHERWEIGHT) || defined(FOR_VALGRIND)
#define CB_N_SIZES              128
#else
#define HEAP_INDEX_TO_CHUNKSIZE(i)      (1UL << HEAP_INDEX_TO_CHUNKSIZE_(i))
#define ESTIMATED_MIN_BLOCKS_   (1<<ESTIMATED_MIN_BLOCKS)

#ifdef NO_BLOCK_ORDERING

#define MAX_CB_                 HEAP_INDEX_TO_CHUNKSIZE (CB_SIZE_MAX - 1)
#define MAX_CB                  ((MAX_CB_INT_T + 1) < MAX_CB_ ? (MAX_CB_INT_T + 1) : MAX_CB_)
#define CB_MAX_FIRST_OFFSET     \
    (CB_FIRST_OFFSET(0,ESTIMATED_MIN_BLOCKS_, 1 << WANTED_ALIGNMENT) > CB_FIRST_OFFSET(1,ESTIMATED_MIN_BLOCKS_, 1 << WANTED_ALIGNMENT) ? \
     CB_FIRST_OFFSET(0,ESTIMATED_MIN_BLOCKS_, 1 << WANTED_ALIGNMENT) : CB_FIRST_OFFSET(1,ESTIMATED_MIN_BLOCKS_, 1 << WANTED_ALIGNMENT))
#define CB_N_SIZES_             ((int) (MAX_CB - sizeof(cb_int_t) - CB_MAX_FIRST_OFFSET) / ESTIMATED_MIN_BLOCKS_)
#define CB_ALIGNMENT            ((WANTED_ALIGNMENT) > (MIN_ALIGN) ? (WANTED_ALIGNMENT) : (MIN_ALIGN))
#define CB_ALIGNMENT_MASK       ((1 << CB_ALIGNMENT) - 1)
#define CB_N_SIZES              (CB_N_SIZES_ & ~CB_ALIGNMENT_MASK)

#else

#define MAX_CB                  HEAP_INDEX_TO_CHUNKSIZE (CB_SIZE_MAX - 1)
#define CB_MAX_FIRST_OFFSET     \
    (CB_FIRST_OFFSET(0,ESTIMATED_MIN_BLOCKS_, 1 << WANTED_ALIGNMENT) > CB_FIRST_OFFSET(1,ESTIMATED_MIN_BLOCKS_, 1 << WANTED_ALIGNMENT) ? \
     CB_FIRST_OFFSET(0,ESTIMATED_MIN_BLOCKS_, 1 << WANTED_ALIGNMENT) : CB_FIRST_OFFSET(1,ESTIMATED_MIN_BLOCKS_, 1 << WANTED_ALIGNMENT))
#define CB_N_SIZES__            ((int) (MAX_CB - sizeof(cb_int_t) - CB_MAX_FIRST_OFFSET) / ESTIMATED_MIN_BLOCKS_)
#define CB_N_SIZES_             (CB_N_SIZES__ <= 65534 ? CB_N_SIZES__ : 65534)
#define CB_ALIGNMENT            ((WANTED_ALIGNMENT) > (MIN_ALIGN) ? (WANTED_ALIGNMENT) : (MIN_ALIGN))
#define CB_ALIGNMENT_MASK       ((1 << CB_ALIGNMENT) - 1)
#define CB_N_SIZES              (CB_N_SIZES_ & ~CB_ALIGNMENT_MASK)

#endif

#endif


/* @INLINE8START@ */

struct cb_page_info {
    unsigned short n_blocks;
    unsigned char heap_index;
    unsigned char mask_packing;
    unsigned short first_block;
    unsigned short round_up;
} __attribute__((packed));      /* struct cb_page_info */

/* @INLINE8END@ */

#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
struct rankheap {
    struct control_block *root;
};
#endif

struct control_block_heap {
    struct heap heap;
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
    struct rankheap rankheap;
#endif
#ifdef USE_TLS
    cb_size_t first_sharedfree_item;
#endif
};

/* @INLINE6START@ */

struct control_block_head {
    struct control_block *partial_blocks_first;
};

struct cb_heaps {
    LONG cb_refs;
#ifdef WITH_ACCOUNTS
    LONG total_alloced;
    LONG cb_total_blocks;
    LONG total_free;
#endif
/* @INLINE6END@ */
    struct control_block_heap heaps[CB_SIZE_MAX];
    struct control_block_head cb_heads[CB_N_HEADS];
    LONG total_heap;
#ifndef NO_HEAP_TRUNCATION
    unsigned int flush;
#endif
#ifndef USE_TLS
    dfwmalloc_mutex_t mutex;
#endif
};

struct cb_heaps_info {
#ifdef USE_PRECOMPILED_STATIC_DATA
    const struct cb_page_info *page_info;
    const cb_headint_t *head_index;
#else
    struct cb_page_info page_info[CB_N_HEADS];
    cb_headint_t head_index[CB_N_SIZES + 1];
#endif
    int max_head;
};

#endif

#if !defined(NO_LOCKFREEVISITOR) && defined(USE_TLS)

struct lfitem {
    void *lft_ptr;
#ifdef WITH_BTANDCHECK
    struct calltrace lft_calltrace;
#endif
};

#ifdef WITH_BTANDCHECK
typedef unsigned char lfv_counter_t;
#define N_LFV_ITEMS                     256
#else
typedef unsigned short lfv_counter_t;
#define N_LFV_ITEMS                     65536
#endif
#define LFV_ARENAID_UNINITIALIZED       0x7FFFFFFFUL


struct lockfreevisitor {
    struct lfitem *lfv_d;
    lfv_counter_t lfv_visitor;
    lfv_counter_t lfv_owner;
    unsigned long lfv_foreign_arena_id;
};

#endif

#ifndef NO_LOCKFREEVISITOR
#define dfwmalloc_shared_free_count(dfw)           ((dfw)->shared_free_count + lfv_count(&(dfw)->lfv))
#else
#define dfwmalloc_shared_free_count(dfw)           ((dfw)->shared_free_count)
#endif

struct dfwmalloc {
#define DFWMALLOC_MAGIC          0x8ff4c8b
    unsigned int dfwmalloc_magic;
#ifdef USE_TLS
    int released;
    int shared_free_count;
    ULONG64 release_age;
#ifndef NO_LOCKFREEVISITOR
    struct lockfreevisitor lfv;
#endif
    dfwmalloc_mutex_t sharedfree_mutex;
    struct free_item *sharedfree_list;
#endif
    int arena_id;
    struct free_list *free_list;
#ifndef NO_CB_HEAPS
    struct cb_heaps cb_heaps;
#endif
#ifndef NO_HEAP_TRUNCATION
    struct free_item *last_block;
#endif
    int free_list_initialized;
#ifdef WITH_BTANDCHECK
    struct backtrace backtrace;
#endif
#ifdef WITH_ACCOUNTS
    LONG total_alloced;
#ifdef WITH_BTANDCHECK
    LONG total_alloced_bt;
#endif
#endif
#if defined(WITH_ACCOUNTS) || defined(USE_TLS)
    LONG total_blocks;
#ifdef WITH_BTANDCHECK
    LONG total_blocks_bt;
#endif
#endif
    size_t max_alloc;
    unsigned int flush;
    struct heap *heap;
};

#ifdef HAVE_PTHREAD

#ifdef __linux__

#define DFWMALLOC_MUTEX_INITIALIZER      PTHREAD_MUTEX_INITIALIZER

#define dfwmalloc_mutex_trylock(s)       pthread_mutex_trylock(s)
#define dfwmalloc_mutex_lock(s)          pthread_mutex_lock(s)
#define dfwmalloc_mutex_unlock(s)        pthread_mutex_unlock(s)
#define dfwmalloc_mutex_init(s,a)        pthread_mutex_init(s,a)

#else

/* We cannot use pthread_mutex functions because some operating systems
 * call malloc from within these functions: */

#define DFWMALLOC_MUTEX_INITIALIZER      0L

STATICFNINLINE int dfwmalloc_mutex_trylock (dfwmalloc_mutex_t *mutex)
{
    if (!__sync_bool_compare_and_swap (mutex, 0UL, 1UL))
        return -EBUSY;
    return 0;
}

STATICFNINLINE void dfwmalloc_mutex_lock (dfwmalloc_mutex_t *mutex)
{
    while (dfwmalloc_mutex_trylock (mutex))
        pthread_yield ();
}

STATICFNINLINE void dfwmalloc_mutex_unlock (dfwmalloc_mutex_t *mutex)
{
    *mutex = 0L;
}

STATICFNINLINE void dfwmalloc_mutex_init (dfwmalloc_mutex_t *mutex, void *attr)
{
    (void) attr;
    *mutex = DFWMALLOC_MUTEX_INITIALIZER;
}

#endif

#else

#define dfwmalloc_mutex_trylock(s)        (0)
#define dfwmalloc_mutex_lock(s)           do { } while (0)
#define dfwmalloc_mutex_unlock(s)         do { } while (0)
#define dfwmalloc_mutex_init(s,a)         do { } while (0)

#endif

#ifdef HAVE_PTHREAD
static dfwmalloc_mutex_t dfwmalloc_global_lock = DFWMALLOC_MUTEX_INITIALIZER;
#endif



/* 
 * The linux stack is around 0x7ffdd7c5c6fc
 * We map all heaps below 0x7e0000000000
 * which is the same as 0x7e00 << 32
 * 
 * 
 *                           CB_SIZE_BITS
 *                                |
 *                          ______^______
 *                         /             \
 * 
 * +--------------------+-------------------+--------------------------------------------------------------+
 * | 0x7E  -  arena_id  |    heap_index     |     useful bits:  47 - N_ARENAS_BITS - 2 - CB_SIZE_BITS      |
 * +--------------------+-------------------+--------------------------------------------------------------+
 *                                                                                                          
 * +--------------------+-------------------+--------------------------------------------------------------+
 * | 0x7E  -  arena_id  |  > heap_index     |       useful bits:  47 - N_ARENAS_BITS - 2                   |
 * +--------------------+-------------------+--------------------------------------------------------------+
 * 
 * 
 */

#ifdef USE_TLS

#define TOP_ADDRESS_U           0x7e000000U
#define TOP_ADDRESS_L           0x7e0000000000UL
#define BOTTOM_ADDRESS_L        0x8000000000UL

#define ARENA_FROM_POINTER(u)  ((int) ((TOP_ADDRESS_U >> (47 - N_ARENAS_BITS - 16)) - ((ULONG64) (u) >> (47 - N_ARENAS_BITS))) - 1)

#define POINTER_FROM_ARENA(u,arena_id,idx) \
    do { \
        (u) = (ULONG64) TOP_ADDRESS_L; \
        (u) -= (ULONG64) ((arena_id) + 1) << (47 - N_ARENAS_BITS); \
        (u) &= (ULONG64) 0xFFFFFFFF << (47 - N_ARENAS_BITS); \
        (u) |= (ULONG64) (idx) << (47 - N_ARENAS_BITS - CB_SIZE_BITS); \
    } while (0)

#else

#define ARENA_FROM_POINTER(u)  ((int) 0)

#define POINTER_FROM_ARENA(u,arena_id,idx) \
    do { \
        (u) = (ULONG64) 0x400000000000UL; \
        (u) |= (ULONG64) (idx) << (47 - N_ARENAS_BITS - CB_SIZE_BITS); \
    } while (0)

#endif


struct arena {
#ifdef USE_TLS
#define OCCUPIED_NOT            0
#define OCCUPIED_INITIALIZING   1
#define OCCUPIED_USED           2
#define OCCUPIED_FREEING        3
#define OCCUPIED_RESERVED       4
    int occupied;
#endif
    int arena_id;
    struct heap heap;
    struct dfwmalloc *dfwmalloc;
};

struct global {
    int initialized;
#ifdef USE_TLS
    struct arena arena[N_ARENAS];
    int n_arenas;
    ULONG64 release_age;
    ULONG64 defunct_arenas;
#else
    struct arena arena;
#endif
#ifdef WITH_BTANDCHECK
    int backtrace_enabled;
    int backtrace_on_exit;
#endif
#if !defined(NO_LOCKFREEVISITOR) && defined(USE_TLS)
    int lfv_enabled;
#endif

/* the entries below are constant data after startup: */
#ifdef USE_PRECOMPILED_STATIC_DATA
    const malloc_size_t *bucket_size;
#else
    malloc_size_t *bucket_size;
#endif
    int n_freelists;
    ULONG max_heap;
#ifndef NO_CB_HEAPS
    ULONG max_cb_heap;
    struct cb_heaps_info cb_heaps_info;
#endif
};

static struct global global_;
static struct global *global = &global_;


#ifdef USE_PRECOMPILED_STATIC_DATA
#include "staticdata.c"
#endif

#ifdef USE_PRECOMPILED_STATIC_DATA_DEBUG
#include "staticdata-debug.c"
#endif

#ifdef USE_PRECOMPILED_STATIC_DATA_COMPACT
#include "staticdata-compact.c"
#endif

#ifdef USE_PRECOMPILED_STATIC_DATA_ENTERPRISE
#include "staticdata-enterprise.c"
#endif


#ifdef USE_TLS
__thread struct dfwmalloc *dfwmalloc_tls = NULL;
#ifdef WITH_BTANDCHECK
__thread int dfwmalloc_backtrace_reentry = 0;
#endif
#else
static struct dfwmalloc *dfwmalloc_tls = NULL;
#ifdef WITH_BTANDCHECK
static int dfwmalloc_backtrace_reentry = 0;
#endif
#endif



#if !defined(NO_LOCKFREEVISITOR) && defined(USE_TLS)

STATICFN void lfv_init (struct lockfreevisitor *lfv)
{
    lfv->lfv_d = NULL;
    lfv->lfv_visitor = 0;
    lfv->lfv_owner = 0;
    lfv->lfv_foreign_arena_id = LFV_ARENAID_UNINITIALIZED;
}

STATICFN lfv_counter_t lfv_count (struct lockfreevisitor *lfv)
{
    return lfv->lfv_visitor - lfv->lfv_owner;
}

STATICFN int lfv_push (int my_arena_id, struct lockfreevisitor *lfv, void *u, struct calltrace *c)
{
    (void) c;
    if (!global->lfv_enabled)
        return 1;
    if (!lfv->lfv_d)
        return 1;
    while (lfv->lfv_foreign_arena_id == LFV_ARENAID_UNINITIALIZED)
        if (__sync_bool_compare_and_swap (&lfv->lfv_foreign_arena_id, LFV_ARENAID_UNINITIALIZED, (unsigned long) my_arena_id))
            break;
    if (lfv->lfv_foreign_arena_id != (unsigned long) my_arena_id)
        return 1;       /* we are not the primary free'er of the foreign thread */
    if (lfv_count (lfv) > N_LFV_ITEMS - 3)
        return 1;               /* full */
    lfv->lfv_d[lfv->lfv_visitor].lft_ptr = u;
#ifdef WITH_BTANDCHECK
    lfv->lfv_d[lfv->lfv_visitor].lft_calltrace = *c;
#endif
    __sync_synchronize ();
    lfv->lfv_visitor++;
    return 0;
}

STATICFN void *lfv_pop (struct lockfreevisitor *lfv, struct calltrace *c)
{
    void *u;
    (void) c;
    if (!lfv_count (lfv))
        return NULL;               /* empty */
    u = lfv->lfv_d[lfv->lfv_owner].lft_ptr;
#ifdef WITH_BTANDCHECK
    *c = lfv->lfv_d[lfv->lfv_owner].lft_calltrace;
#endif
    __sync_synchronize ();
    lfv->lfv_owner++;
    return u;
}

#endif


#ifdef USE_TLS

STATICFN int dfwmalloc_get_oldest_released_arena (struct global *g)
{
    int i, oldest = -1;
    ULONG64 oldest_age = 0;

    for (i = 0; i < g->n_arenas; i++) {
        if (g->arena[i].occupied == OCCUPIED_USED) {
            struct dfwmalloc *dfw;
            dfw = g->arena[i].dfwmalloc;
            if (dfw && dfw->released) {
                if (oldest == -1 || oldest_age > dfw->release_age) {
                    oldest = i;
                    oldest_age = dfw->release_age;
                }
            }
        }
    }

    return oldest;
}

STATICFN struct arena *dfwmalloc_get_arena (struct global *g)
{
    struct arena *t = NULL;
#ifdef LOG_ARENAS
    char ms[256];
#endif
    int i;
    for (i = 0; i < g->n_arenas; i++) {
        if (g->arena[i].occupied == OCCUPIED_NOT) {
#ifdef LOG_ARENAS
            snprintf_simple (ms, sizeof (ms), "new arena, found. n=%d", (int) g->n_arenas);
            DFWMALLOC_LOG (ms, (int) strlen (ms));
#endif
            t = &g->arena[i];
            t->arena_id = i;
            t->occupied = OCCUPIED_INITIALIZING;
            return t;
        }
    }

    while (g->n_arenas < (int) N_ARENAS) {
        i = g->n_arenas++;
        if (g->arena[i].occupied == OCCUPIED_NOT) {
#ifdef LOG_ARENAS
            snprintf_simple (ms, sizeof (ms), "new arena, extra. n=%d", (int) g->n_arenas);
            DFWMALLOC_LOG (ms, (int) strlen (ms));
#endif
            t = &g->arena[i];
            t->arena_id = i;
            t->occupied = OCCUPIED_INITIALIZING;
            return t;
        }
    }

/* The user is continually calling pthread_create+pthread_exit while leaking memory each time.
* So we recycle the oldest arena: */
    i = dfwmalloc_get_oldest_released_arena (g);
    if (i == -1)
        return NULL;

#ifdef LOG_ARENAS
    snprintf_simple (ms, sizeof (ms), "new arena, re-purpose. n=%d", (int) g->n_arenas);
    DFWMALLOC_LOG (ms, (int) strlen (ms));
#endif

/* Reuse arena: */
    t = &g->arena[i];
    dfwmalloc_assert (t->occupied);
    dfwmalloc_assert (t->dfwmalloc);
    dfwmalloc_assert (t->arena_id == i);
    t->dfwmalloc->released = 0;

/* This counter is only here for debugging. If something produces a core and it has lots of these, then you know: */
    g->defunct_arenas++;

#ifdef TEST_FOR_REUSING_THREADS
    printf_simple ("refs=%d %d %d\n", (int) t->dfwmalloc->total_blocks_bt, (int) t->dfwmalloc->total_blocks, (int) t->dfwmalloc->cb_heaps.cb_refs);
#endif

    return t;
}

STATICFN void dfwmalloc_release_arena (struct arena *t)
{
    t->occupied = OCCUPIED_FREEING;
    __sync_synchronize ();
    t->arena_id = -1;
    dfwmalloc_assert (t->occupied);
    t->dfwmalloc = NULL;
    __sync_synchronize ();
    t->occupied = OCCUPIED_NOT;
}

#endif


STATICFN int dfwmalloc_heap_init (struct heap *h, int arena_id);
#ifdef USE_TLS
STATICFN void dfwmalloc_heap_free (struct heap *h);
#endif
STATICFN void *dfwmalloc_heap_rawalloc_locked (struct heap *h, LONG n, int alignment, int *need_zero);
STATICFN int dfwmalloc_startup (struct dfwmalloc *dfw, struct heap *heap, int arena_id);
#ifdef USE_TLS
STATICFN void dfwmalloc_free_shared_pointers (struct dfwmalloc *dfw, struct calltrace *c);
STATICFNNOINLINE void dfwmalloc_free_shared_pointers_ (struct dfwmalloc *dfw, struct calltrace *c);
#endif
#if !defined(NO_LOCKFREEVISITOR) && defined(USE_TLS)
STATICFNNOINLINE void dfwmalloc_lfv_free_shared (struct dfwmalloc *dfw);
#endif
STATICFNINLINE int dfwmalloc_get_bucket (struct dfwmalloc *dfw, LONG n);
STATICFN void dfwmalloc_zero_free_list_head (struct dfwmalloc *dfw, int k);
#ifdef WITH_BTANDCHECK
STATICFN void dfwmalloc_poison_check_block (struct dfwmalloc *dfw, const struct free_item *p, const char *s, struct calltrace *c);
STATICFN void dfwmalloc_alloc_backtrace_head (struct dfwmalloc *dfw);
STATICFN backtrace_id_t dfwmalloc_backtrace (struct dfwmalloc *dfw, struct backtrace *g, struct calltrace *c);
STATICFN void dfwmalloc_backtrace_account (struct dfwmalloc *dfw, backtrace_id_t backtrace_id, LONG alloced, LONG blocks);
#endif
#ifndef NO_CB_HEAPS
STATICFN void dfwmalloc_cb_destroy_heap (struct cb_heaps *cbhs);
STATICFN void dfwmalloc_cb_init_page_info (struct cb_heaps_info *cb_heaps_info);
#endif
#if !defined(USE_PRECOMPILED_STATIC_DATA) || defined(WITH_BTANDCHECK)
STATICFN void *dfwmalloc_raw_calloc (struct dfwmalloc *dfw, LONG n);
#endif

STATICFN void dfwmalloc_initialize_global (struct global *g)
{
#ifdef USE_TLS
    int i;
#endif
    if (g->initialized)
        return;
    g->initialized = 1;
#if !defined(NO_LOCKFREEVISITOR) && defined(USE_TLS)
    g->lfv_enabled = 1;
#endif
    g->max_heap = (ULONG) MAX_HEAP;
#ifdef USE_TLS
    if (g->max_heap > (ULONG) MAX_TLS_HEAP)
        g->max_heap = (ULONG) MAX_TLS_HEAP;
/* avoid ranges reserved by kernel: */
    for (i = 0; i < N_ARENAS; i++) {
        ULONG64 v_start, v_end;
        POINTER_FROM_ARENA (v_end, i, 0);
        POINTER_FROM_ARENA (v_start, i + 1, 0);
        if (v_end <= v_start)
            g->arena[i].occupied = OCCUPIED_RESERVED;
        else if (v_end <= BOTTOM_ADDRESS_L || v_start < BOTTOM_ADDRESS_L)
            g->arena[i].occupied = OCCUPIED_RESERVED;

/* 
 * https://github.com/gcc-mirror/gcc/blob/master/libsanitizer/tsan/tsan_platform.h
 * 
 * 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
 * 0001 0000 0000 - ???? 0000 0000: See eab09532d40090698b05a07c1c87f39fdbc5fab5
 * 0040 0000 0000 - 0100 0000 0000: -
 * 0100 0000 0000 - 2000 0000 0000: shadow
 * 2000 0000 0000 - 3000 0000 0000: -
 * 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 * 4000 0000 0000 - 5500 0000 0000: -
 * 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
 * 5680 0000 0000 - 6000 0000 0000: -
 * 6000 0000 0000 - 6200 0000 0000: traces
 * 6200 0000 0000 - 7d00 0000 0000: -
 * 7b00 0000 0000 - 7c00 0000 0000: heap
 * 7c00 0000 0000 - 7e80 0000 0000: -
 * 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
 */

        else if (!(v_end <= 0x550000000000UL || v_start >= 0x568000000000UL))
            g->arena[i].occupied = OCCUPIED_RESERVED;
#ifdef AVOID_ADDRESS_RANGE_1_START
        else if (!(v_end <= AVOID_ADDRESS_RANGE_1_START || v_start >= AVOID_ADDRESS_RANGE_1_END))
            g->arena[i].occupied = OCCUPIED_RESERVED;
#endif
#ifdef AVOID_ADDRESS_RANGE_2_START
        else if (!(v_end <= AVOID_ADDRESS_RANGE_2_START || v_start >= AVOID_ADDRESS_RANGE_2_END))
            g->arena[i].occupied = OCCUPIED_RESERVED;
#endif
#ifdef AVOID_ADDRESS_RANGE_3_START
        else if (!(v_end <= AVOID_ADDRESS_RANGE_3_START || v_start >= AVOID_ADDRESS_RANGE_3_END))
            g->arena[i].occupied = OCCUPIED_RESERVED;
#endif
#ifdef AVOID_ADDRESS_RANGE_4_START
        else if (!(v_end <= AVOID_ADDRESS_RANGE_4_START || v_start >= AVOID_ADDRESS_RANGE_4_END))
            g->arena[i].occupied = OCCUPIED_RESERVED;
#endif
#ifdef AVOID_ADDRESS_RANGE_5_START
        else if (!(v_end <= AVOID_ADDRESS_RANGE_5_START || v_start >= AVOID_ADDRESS_RANGE_5_END))
            g->arena[i].occupied = OCCUPIED_RESERVED;
#endif
        else if (v_end > TOP_ADDRESS_L || v_start >= TOP_ADDRESS_L)
            g->arena[i].occupied = OCCUPIED_RESERVED;
    }
#endif
#ifndef NO_CB_HEAPS
    g->max_cb_heap = (ULONG) MAX_CB_HEAP;
    dfwmalloc_cb_init_page_info (&g->cb_heaps_info);
#endif
}

STATICFN struct dfwmalloc *dfwmalloc_get_dfwmalloc (struct global *g, struct calltrace *c)
{
    struct arena *t;
    struct dfwmalloc *dfw;
    (void) c;
    dfwmalloc_initialize_global (g);
#ifdef USE_TLS
    t = dfwmalloc_get_arena (g);
    if (!t) {
        const char *msg_ = "DFWMALLOC: out of arenas";
        DFWMALLOC_LOG (msg_, (int) strlen (msg_));
        errno = ENOMEM;
        return NULL;
    }
#else
    t = &g->arena;
#endif
#ifdef USE_TLS
/* this thread is taking over as new master of this arena: */
    if (t->dfwmalloc) {
        dfwmalloc_assert (!t->dfwmalloc->released);
        dfwmalloc_assert (t->dfwmalloc->dfwmalloc_magic == DFWMALLOC_MAGIC);
        return t->dfwmalloc;
    }
#endif
    if (dfwmalloc_heap_init (&t->heap, t->arena_id)) {
#ifdef USE_TLS
        dfwmalloc_release_arena (t);
#endif
        return NULL;
    }
#ifndef USE_TLS
    dfwmalloc_mutex_init (&t->heap.heap_mutex, NULL);
#endif
    dfw = t->dfwmalloc = (struct dfwmalloc *) dfwmalloc_heap_rawalloc_locked (&t->heap, sizeof (struct dfwmalloc), POINTER_ALIGNMENT, NULL);
    if (!dfw) {
#ifdef USE_TLS
        dfwmalloc_heap_free (&t->heap);
        dfwmalloc_release_arena (t);
#endif
        return NULL;
    }
#ifndef NO_HEAP_TRUNCATION
    dfw->last_block = NULL;
#endif
    if (dfwmalloc_startup (dfw, &t->heap, t->arena_id)) {
#ifdef USE_TLS
        dfwmalloc_heap_free (&t->heap);
        dfwmalloc_release_arena (t);
#endif
        return NULL;
    }

#ifdef WITH_BTANDCHECK
    if (global->backtrace_enabled) {
        dfwmalloc_alloc_backtrace_head (dfw);
        {
            backtrace_id_t trace;
            trace = dfwmalloc_backtrace (dfw, &dfw->backtrace, c);
            dfwmalloc_backtrace_account (dfw, trace, (LONG) sizeof (struct dfwmalloc), 1L);
        }
    }
#endif

#ifdef WITH_ACCOUNTS
#ifdef WITH_BTANDCHECK
    if (global->backtrace_enabled)
        dfw->total_alloced_bt = (LONG) sizeof (struct dfwmalloc);
    else
#endif
        dfw->total_alloced = (LONG) sizeof (struct dfwmalloc);
#endif

#if defined(WITH_ACCOUNTS) || defined(USE_TLS)
#ifdef WITH_BTANDCHECK
    if (global->backtrace_enabled)
        dfw->total_blocks_bt = 1L;
    else
#endif
        dfw->total_blocks = 1L;
#endif

    dfw->dfwmalloc_magic = DFWMALLOC_MAGIC;
    __sync_synchronize ();
#ifdef USE_TLS
    t->occupied = OCCUPIED_USED;
#endif

    return t->dfwmalloc;
}

#ifdef USE_TLS

STATICFN void dfwmalloc_force_free_dfwmalloc (struct global *g, struct dfwmalloc *dfw)
{
    int arena_id;
    arena_id = dfw->arena_id;
    g->arena[arena_id].occupied = OCCUPIED_FREEING;
    __sync_synchronize ();
    dfw->dfwmalloc_magic = 0;
#ifndef NO_CB_HEAPS
    dfwmalloc_cb_destroy_heap (&dfw->cb_heaps);
#endif
    dfwmalloc_heap_free (dfw->heap);
    if (dfwmalloc_tls == dfw)
        dfwmalloc_tls = NULL;
    __sync_synchronize ();
    dfwmalloc_release_arena (&g->arena[arena_id]);
}

STATICFNNOINLINE int dfwmalloc_try_free_dfwmalloc (struct global *g, struct dfwmalloc *dfw, struct calltrace *c)
{
    LONG refs;

    dfwmalloc_assert (dfw->dfwmalloc_magic == DFWMALLOC_MAGIC);
    dfwmalloc_assert (dfw->heap);
    dfwmalloc_assert (dfw->heap->start_of_heap);
    dfwmalloc_assert (dfw->heap->end_of_heap);

    if (dfwmalloc_shared_free_count (dfw))
        dfwmalloc_free_shared_pointers_ (dfw, c);

/* 
 * Throughout this code base, the members total_blocks, total_blocks_bt,
 * and cb_refs are decremented as the last statement before returning to
 * the user. I.e. no access to struct dfwmalloc is performed after the
 * decrement. The decrement is assured to be the last access with
 * the optimization protection 'asm volatile ("":::"memory")'
 */

    refs = dfw->total_blocks;
#ifdef WITH_BTANDCHECK
    refs += dfw->total_blocks_bt;
#endif
#ifndef NO_CB_HEAPS
    refs += dfw->cb_heaps.cb_refs;
#endif
    dfwmalloc_assert (refs >= 1L);

    if (refs > 1L) {
        dfw->release_age = ++g->release_age;
        __sync_synchronize ();
        dfw->released = 1;
        return 1;
    }
    __sync_synchronize ();

    dfwmalloc_assert (dfw->sharedfree_list == NULL);
    dfwmalloc_assert (!dfw->shared_free_count);
#ifndef NO_LOCKFREEVISITOR
    dfwmalloc_assert (dfw->lfv.lfv_visitor == dfw->lfv.lfv_owner);
#endif
    dfwmalloc_force_free_dfwmalloc (g, dfw);
    return 0;
}

#endif

#if (defined(WITH_BTANDCHECK) && !defined(NO_OVERRIDE_EXIT_FUNCTIONS)) || (!defined(NO_PTHREAD_EXIT_OVERRIDE) && defined(USE_TLS) && defined(HAVE_PTHREAD))
typedef void (*dlsym_generitc_fn_t) (void);

STATICFN dlsym_generitc_fn_t dfwmalloc_dlsym (const char *symname)
{
    return (dlsym_generitc_fn_t) dlsym (RTLD_NEXT, symname);
}
#endif

#if defined(WITH_ACCOUNTS) || defined(WITH_BTANDCHECK) || defined(USE_TLS)
STATICFNINLINE void dfwmalloc_synchronous_add (volatile LONG * p, LONG u)
{
#if defined(HAVE_PTHREAD) && !defined(USE_TLS)
    (void) __sync_fetch_and_add (p, u);
#else
    *p = *p + u;
#endif
}
#endif

#ifdef WITH_BTANDCHECK

STATICFN void dfwmalloc_backtrace_if_enabled (struct calltrace *c)
{
    if (global->backtrace_enabled) {
#if defined(HAVE_PTHREAD) && !defined(USE_TLS)
        static dfwmalloc_mutex_t mutex = DFWMALLOC_MUTEX_INITIALIZER;
#endif
        c->n = 0;
#if defined(HAVE_PTHREAD) && !defined(USE_TLS)
        dfwmalloc_mutex_lock (&mutex);
#endif
        if (!dfwmalloc_backtrace_reentry) {
            dfwmalloc_backtrace_reentry = 1;
            c->n = backtrace (c->addr, CALLTRACE_DEPTH);
            dfwmalloc_backtrace_reentry = 0;
        }
#if defined(HAVE_PTHREAD) && !defined(USE_TLS)
        dfwmalloc_mutex_unlock (&mutex);
#endif
        return;
    }
    c->n = 0;
}

STATICFN void dfwmalloc_at_exit_fn (void)
{
    if (global->backtrace_on_exit) {
        global->backtrace_on_exit = 0;
        dfwmalloc_log_leaks ();
    }
}

#ifndef NO_OVERRIDE_EXIT_FUNCTIONS

typedef void (*posix_exit_fn_t) (int);

void _exit (int r)
{
    posix_exit_fn_t posix_exit_fn = NULL;
    posix_exit_fn = (posix_exit_fn_t) dfwmalloc_dlsym ("_exit");
    dfwmalloc_at_exit_fn ();
    (*posix_exit_fn) (r);
    DFWMALLOC_LOG ("return from _exit() - ???", 25);
    abort ();
}

void _Exit (int r)
{
    posix_exit_fn_t posix_exit_fn = NULL;
    posix_exit_fn = (posix_exit_fn_t) dfwmalloc_dlsym ("_Exit");
    dfwmalloc_at_exit_fn ();
    (*posix_exit_fn) (r);
    DFWMALLOC_LOG ("return from _Exit() - ???", 25);
    abort ();
}

void exit (int r)
{
    posix_exit_fn_t posix_exit_fn = NULL;
    posix_exit_fn = (posix_exit_fn_t) dfwmalloc_dlsym ("exit");
    dfwmalloc_at_exit_fn ();
    (*posix_exit_fn) (r);
    DFWMALLOC_LOG ("return from exit() - ???", 24);
    abort ();
}

#endif

__attribute__ ((constructor))
static void dfwmalloc_constructor ()
{
    char *e;
    if (getenv ("DFWMALLOC_BACKTRACE")) {
        atexit (dfwmalloc_at_exit_fn);
        dfwmalloc_enable_backtrace ();
        global->backtrace_on_exit = 1;
    }
    e = getenv ("DFWMALLOC_STARTUP_MESSAGE");
    if (e) {
        char *t;
        dfwmalloc_writestr ("DFWMalloc is being used by process ");
        t = itoa_base ((ULONG64) getpid (), 1, 10);
        dfwmalloc_writestr (t);
        dfwmalloc_writestr (" \"");
        dfwmalloc_writestr (e);
        dfwmalloc_writestr ("\"\n");
    }
}

#else
#define dfwmalloc_backtrace_if_enabled(c)        do { } while(0)
#endif


#ifdef USE_TLS

#define INIT__(dfw, c, ret_stmnt, shrd) \
    do { \
        dfwmalloc_backtrace_if_enabled (&(c)); \
        if (__builtin_expect (!dfwmalloc_tls, 0)) { \
            dfwmalloc_mutex_lock (&dfwmalloc_global_lock); \
            dfwmalloc_tls = dfwmalloc_get_dfwmalloc (global, &(c)); \
            dfwmalloc_mutex_unlock (&dfwmalloc_global_lock); \
            if (!dfwmalloc_tls) \
                ret_stmnt; \
        } \
        (dfw) = dfwmalloc_tls; \
        dfwmalloc_assert ((dfw)->dfwmalloc_magic == DFWMALLOC_MAGIC); \
        if (__builtin_expect ((shrd), 0)) /* inline this check as an optimization */ \
            dfwmalloc_free_shared_pointers ((dfw), &(c)); \
    } while (0)

#define INIT_NOCHK(dfw, c, ret_stmnt) \
    do { \
        dfwmalloc_backtrace_if_enabled (&(c)); \
        (dfw) = dfwmalloc_tls; \
        /* dfwmalloc_assert (!(dfw)->released); */ \
        dfwmalloc_assert ((dfw) == NULL || (dfw)->dfwmalloc_magic == DFWMALLOC_MAGIC); \
        if (__builtin_expect ((dfw) && dfwmalloc_shared_free_count (dfw), 0)) /* inline this check as an optimization */ \
            dfwmalloc_free_shared_pointers ((dfw), &(c)); \
    } while (0)

#define INIT(dfw, c, ret_stmnt) \
    do { \
        INIT__(dfw, c, ret_stmnt, dfwmalloc_shared_free_count (dfw)); \
    } while (0)

#define INIT_SELDOMSHRDFREE(dfw, c, ret_stmnt) \
    do { \
        INIT__(dfw, c, ret_stmnt, dfwmalloc_shared_free_count (dfw) > 100); \
    } while (0)

#define INIT_NOSHRDFREE(dfw, c, ret_stmnt) \
    do { \
        INIT__(dfw, c, ret_stmnt, 0); \
    } while (0)

#define INITF(dfw, c, ret_stmnt) \
    do { \
        dfwmalloc_backtrace_if_enabled (&(c)); \
        dfwmalloc_assert (!(dfw)->released); \
        if (__builtin_expect (dfwmalloc_shared_free_count (dfw), 0)) /* inline this check as an optimization */ \
            dfwmalloc_free_shared_pointers ((dfw), &(c)); \
        dfwmalloc_assert ((dfw)->dfwmalloc_magic == DFWMALLOC_MAGIC); \
    } while (0)

#else

#define INIT__(dfw, c, ret_stmnt) \
    do { \
        dfwmalloc_backtrace_if_enabled (&(c)); \
        if (__builtin_expect (!dfwmalloc_tls, 0)) { \
            dfwmalloc_mutex_lock (&dfwmalloc_global_lock); \
            dfwmalloc_tls = dfwmalloc_get_dfwmalloc (global, &(c)); \
            dfwmalloc_mutex_unlock (&dfwmalloc_global_lock); \
            if (!dfwmalloc_tls) \
                ret_stmnt; \
        } \
        (dfw) = dfwmalloc_tls; \
        dfwmalloc_assert ((dfw)->dfwmalloc_magic == DFWMALLOC_MAGIC); \
    } while (0)

#define INIT(dfw, c, ret_stmnt)                   INIT__(dfw, c, ret_stmnt)
#define INIT_SELDOMSHRDFREE(dfw, c, ret_stmnt)    INIT__(dfw, c, ret_stmnt)
#define INIT_NOSHRDFREE(dfw, c, ret_stmnt)        INIT__(dfw, c, ret_stmnt)

#define INIT_NOCHK(dfw, c, ret_stmnt) \
    do { \
        dfwmalloc_backtrace_if_enabled (&(c)); \
        (dfw) = dfwmalloc_tls; \
        dfwmalloc_assert ((dfw) == NULL || (dfw)->dfwmalloc_magic == DFWMALLOC_MAGIC); \
    } while (0)

#define INITF(dfw, c, ret_stmnt) \
    do { \
        dfwmalloc_backtrace_if_enabled (&(c)); \
        dfwmalloc_assert ((dfw)->dfwmalloc_magic == DFWMALLOC_MAGIC); \
    } while (0)

#endif


#define MALLOC_ZERO             ((void *) malloc_zero)
static const char malloc_zero[1024] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";

#ifdef WITH_BTANDCHECK
STATICFN void dfwmalloc_print_backtrace (const char *msg, void **addr, int n)
{
    char s[256] = "    ";
    int i, l;
    DFWMALLOC_LOG (msg, strlen (msg));
    i = 0;
    l = strlen (s);
    while (l < (int) sizeof (s) && i < n)
	l += snprintf_simple (s + l, sizeof (s) - l, "%p ", addr[i++]);
    DFWMALLOC_LOG (s, l);
}
#endif

struct dfwmalloc;

STATICFN void dfwmalloc_assert_ (const char *file, int line, const char *msg)
{
#ifdef WITH_BTANDCHECK
    struct calltrace c;
#endif
    char s[256] = "";

    strlimcat (s, sizeof (s), file);
    strlimcat (s, sizeof (s), ":");
    strlimcat (s, sizeof (s), itoa_base (line, 1, 10));
    strlimcat (s, sizeof (s), ": assert fail: (");
    strlimcat (s, sizeof (s), msg);
    strlimcat (s, sizeof (s), ")");

    DFWMALLOC_LOG (s, strlen (s));

#ifdef WITH_BTANDCHECK
    dfwmalloc_backtrace_if_enabled (&c);
    if (c.n)
	dfwmalloc_print_backtrace ("Backtrace ==>", c.addr, c.n);
#endif
}


#ifdef WITH_BTANDCHECK

#define USER_ALERT(cond, msg) \
    do { \
        if (!(cond)) \
            dfwmalloc_user_alert (dfw, c, msg, p); \
    } while (0)

#else

#define USER_ALERT(cond, msg) do { } while (0)

#endif

#ifdef USE_TLS
STATICFN struct dfwmalloc *dfwmalloc_find_pointer (struct global *g, void *u)
{
    int i;
    i = ARENA_FROM_POINTER (u);
    if (i >= 0 && i < g->n_arenas && g->arena[i].occupied == OCCUPIED_USED)
        return g->arena[i].dfwmalloc;
    return NULL;
}
#endif

#ifdef WITH_BTANDCHECK

STATICFN void dfwmalloc_backtrace_array (struct dfwmalloc *dfw, struct backtrace *g, ULONG i, struct backtrace_item ***b)
{
    ULONG hi, low;
    hi = i >> BACKTRACE_ID_LOW;
    low = i & ((1UL << BACKTRACE_ID_LOW) - 1);
    if (!g->backtrace_array[hi]) {
        g->backtrace_array[hi] = (struct backtrace_item **) dfwmalloc_raw_calloc (dfw, sizeof (struct backtrace_item *) * (1UL << BACKTRACE_ID_LOW));
/* Reduce the chance another thread could try fill this in: */
        g->backtrace_array[hi + 1] = (struct backtrace_item **) dfwmalloc_raw_calloc (dfw, sizeof (struct backtrace_item *) * (1UL << BACKTRACE_ID_LOW));
    }
    *b = &g->backtrace_array[hi][low];
}

STATICFN void dfwmalloc_user_alert (struct dfwmalloc *dfw, struct calltrace *c, const char *msg,
                                const struct free_item *p)
{
    DFWMALLOC_LOG (msg, strlen (msg));
    if (global->backtrace_enabled) {
        struct backtrace_item **b;
        dfwmalloc_print_backtrace ("Found here ==>", c->addr, c->n);
#ifdef USE_TLS
        if (!dfw && p) {
            dfw = dfwmalloc_find_pointer (global, (void *) p);
        }
#endif
        if (dfw) {
            ULONG malloc_trace, free_trace;
            malloc_trace = backtrace_id_to_long (&p->malloc_trace);
            free_trace = backtrace_id_to_long (&p->free_trace);
            if (malloc_trace != INVALID_BACKTRACE_ID) {
                dfwmalloc_backtrace_array (dfw, &dfw->backtrace, malloc_trace, &b);
                dfwmalloc_print_backtrace ("Alloced here ==>", (*b)->addr, (*b)->n);
            }
            if (free_trace != INVALID_BACKTRACE_ID) {
                dfwmalloc_backtrace_array (dfw, &dfw->backtrace, free_trace, &b);
/* This could crash because free_trace can be allocated in another thread due to (*11*).
 * That thread could have cleaned out its heap, thus *b points to nowhere.
 * But no matter, the user has seen the log at this point: */
                dfwmalloc_print_backtrace ("Freed here ==>", (*b)->addr, (*b)->n);
            }
        }
    }
    abort ();
}
#endif

STATICFN int dfwmalloc_get_alignment_adjustment (char *p, int alignment)
{
    ULONG mask;
    mask = alignment - 1;
    return ALIGN_ADJUST ((ULONG) p, mask, HEAD_SIZE);
}

#ifdef FOR_VALGRIND
void memset_uninitialized (void *u, long len)
{
    char *p;
    p = (char *) alloca (1);
    memset (u, *p, len);
}
#endif



#define MMAP_PAGE_SIZE_SMALL            (4096)

#if !defined(NO_CB_HEAPS) || defined(USE_TLS)

#define MMAP_PAGE_SIZE_BIG              (32 * 4096)

STATICFNNOINLINE void perror_abort (const char *msg)
{
    char s[256] = "";
    strlimcat (s, sizeof (s), msg);
    strlimcat (s, sizeof (s), ": errno=");
    strlimcat (s, sizeof (s), itoa_base ((ULONG64) errno, 1, 10));
    DFWMALLOC_LOG (s, strlen (s));
    abort ();
}

STATICFN int dfwmalloc_mmap_heap_init (struct heap *h, LONG chunkincr, ULONG64 the_pointer)
{
    const int prot = PROT_READ | PROT_WRITE;
    const int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED;
    void *r;

    dfwmalloc_assert (!h->end_of_heap && !h->start_of_heap);

    if (chunkincr < MMAP_PAGE_SIZE_SMALL)
        chunkincr = MMAP_PAGE_SIZE_SMALL;

    h->mmap_maplength = chunkincr;
    h->nonzero_heap = h->end_of_heap = h->start_of_heap = (char *) the_pointer;
    h->mmap_chunkincr = chunkincr;

    dfwmalloc_assert (!(h->mmap_maplength & 0xF));
    r = mmap (h->start_of_heap, h->mmap_maplength, prot, flags, -1, 0);
    if (r != h->start_of_heap) {
        if ((r == MAP_FAILED || r == (void *) -1 || !r) && errno == ENOMEM) {
            return 1;
        } else {
            perror_abort ("mmap init failed");
        }
    }
    return 0;
}

STATICFN void dfwmalloc_mmap_heap_free (struct heap *h)
{
    int r;
    dfwmalloc_assert (h->end_of_heap && h->start_of_heap);
    dfwmalloc_assert (h->mmap_maplength > 0);
    dfwmalloc_assert (!(h->mmap_maplength & 0xF));
    r = munmap (h->start_of_heap, h->mmap_maplength);
    if (r) {
        perror_abort ("munmap failed");
    }
    memset (h, '\0', sizeof (*h));
}

STATICFN void *dfwmalloc_mmap_heap_rawalloc_locked (struct heap *h, LONG n, int alignment, ULONG max_heap, int *need_zero)
{
    const int prot = PROT_READ | PROT_WRITE;
    const int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED;
    void *r, *res, *old;

#ifdef MAP_UNINITIALIZED
    flags |= MAP_UNINITIALIZED;
#endif

    dfwmalloc_assert (n > 0);
    dfwmalloc_assert (h->end_of_heap && h->start_of_heap);

    old = (void *) h->end_of_heap;
    h->end_of_heap += dfwmalloc_get_alignment_adjustment (h->end_of_heap, alignment);
    res = (void *) h->end_of_heap;

    if (h->nonzero_heap <= h->end_of_heap)
        if (need_zero)
            *need_zero = 0;
    h->end_of_heap += n;

    if ((ULONG) (h->end_of_heap - h->start_of_heap) > max_heap) {
        h->end_of_heap = (char *) old;               /* revert */
        errno = ENOMEM;
        return NULL;
    }

    if ((LONG) (h->end_of_heap - h->start_of_heap) > h->mmap_maplength) {
        int factor = 0;
        LONG mul;
        LONG old_length = h->mmap_maplength;
        h->mmap_maplength = (LONG) (h->end_of_heap - h->start_of_heap);
        mul = h->mmap_maplength / h->mmap_chunkincr;
/* Acceleration is appropriate for programs that allocate a lot of memory fast: */
        factor = (mul > 8) ? ((64 - __builtin_clzl (mul - 8)) / 2) : 0;
        h->mmap_maplength += ALIGN_ADJUST (h->mmap_maplength, (h->mmap_chunkincr << factor) - 1, 0);       /* align on page boundary */
        dfwmalloc_assert (!(h->mmap_maplength & 0xF));
        r = mmap (h->start_of_heap + old_length, h->mmap_maplength - old_length, prot, flags, -1, 0);
        if (r != h->start_of_heap + old_length) {
            if ((r == MAP_FAILED || r == (void *) -1 || !r) && errno == ENOMEM) {
                h->end_of_heap = (char *) old;               /* revert */
                h->mmap_maplength = old_length;     /* revert */
                return NULL;
            } else {
                perror_abort ("mmap failed");
            }
        }
    }
#ifdef FOR_VALGRIND
    memset_uninitialized (old, (long) (h->end_of_heap - (char *) old));
#endif

    if (h->nonzero_heap < h->end_of_heap)
        h->nonzero_heap = h->end_of_heap;

    return res;
}

STATICFN void dfwmalloc_mmap_heap_rawalloc_shrink_heap_locked (struct heap *h, LONG n, unsigned int *flush)
{
#ifndef NO_HEAP_TRUNCATION
    ULONG64 now;
#endif
    LONG old_length;
    int r;
    (void) flush;
    dfwmalloc_assert (n >= 0);
    dfwmalloc_assert (h->end_of_heap && h->start_of_heap);
    dfwmalloc_assert ((LONG) (h->end_of_heap - h->start_of_heap) >= n);
    old_length = h->mmap_maplength;
    h->end_of_heap -= n;

#ifndef NO_HEAP_TRUNCATION

/* Do this 1<<3 times per second unless malloc_trim() is called: */
    now = (ULONG64) rdtsc () >> (ROUGH_CPU_SPEED_MHZ + 20 /* million */ - 3 /* 8 */);

    if ((flush && (h->flush & HEAP_FLAG_FLUSH) != *flush) || h->last_timer != now) {
        if (flush) {
            h->flush &= ~HEAP_FLAG_FLUSH;
            h->flush |= (*flush & HEAP_FLAG_FLUSH);
        }
        h->last_timer = now;
    } else {
        return;
    }

#endif

    h->mmap_maplength = (LONG) (h->end_of_heap - h->start_of_heap);
    h->mmap_maplength += ALIGN_ADJUST (h->mmap_maplength, h->mmap_chunkincr - 1, 0);        /* align on page boundary */
    if (h->mmap_maplength < h->mmap_chunkincr)
        h->mmap_maplength = h->mmap_chunkincr;
    dfwmalloc_assert (h->mmap_maplength <= old_length);
    if (h->mmap_maplength != old_length) {
        dfwmalloc_assert (!(h->mmap_maplength & 0xF));
        r = munmap (h->start_of_heap + h->mmap_maplength, old_length - h->mmap_maplength);
        if (r) {
            perror_abort ("munmap shrink failed");
        } else {
            h->nonzero_heap = h->start_of_heap + h->mmap_maplength;
        }
    }
}


#ifndef NO_CB_HEAPS

#define CB_HEAP_GET_INDEX(u)            (((ULONG64) (u) >> (47 - N_ARENAS_BITS - CB_SIZE_BITS)) & CB_SIZE_MASK)
#ifdef USE_TLS
#define CB_HEAP_CHECK(u)                (CB_HEAP_GET_INDEX(u) < CB_SIZE_MAX)
#else
#define CB_HEAP_CHECK(u)                (!!((ULONG64) (u) & (ULONG64) 0x400000000000UL))
#endif
#define CB_INDEX_VRFY(i)                ((i) < CB_SIZE_MAX)

#ifdef USE_TLS
static const unsigned char empty_cb_list[8] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
#endif

#ifndef NO_BLOCK_ORDERING
#define PACKING_STYLE_HEAP_OR_LIST      0
#define PACKING_STYLE_SHIFT             1
#define PACKING_STYLE_MUL               2
#endif

#ifndef USE_PRECOMPILED_STATIC_DATA

STATICFN int calc_alignment (int n)
{
    int align = 1;
/* It seems that both Gtk and KDE have font libraries that expect very strict alignment.
 * Otherwise they segfault. */
    if (n >= (1 << WANTED_ALIGNMENT)) {
        align = (1 << WANTED_ALIGNMENT);
    } else {
        int i;
        for (i = WANTED_ALIGNMENT - 1; i >= 0; i--) {
            if ((n & (1 << i))) {
                align = (1 << i);
                break;
            }
        }
    }
    return align;
}

/* This function tries to fill up a control block accounting for the
 * fact that the header gets larger with each block due per-block
 * housekeeping info: */
STATICFN int dfwmalloc_iterate_malloc_per_cb (const int is_mask, const int align, const int n, const int ideal)
{
    int delta, cnt;
    cnt = ideal / 2;
    delta = cnt / 2;
    dfwmalloc_always_assert (delta > 0);
    for (;;) {
        LONG64 s;
        s = CB_SIZE (is_mask, n, align, cnt);
        dfwmalloc_assert (s >= 0);
        if (s > ideal) {
            cnt -= delta;
        } else if (s <= ideal && delta == 1) {
            break;
        } else {
            cnt += delta;
            delta >>= 1;
        }
    }
    dfwmalloc_always_assert (cnt > 0);
    return cnt;
}

#ifndef NO_BLOCK_ORDERING
STATICFN int dfwmalloc_is_near_power_of_two (int n_)
{
    int n;
    /* check if n_ is near a power of two: */
    for (n = 1;; n <<= 1) {
#ifdef USE_TLS
        if (n < SIZEOF_CB_SIZE)
            continue;
#endif
#ifdef COMPACT_MODE
        if (n > CB_N_SIZES)
            break;
#else
        if (n > 1024)
            break;
#endif
#ifndef COMPACT_MODE
        if (!(n % 64))          /* things that are multiples of the cache line size do not need to be ordered */
            continue;
#endif
        if (n_ == n)
            return n;
        if (n_ + 1 == n)
            return n;
        if (n_ < n && (n_ + ALIGN_ADJUST (n_, (1 << WANTED_ALIGNMENT) - 1, 0) >= n || n_ * ALLOCATION_SIZE_RATIO / 1000 >= n))
            return n;
    }
    return 0;
}
#endif

/* find the nearest heap size for these mallocs: */
STATICFN int dfwmalloc_calc_ideal_cb_size (int is_mask, int n, int align, int *heap_index_, int unordered)
{
    int n_blocks, i, ideal, chunk = 0;
    n_blocks = BLOCKS_FORMULA;
    ideal = CB_SIZE (is_mask, n, align, n_blocks);
    for (i = 0;; i++) {
        int next_size;
        chunk = (int) HEAP_INDEX_TO_CHUNKSIZE (i);
        next_size = (int) HEAP_INDEX_TO_CHUNKSIZE (i + 1);
        if (unordered) {
            if (chunk >= ideal || next_size > ideal || chunk >= MAX_CB_INT_T + 1 || next_size > MAX_CB_INT_T + 1)
                break;
        } else {
            if (chunk >= ideal || next_size > ideal)
                break;
        }
        if (i == CB_SIZE_MAX - 1)
            break;
    }
    if (heap_index_)
        *heap_index_ = i;
    return chunk;
}

STATICFN void dfwmalloc_calc_pages (int n_, struct cb_page_info *r)
{
    int cnt, ideal, align, n, heap_index, t;
#ifndef NO_BLOCK_ORDERING
    int is_mask = PACKING_STYLE_HEAP_OR_LIST;
#else
#define is_mask         0
#endif

#ifndef NO_BLOCK_ORDERING
    n = dfwmalloc_is_near_power_of_two (n_);
    if (n) {
        memset (r, '\0', sizeof (*r));
        is_mask = PACKING_STYLE_SHIFT;
        n_ = n;
        align = 1;
        if (n >= (1 << WANTED_ALIGNMENT)) {
            align = (1 << WANTED_ALIGNMENT);
        } else {
            align = n;
        }
    } else {
#endif
        t = r->round_up * ALLOCATION_SIZE_RATIO / 1000; /* Note (*10*). See documentation. */
        if (r->round_up && n_ < t)
            n_ = t;
        if (n_ > CB_N_SIZES)
            n_ = CB_N_SIZES;
#ifdef USE_TLS
        if (n_ < SIZEOF_CB_SIZE)
            n_ = SIZEOF_CB_SIZE;
#endif

        memset (r, '\0', sizeof (*r));

        n = n_;
        align = calc_alignment (n);

        n += ALIGN_ADJUST (n, align - 1, 0);

#ifndef NO_BLOCK_ORDERING
#ifdef COMPACT_MODE
        is_mask = PACKING_STYLE_MUL;
#else
        if (dfwmalloc_calc_ideal_cb_size (0, n, align, NULL, 0) > MAX_CB_INT_T + 1) {
            is_mask = PACKING_STYLE_MUL;        /* can't reach across the control block with an unsigned short, so change packing method */
        } else if (!(n % 64)) {
            /* things that are multiples of the cache line size do not need to be ordered */
        } else if (n <= 255) {      /* sizeof(sz_shift_mul) == 1 */
            is_mask = PACKING_STYLE_MUL;
        }
#endif
    }
#endif

    for (;;) {
        n_ = n;

#ifndef NO_BLOCK_ORDERING
        if (is_mask != PACKING_STYLE_SHIFT) {
#endif
            align = calc_alignment (n);
            if (align < (1 << MIN_ALIGN))
                align = (1 << MIN_ALIGN);
#ifndef NO_BLOCK_ORDERING
        }
#endif
#ifdef USE_TLS
        if (n < SIZEOF_CB_SIZE)
            n = SIZEOF_CB_SIZE;
#endif
        n += ALIGN_ADJUST (n, align - 1, 0);
        dfwmalloc_always_assert (n <= CB_N_SIZES);
        ideal = dfwmalloc_calc_ideal_cb_size (is_mask, n, align, &heap_index, !is_mask);

        for (;;) {
            cnt = dfwmalloc_iterate_malloc_per_cb (is_mask, align, n, ideal);
/* The maximum number of masks is 255 due n_masks being stored in an unsigned char,
 * hence the maximum number of mask bits is 64 * 255: */
            if (cnt < 64 * 255)
                break;
            heap_index--;
            ideal = HEAP_INDEX_TO_CHUNKSIZE (heap_index);
        }

/* round up n until we have filled the control block: */
        while (n < CB_N_SIZES && CB_SIZE (is_mask, n + align, align, cnt) <= (unsigned int) HEAP_INDEX_TO_CHUNKSIZE (heap_index))
            n += align;

/* we keep trying until things stabilize to the same number: */
        if (n == n_)
            break;

#ifndef NO_BLOCK_ORDERING
        if (is_mask == PACKING_STYLE_SHIFT)     /* we have lost our power of two here */
            is_mask = PACKING_STYLE_MUL;
#endif
    }

    /* save everything: */
    r->n_blocks = cnt;
    r->heap_index = heap_index;
    r->mask_packing = is_mask;
    r->first_block = CB_FIRST_OFFSET (is_mask, r->n_blocks, align);
    r->round_up = n;

    dfwmalloc_always_assert (CB_INDEX_VRFY (heap_index));
    dfwmalloc_always_assert (n < 65536);
    dfwmalloc_always_assert (cnt < 65536);

/* printf_simple ("n=%d waist=%d\n", (int) n, (int) (HEAP_INDEX_TO_CHUNKSIZE (r->heap_index) - CB_SIZE (is_mask, n, align, r->n_blocks))); */

    dfwmalloc_always_assert (CB_SIZE (is_mask, n, align, r->n_blocks) <= (unsigned int) HEAP_INDEX_TO_CHUNKSIZE (r->heap_index));
}
#endif

STATICFN void dfwmalloc_cb_init_page_info (struct cb_heaps_info *cb_heaps_info)
{
#ifdef USE_PRECOMPILED_STATIC_DATA
    cb_heaps_info->page_info = global_page_info;
    cb_heaps_info->head_index = global_head_index;
    cb_heaps_info->max_head = global_max_heads;
#else
    int n, q, max_head = 0;
    struct cb_page_info info;

/* The regular heap has to be distinguishable from the CB heap: */
    dfwmalloc_always_assert (CB_SIZE_MAX < (1 << CB_SIZE_BITS));

    cb_heaps_info->head_index[0] = 0;
    memset (&cb_heaps_info->page_info[0], 0, sizeof (cb_heaps_info->page_info[0]));
    memset (&info, '\0', sizeof (info));

    for (n = 1, q = 1; n <= CB_N_SIZES; n++, q++) {
        dfwmalloc_calc_pages (n, &info);
        if (q >= CB_N_HEADS)
            dfwmalloc_writestr ("Internal parameters exceeded. Increase ESTIMATED_MIN_BLOCKS or ALLOCATION_SIZE_RATIO and try again.\n");
        dfwmalloc_always_assert (q < CB_N_HEADS);
        for (; n <= CB_N_SIZES; n++) {
            cb_heaps_info->head_index[n] = q;
            if (n == info.round_up)
                break;
        }
        cb_heaps_info->page_info[q] = info;
        if (max_head < q)
            max_head = q;
    }
    cb_heaps_info->max_head = max_head;
#endif
    dfwmalloc_always_assert (cb_heaps_info->page_info[cb_heaps_info->max_head].round_up == CB_N_SIZES);

#if 0
    for (n = 1; n <= CB_N_SIZES; n++) {
        struct cb_page_info info;
        int head_index;
        head_index = cb_heaps_info->head_index[n];
        info = cb_heaps_info->page_info[head_index];
        if (info.round_up == n)
            printf_simple ("%d: bytes_per_block=%d round_up=%d n_blocks=%d index=%d\n", n, (int) HEAP_INDEX_TO_CHUNKSIZE (info.heap_index),
                           (int) info.round_up, (int) info.n_blocks, (int) info.heap_index);
        else
            printf_simple ("%d: round_up=%d\n", n, (int) info.round_up);
    }
    abort ();
#endif
}

STATICFN void dfwmalloc_cb_init (struct cb_heaps *cbh)
{
#ifdef USE_TLS
    dfwmalloc_assert (SIZEOF_CB_SIZE == sizeof (cb_size_t));
#endif
    (void) cbh;
#ifndef USE_TLS
    dfwmalloc_mutex_init (&cbh->mutex, NULL);
#endif
}

STATICFN void dfwmalloc_cb_destroy_heap (struct cb_heaps *cbhs)
{
    int heap_index;
    for (heap_index = 0; heap_index < (int) CB_SIZE_MAX; heap_index++) {
        struct control_block_heap *cbh;
        cbh = &cbhs->heaps[heap_index];
        if (!cbh->heap.start_of_heap)
            continue;
        dfwmalloc_mmap_heap_free (&cbh->heap);
    }
}

STATICFN int dfwmalloc_cb_init_heap (struct control_block_heap *cbh, int arena_id, int heap_index)
{
    ULONG64 u;
    (void) arena_id;
    dfwmalloc_assert (CB_INDEX_VRFY (heap_index));
    POINTER_FROM_ARENA (u, arena_id, heap_index);
    dfwmalloc_assert (ARENA_FROM_POINTER (u) == arena_id);
    if (dfwmalloc_mmap_heap_init (&cbh->heap, HEAP_INDEX_TO_CHUNKSIZE (heap_index), u)) {
        memset (cbh, '\0', sizeof (*cbh));
        return 1;
    }
#ifdef USE_TLS
    memcpy (&cbh->first_sharedfree_item, &empty_cb_list[0], sizeof (cb_size_t));
#endif
    return 0;
}

#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
STATICFN struct control_block *dfwmalloc_cb_rankheap_merge (struct control_block *a, struct control_block *b)
{
    if (a < b) {
        if (a->rh_child != NULL)
            a->rh_child->rh_prev = b;
        if (b->rh_next != NULL)
            b->rh_next->rh_prev = a;
        a->rh_next = b->rh_next;
        b->rh_next = a->rh_child;
        a->rh_child = b;
        b->rh_prev = a;
        return a;
    }
    if (b->rh_child != NULL)
        b->rh_child->rh_prev = a;
    if (a->rh_prev != NULL && a->rh_prev->rh_child != a)
        a->rh_prev->rh_next = b;
    b->rh_prev = a->rh_prev;
    a->rh_prev = b;
    a->rh_next = b->rh_child;
    b->rh_child = a;
    return b;
}

STATICFNINLINE void dfwmalloc_cb_rankheap_insert (struct rankheap *rh, struct control_block *cb)
{
    cb->rh_prev = NULL;
    cb->rh_next = NULL;
    cb->rh_child = NULL;

    if (rh->root == NULL)
        rh->root = cb;
    else
        rh->root = dfwmalloc_cb_rankheap_merge (cb, rh->root);
}

STATICFNINLINE struct control_block *dfwmalloc_cb_rankheap_merge_right (struct control_block *a)
{
    struct control_block *b;
    for (b = NULL; a != NULL; a = b->rh_next) {
        if ((b = a->rh_next) == NULL)
            return a;
        b = dfwmalloc_cb_rankheap_merge (a, b);
    }
    return b;
}

STATICFNINLINE struct control_block *dfwmalloc_cb_rankheap_merge_left (struct control_block *a)
{
    struct control_block *b;
    for (b = a->rh_prev; b != NULL; b = a->rh_prev)
        a = dfwmalloc_cb_rankheap_merge (b, a);
    return a;
}

STATICFNINLINE struct control_block *dfwmalloc_cb_rankheap_merge_subheaps (struct control_block *a)
{
    struct control_block *e;

    a->rh_child->rh_prev = NULL;

    e = dfwmalloc_cb_rankheap_merge_right (a->rh_child);
    e = dfwmalloc_cb_rankheap_merge_left (e);

    return e;
}

STATICFNINLINE struct control_block *dfwmalloc_cb_rankheap_pop (struct rankheap *rh)
{
    struct control_block *root = rh->root;
    if (root->rh_child == NULL) {
        rh->root = NULL;
        return root;
    }

    rh->root = dfwmalloc_cb_rankheap_merge_subheaps (root);
    return root;
}

STATICFNINLINE void detach_subheap (struct control_block *cb)
{
    if (cb->rh_prev->rh_child == cb)
        cb->rh_prev->rh_child = cb->rh_next;
    else
        cb->rh_prev->rh_next = cb->rh_next;

    if (cb->rh_next != NULL)
        cb->rh_next->rh_prev = cb->rh_prev;

    cb->rh_prev = NULL;
    cb->rh_next = NULL;
}

STATICFNINLINE void dfwmalloc_cb_rankheap_remove (struct rankheap *rh, struct control_block *cb_)
{
    struct control_block *cb = cb_;

    if (cb == rh->root) {
        dfwmalloc_cb_rankheap_pop (rh);
        return;
    }

    detach_subheap (cb);
    if (cb->rh_child) {
        cb = dfwmalloc_cb_rankheap_merge_subheaps (cb);
        rh->root = dfwmalloc_cb_rankheap_merge (rh->root, cb);
    }

    cb_->rh_prev = NULL;
    cb_->rh_next = NULL;
    cb_->rh_child = NULL;
}

#endif

#ifndef NO_HEAP_TRUNCATION

#define UNLINK_CB(l, p) \
    do { \
        if (((p)->cb_magic_flags & CB_FLAG_PARTIAL_LINKED)) { \
            (p)->cb_magic_flags &= ~CB_FLAG_PARTIAL_LINKED; \
            dfwmalloc_assert ((l)->partial_blocks_first); \
            if ((l)->partial_blocks_first == (p)) { \
                dfwmalloc_assert (!(p)->partial_blocks_prev); \
                (l)->partial_blocks_first = (p)->partial_blocks_next; \
                if ((l)->partial_blocks_first) \
                    (l)->partial_blocks_first->partial_blocks_prev = NULL; \
            } else { \
                if ((p)->partial_blocks_next) { \
                    struct control_block *partial_blocks_next; \
                    partial_blocks_next = (p)->partial_blocks_next; \
                    partial_blocks_next->partial_blocks_prev = (p)->partial_blocks_prev; \
                } \
                { \
                    struct control_block *partial_blocks_prev; \
                    partial_blocks_prev = (p)->partial_blocks_prev; \
                    partial_blocks_prev->partial_blocks_next = (p)->partial_blocks_next; \
                } \
            } \
            (p)->partial_blocks_prev = NULL; \
            (p)->partial_blocks_next = NULL; \
        } \
    } while (0)

#define LINK_CB(l, p) \
    do { \
        (p)->cb_magic_flags |= CB_FLAG_PARTIAL_LINKED; \
        if ((l)->partial_blocks_first) \
            (l)->partial_blocks_first->partial_blocks_prev = (p); \
        (p)->partial_blocks_next = (l)->partial_blocks_first; \
        (p)->partial_blocks_prev = NULL; \
        (l)->partial_blocks_first = (p); \
    } while (0)

#else

#define UNLINK_CB(l, p) \
    do { \
        if (((p)->cb_magic_flags & CB_FLAG_PARTIAL_LINKED)) { \
            (p)->cb_magic_flags &= ~CB_FLAG_PARTIAL_LINKED; \
            dfwmalloc_assert ((l)->partial_blocks_first); \
            dfwmalloc_assert ((l)->partial_blocks_first == (p)); \
            (l)->partial_blocks_first = (p)->partial_blocks_next; \
            (p)->partial_blocks_next = NULL; \
        } \
    } while (0)

#define LINK_CB(l, p) \
    do { \
        (p)->cb_magic_flags |= CB_FLAG_PARTIAL_LINKED; \
        (p)->partial_blocks_next = (l)->partial_blocks_first; \
        (l)->partial_blocks_first = (p); \
    } while (0)

#endif

#define CB_TAIL_P(cb,i)   ((cb_int_t *) ((char *) (cb) + HEAP_INDEX_TO_CHUNKSIZE(i) - sizeof(cb_int_t)))


#ifndef NO_HEAP_TRUNCATION
STATICFN struct control_block *dfwmalloc_cb_unlink_heap_tail (struct cb_heaps *cbhs, int heap_index)
{
    struct control_block *cb;
    struct control_block_heap *heap;
    unsigned long chunk_size;
    ULONG64 delta;
    struct control_block *found_cb;
    found_cb = NULL;
    heap = &cbhs->heaps[heap_index];
    chunk_size = HEAP_INDEX_TO_CHUNKSIZE (heap_index);
    delta = (heap->heap.end_of_heap - heap->heap.start_of_heap);
    while (delta >= chunk_size) {
        delta -= chunk_size;
        cb = (struct control_block *) &heap->heap.start_of_heap[delta];
#ifdef WITH_BTANDCHECK
        dfwmalloc_assert (CB_MAGIC_CHK (cb));
        dfwmalloc_assert (CB_TAIL_P(cb, heap_index)[0] == CB_TAIL_MAGIC);
#endif
        if (cb->used_count != cb->free_count)
            break;
/* every block is free'd, so we can clobber this control block */
        if (cb->head_index) {
            UNLINK_CB (&cbhs->cb_heads[cb->head_index], cb);
            cb->head_index = 0;
        }
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
        else {
            dfwmalloc_cb_rankheap_remove (&heap->rankheap, cb);
        }
#endif
        found_cb = cb;
    }
    return found_cb;
}

STATICFNINLINE void dfwmalloc_cb_shrink_heap (struct heap *heap, struct cb_heaps *cbhs, char *p)
{
    if (p) {
        LONG delta;
        delta = (LONG) (heap->end_of_heap - p);
        dfwmalloc_mmap_heap_rawalloc_shrink_heap_locked (heap, delta, &cbhs->flush);
        cbhs->total_heap -= delta;
    } else {
        dfwmalloc_mmap_heap_rawalloc_shrink_heap_locked (heap, 0, &cbhs->flush);
    }
}

STATICFN void dfwmalloc_cb_truncate (struct cb_heaps *cbhs)
{
    int heap_index;
    cbhs->flush ^= CB_FLAG_FLUSH;
    for (heap_index = 0; heap_index < (int) CB_SIZE_MAX; heap_index++) {
        struct control_block_heap *heap;
        heap = &cbhs->heaps[heap_index];
        if (!heap->heap.start_of_heap)
            continue;
        dfwmalloc_cb_shrink_heap (&heap->heap, cbhs, (char *) dfwmalloc_cb_unlink_heap_tail (cbhs, heap_index));
    }
}

#endif

STATICFN LONG dfwmalloc_cb_efficiency_ (struct cb_heaps *cbhs, int heap_index, int arena)
{
    LONG total;
    struct control_block *cb;
    struct control_block_heap *heap;
    unsigned long chunk_size;
    heap = &cbhs->heaps[heap_index];
    chunk_size = HEAP_INDEX_TO_CHUNKSIZE (heap_index);
    total = 0L;
    for (cb = (struct control_block *) heap->heap.start_of_heap; cb < (struct control_block *) heap->heap.end_of_heap; cb = (struct control_block *) ((char *) cb + chunk_size)) {
        struct cb_page_info info;
        char s[256], *p;
        total += chunk_size;
#ifdef WITH_BTANDCHECK
        dfwmalloc_assert (CB_MAGIC_CHK (cb));
        dfwmalloc_assert (CB_TAIL_P(cb, heap_index)[0] == CB_TAIL_MAGIC);
#endif

        p = s;

#define FLD(u,b) \
    strcpy (p, itoa_base ((ULONG64) (u), 0, b)); \
    p += strlen (p); \
    *p++ = ' ';

        FLD (arena, 10);
        FLD (heap_index, 10);
        FLD (cb, 16);
        FLD (chunk_size, 10);

        if (cb->used_count == cb->free_count) {
            FLD (0, 10);
            FLD (0, 10);
            FLD (0, 10);
        } else {
            info = global->cb_heaps_info.page_info[cb->head_index];
            FLD (info.round_up, 10);
            FLD (cb->used_count - cb->free_count, 10);
            FLD (cb->total_count, 10);
        }
        DFWMALLOC_LOG (s, (int) (p - s));
    }
    return total;
}

STATICFN LONG dfwmalloc_cb_efficiency (struct cb_heaps *cbhs, int arena)
{
    LONG total;
    int heap_index;
    total = 0L;
    for (heap_index = 0; heap_index < (int) CB_SIZE_MAX; heap_index++) {
        struct control_block_heap *heap;
        heap = &cbhs->heaps[heap_index];
        if (!heap->heap.start_of_heap)
            continue;
        total += dfwmalloc_cb_efficiency_ (cbhs, heap_index, arena);
    }
    return total;
}

#ifdef WITH_UNITTESTS

/* used to force a condition -- for testing only */
void __dfwmalloc_dfwforce_total_heap (struct dfwmalloc *dfw, long n)
{
    dfw->cb_heaps.total_heap += n;
}

#endif

STATICFNNOINLINE struct control_block *dfwmalloc_cb_add (struct cb_heaps *cbhs, int arena_id, malloc_size_t n)
{
    struct control_block *cb;
    struct control_block_head *cbh;
    struct control_block_heap *heap;
#ifndef NO_HEAP_TRUNCATION
    struct control_block *found_cb;
#endif
    unsigned long chunk_size;
    unsigned int heap_index;
    int head_index;
    const struct cb_page_info *page_info;

    dfwmalloc_assert (n <= CB_N_SIZES && n > 0);
    head_index = global->cb_heaps_info.head_index[n];
    page_info = &global->cb_heaps_info.page_info[head_index];
    heap_index = page_info->heap_index;
    heap = &cbhs->heaps[heap_index];

    if (!heap->heap.start_of_heap) {
        int r = 0;
        dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
        if (!heap->heap.start_of_heap)
            r = dfwmalloc_cb_init_heap (heap, arena_id, heap_index);
        dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);
        if (r)
            return NULL;
        dfwmalloc_assert (heap_index == CB_HEAP_GET_INDEX (heap->heap.start_of_heap));
    }

    chunk_size = HEAP_INDEX_TO_CHUNKSIZE (heap_index);

#ifndef NO_HEAP_TRUNCATION
    found_cb = dfwmalloc_cb_unlink_heap_tail (cbhs, heap_index);
#endif

#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
/* we want to try fill memory closest to the start so that heap truncation can take effect */
    if (heap->rankheap.root) {
/* case 1: there is an empty block somewhere in the middle of the heap, so find the one closest to the start */
        cb = dfwmalloc_cb_rankheap_pop (&heap->rankheap);
        dfwmalloc_assert (cb != NULL);
        dfwmalloc_assert (cb->head_index == 0);
        dfwmalloc_assert (cb != found_cb);
        dfwmalloc_assert (cb->free_count == cb->used_count);
        if (found_cb) {
            dfwmalloc_assert (found_cb->free_count == found_cb->used_count);
            dfwmalloc_assert (cb < found_cb);
            dfwmalloc_assert ((char *) found_cb < heap->heap.end_of_heap);
            dfwmalloc_cb_shrink_heap (&heap->heap, cbhs, (char *) found_cb);
        }
    } else
#endif
#ifndef NO_HEAP_TRUNCATION
    if (found_cb) {
/* case 2: the heap has a tail of empty blocks */
        cb = found_cb;
        dfwmalloc_assert (cb->free_count == cb->used_count);
        if ((char *) found_cb + chunk_size < heap->heap.end_of_heap)
            dfwmalloc_cb_shrink_heap (&heap->heap, cbhs, (char *) found_cb + chunk_size);
    } else
#endif
    {
/* case 3: no empty blocks */
        cb = (struct control_block *) dfwmalloc_mmap_heap_rawalloc_locked (&heap->heap, chunk_size, 1, global->max_cb_heap, NULL);
        if (!cb)
            goto out;
        cbhs->total_heap += chunk_size;
    }

#ifdef WITH_BTANDCHECK
    if (page_info->mask_packing) {
        /* TBD */
    } else {
        ULONG64 m;
        m = (((ULONG64) 1UL << (47 - N_ARENAS_BITS - CB_SIZE_BITS)) - 1);
        dfwmalloc_assert (heap_index == CB_HEAP_GET_INDEX (cb));
        dfwmalloc_assert (cb == ((struct control_block *) (void *) ((ULONG64) &cb->cb_free_list[page_info->n_blocks - 1] - (((ULONG64) &cb->cb_free_list[page_info->n_blocks - 1]) & m) % chunk_size)));
    }
#endif

    cb->cb_magic_flags = CB_MAGIC;
#ifndef NO_HEAP_TRUNCATION
    cb->cb_magic_flags |= (cbhs->flush & CB_FLAG_FLUSH);
#endif

#ifndef NO_BLOCK_ORDERING
    if (page_info->mask_packing) {
        int i;
        if (page_info->mask_packing == PACKING_STYLE_MUL) {
            cb->cb_magic_flags |= CB_FLAG_PACKING_M;
            cb->sz_shift_mul =  page_info->round_up;
        } else {
            cb->cb_magic_flags |= CB_FLAG_PACKING_S;
            cb->sz_shift_mul =  63 - __builtin_clzl ((ULONG64) page_info->round_up);
        }
        cb->n_masks = (page_info->n_blocks + 63) >> 6;
        cb->cb_first_block = page_info->first_block;
        for (i = 0; i < cb->n_masks; i++)
            cb->cb_bit_mask[i] = 0UL;
#if 0   /* See note (*9*) */
/* There seems no point in ordering allocs if they span more than two
 * cache lines because their adjacent-ness is not an advantage. Also, tiny
 * programs probably like their speed: */
    } else if (n < 256 /* two cache lines */ && cbhs->total_heap > 1024 * 1024 /* not a tiny process (*7*) */) {
        cb->cb_magic_flags |= CB_FLAG_PACKING_H;
#endif
    } else {
        cb->cb_magic_flags |= CB_FLAG_PACKING_L;
    }
#endif

#define GET_CREATE_TIME ((ULONG64) rdtsc () >> (ROUGH_CPU_SPEED_MHZ + 20 /* million */ - 10 /* 256 */))

#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
    cb->create_time = GET_CREATE_TIME;
#endif
    cb->head_index = head_index;
#ifdef SECURITY_RANDOMIZATION
    /* ... */
#else
    cb->free_count = 0;
    cb->used_count = 0;
#endif
    cb->total_count = page_info->n_blocks;
    CB_TAIL_P(cb, heap_index)[0] = CB_TAIL_MAGIC;

    cbh = &cbhs->cb_heads[head_index];
    LINK_CB (cbh, cb);

  out:
    return cb;
}

/* @INLINE2START@ */

/* inlined version of malloc/free */
struct malloc_calc {
    struct control_block_head *cbh;
    struct cb_heaps *cbhs;
    int len;
    unsigned long long mask;
    const struct cb_page_info *page_info;
};

/* @INLINE2END@ */

int dfwmalloc_prepare (struct malloc_calc *precalc, size_t n)
{
    struct calltrace c;
    struct dfwmalloc *dfw;
    int head_index;
    int heap_index;

    if (!n)
        return 1;

    INIT_NOSHRDFREE (dfw, c, return 1);

    if (n > CB_N_SIZES)
        return 1;

    head_index = global_.cb_heaps_info.head_index[n];   /* don't use global-> since here every clock tick counts */
    heap_index = global->cb_heaps_info.page_info[head_index].heap_index;
#ifdef WITH_BTANDCHECK
    dfwmalloc_assert (global->cb_heaps_info.page_info[head_index].round_up >= n);
#endif
    precalc->cbhs = &dfw->cb_heaps;
    precalc->cbh = &precalc->cbhs->cb_heads[head_index];
    precalc->len = (int) n;
    precalc->mask = ~(((ULONG64) HEAP_INDEX_TO_CHUNKSIZE (heap_index)) - 1);
    precalc->page_info = &global->cb_heaps_info.page_info[head_index];

    return 0;
}


/* @INLINE5START@ */

#define HI_BIT        ((ULONG64) 1UL << 63)

/* Vector heap implementation for free list within a control block so as to
 * efficiently extract the lowest numbered block: */
STATICFNINLINE void dfwmalloc_cb_insert_free_block (struct control_block *cb, void *u)
{
#ifndef NO_BLOCK_ORDERING

#define HSWAP(a,b) \
    do { \
        cb_int_t __t; \
        __t = (a); \
        (a) = (b); \
        (b) = __t; \
    } while (0)

    switch ((cb->cb_magic_flags & CB_FLAG_PACKING)) {
    case CB_FLAG_PACKING_H:  /* See note (*9*) */
        {
            cb_int_t parent, i;
            cb_int_t *p;

            p = cb->cb_free_list;
            i = cb->free_count;
            p[cb->free_count++] = ((char *) u - (char *) cb) >> MIN_ALIGN;
            while (i != 0) {
                parent = (i - 1) / 2;
                if (p[parent] <= p[i]) 
                    break;
                HSWAP (p[parent], p[i]);
                i = parent;
            }
        }
        break;
    case CB_FLAG_PACKING_L:
        cb->cb_free_list[cb->free_count++] = ((char *) u - (char *) cb) >> MIN_ALIGN;
        break;
    case CB_FLAG_PACKING_S:
        {
            LONG b;
            cb->free_count++;
            b = ((char *) u - (char *) cb - cb->cb_first_block);
            b >>= cb->sz_shift_mul;
            dfwmalloc_assert ((b >> 6) < cb->n_masks);
#ifndef HEAP_TEST
            dfwmalloc_assert ((char *) &((struct control_block *) NULL)->cb_bit_mask[cb->n_masks] <= (char *) (void *) (uintptr_t) cb->cb_first_block);
#endif
            dfwmalloc_assert (!(cb->cb_bit_mask[b >> 6] & (HI_BIT >> (b & 63))));
            cb->cb_bit_mask[b >> 6] |= (HI_BIT >> (b & 63));
        }
        break;
    case CB_FLAG_PACKING_M:
        {
            LONG b;
            cb->free_count++;
            b = ((char *) u - (char *) cb - cb->cb_first_block);
            b /= cb->sz_shift_mul;
            dfwmalloc_assert ((b >> 6) < cb->n_masks);
#ifndef HEAP_TEST
            dfwmalloc_assert ((char *) &((struct control_block *) NULL)->cb_bit_mask[cb->n_masks] <= (char *) (void *) (uintptr_t) cb->cb_first_block);
#endif
            dfwmalloc_assert (!(cb->cb_bit_mask[b >> 6] & (HI_BIT >> (b & 63))));
            cb->cb_bit_mask[b >> 6] |= (HI_BIT >> (b & 63));
        }
        break;
    }
#else
    cb->cb_free_list[cb->free_count++] = ((char *) u - (char *) cb) >> MIN_ALIGN;
#endif
}

STATICFNINLINE void *dfwmalloc_cb_get_min_block (struct control_block *cb)
{
#ifndef NO_BLOCK_ORDERING
    switch ((cb->cb_magic_flags & CB_FLAG_PACKING)) {
    case CB_FLAG_PACKING_H:  /* See note (*9*) */
        {
            cb_int_t i = 0, left, right, largest, min_item;
            cb_int_t *p;

            p = cb->cb_free_list;
            min_item = p[0];
            cb->free_count--;
            p[0] = p[cb->free_count];

#if __GNUC__ < 7
            __sync_synchronize ();       /* gcc 4.8.5 optimization bug? */
#endif

            for (;;) {
                left = 2 * i + 1;
                right = 2 * i + 2;
                largest = i;

                if (left <= cb->free_count && p[left] < p[largest])
                    largest = left;

                if (right <= cb->free_count && p[right] < p[largest])
                    largest = right;

                if (largest == i)
                    break;

                HSWAP (p[i], p[largest]);
                i = largest;
            }
            return ((char *) cb + ((unsigned long) min_item << MIN_ALIGN));
        }
    case CB_FLAG_PACKING_L:
        return ((char *) cb + ((unsigned long) cb->cb_free_list[--cb->free_count] << MIN_ALIGN));
    case CB_FLAG_PACKING_S:
        {
            int i = 0, n;
            ULONG64 *p;
            cb->free_count--;
            p = cb->cb_bit_mask;
            n = cb->n_masks;
            do {
                if (*p) {
                    LONG b;
                    b = __builtin_clzl (*p);
                    *p &= ~(HI_BIT >> b);
                    b += (i << 6);
                    return ((char *) cb + (b << cb->sz_shift_mul) + cb->cb_first_block);
                }
                p++;
            } while (++i < n);
        }
        break;
    case CB_FLAG_PACKING_M:
        {
            int i = 0, n;
            ULONG64 *p;
            cb->free_count--;
            p = cb->cb_bit_mask;
            n = cb->n_masks;
            do {
                if (*p) {
                    LONG b;
                    b = __builtin_clzl (*p);
                    *p &= ~(HI_BIT >> b);
                    b += (i << 6);
                    return ((char *) cb + (b * cb->sz_shift_mul) + cb->cb_first_block);
                }
                p++;
            } while (++i < n);
        }
    }
    return NULL;
#else
    return ((char *) cb + ((unsigned long) cb->cb_free_list[--cb->free_count] << MIN_ALIGN));
#endif
}

/* @INLINE5END@ */


STATICFNINLINE void *dfwmalloc_cb_malloc (struct cb_heaps *cbhs, int arena_id, malloc_size_t n)
{
    struct control_block *cb;
    struct control_block_head *cbh;
    int head_index;

#ifdef WITH_BTANDCHECK
    dfwmalloc_assert (n <= CB_N_SIZES);
#endif
    head_index = global_.cb_heaps_info.head_index[n];   /* don't use global-> since here every clock tick counts */
#ifdef WITH_BTANDCHECK
    dfwmalloc_assert (global->cb_heaps_info.page_info[head_index].round_up >= n);
#endif
    cbh = &cbhs->cb_heads[head_index];

    for (;;) {
        cb = cbh->partial_blocks_first;
#ifdef WITH_BTANDCHECK
        dfwmalloc_assert (!cb || CB_MAGIC_CHK (cb));
        dfwmalloc_assert (!cb || CB_TAIL_P (cb, global->cb_heaps_info.page_info[head_index].heap_index)[0] == CB_TAIL_MAGIC);
#endif
        if (__builtin_expect (cb && cb->free_count, 1)) {
            if (__builtin_expect (cb->free_count == cb->used_count, 0)) {
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
                cb->create_time = GET_CREATE_TIME;
#endif
                cbhs->cb_refs++;
            }
#ifdef WITH_ACCOUNTS
            cbhs->total_alloced += global->cb_heaps_info.page_info[head_index].round_up;
            cbhs->cb_total_blocks++;
            cbhs->total_free--;
#endif
/* the common case ==> */
            return dfwmalloc_cb_get_min_block (cb);
        } else {
            const struct cb_page_info *page_info;
            page_info = &global->cb_heaps_info.page_info[head_index];
            if (!cb) {
                if (!(cb = dfwmalloc_cb_add (cbhs, arena_id, n)))
                    break;
                cb->used_count = 1;
                cbhs->cb_refs++;
#ifdef WITH_ACCOUNTS
                cbhs->total_alloced += global->cb_heaps_info.page_info[head_index].round_up;
                cbhs->cb_total_blocks++;
#endif
                return CB_POINTER (cb, page_info, 0);
            } else if (cb->used_count < cb->total_count) {
                void *u;
                u = CB_POINTER (cb, page_info, cb->used_count);
                if (cb->free_count == cb->used_count) {
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
                    cb->create_time = GET_CREATE_TIME;
#endif
                    cbhs->cb_refs++;
                }
                cb->used_count++;
#ifdef WITH_ACCOUNTS
                cbhs->total_alloced += global->cb_heaps_info.page_info[head_index].round_up;
                cbhs->cb_total_blocks++;
#endif
                return u;
            }

            UNLINK_CB (cbh, cb);
            dfwmalloc_assert (!cbh->partial_blocks_first || CB_MAGIC_CHK (cbh->partial_blocks_first));
            dfwmalloc_assert (!cbh->partial_blocks_first || CB_TAIL_P (cbh->partial_blocks_first, page_info->heap_index)[0] == CB_TAIL_MAGIC);
        }
    }

    return NULL;
}

STATICFNINLINE int dfwmalloc_cb_get_size (void *u)
{
    struct control_block *cb;
    int heap_index;
    ULONG64 m;
    heap_index = CB_HEAP_GET_INDEX (u);
    m = (((ULONG64) 1UL << (47 - N_ARENAS_BITS - CB_SIZE_BITS)) - 1);
    cb = (struct control_block *) (void *) ((ULONG64) u - (((ULONG64) u) & m) % HEAP_INDEX_TO_CHUNKSIZE (heap_index));
#ifdef WITH_BTANDCHECK
    dfwmalloc_assert (CB_MAGIC_CHK (cb));
    dfwmalloc_assert (CB_TAIL_P(cb, heap_index)[0] == CB_TAIL_MAGIC);
#endif
    return global->cb_heaps_info.page_info[cb->head_index].round_up;
}

#ifndef NO_HEAP_TRUNCATION
STATICFNINLINE void dfwmalloc_cb_bubblesort (struct control_block_head *cbh, struct control_block *cb)
{
    struct control_block *n;
    n = cb->partial_blocks_next;
    if (n && n->used_count - n->free_count > cb->used_count - cb->free_count) {
        struct control_block *nn, *p;
        nn = n->partial_blocks_next;
        p = cb->partial_blocks_prev;
        if (p)
            p->partial_blocks_next = n;
        else
            cbh->partial_blocks_first = n;
        n->partial_blocks_next = cb;
        cb->partial_blocks_next = nn;
        if (nn)
            nn->partial_blocks_prev = cb;
        cb->partial_blocks_prev = n;
        n->partial_blocks_prev = p;
    }
}
#endif


STATICFNINLINE void dfwmalloc_cb_free (struct cb_heaps *cbhs, void *u)
{
    int deref = 0;
    struct control_block_head *cbh;
    struct control_block *cb;
    ULONG64 m;
    int heap_index;

    heap_index = CB_HEAP_GET_INDEX (u);
    m = ((ULONG64) HEAP_INDEX_TO_CHUNKSIZE (heap_index)) - 1;
    cb = (struct control_block *) (void *) ((ULONG64) u & ~m);
    cbh = &cbhs->cb_heads[cb->head_index];
    dfwmalloc_assert (CB_MAGIC_CHK (cb));         /* even without WITH_BTANDCHECK we should have a minimal check here */
#ifdef WITH_BTANDCHECK
    dfwmalloc_assert (CB_TAIL_P(cb, heap_index)[0] == CB_TAIL_MAGIC);
    dfwmalloc_assert (cb->head_index != 0);
    if (__builtin_expect (!(cb->cb_magic_flags & CB_FLAG_PARTIAL_LINKED), 0)) {
#ifndef NO_HEAP_TRUNCATION
        dfwmalloc_assert (cb->partial_blocks_prev == NULL);
#endif
        dfwmalloc_assert (cb->partial_blocks_next == NULL);
        dfwmalloc_assert (!cbh->partial_blocks_first || CB_MAGIC_CHK (cbh->partial_blocks_first));
        dfwmalloc_assert (!cbh->partial_blocks_first || CB_TAIL_P(cbh->partial_blocks_first, heap_index)[0] == CB_TAIL_MAGIC);
    }
#endif
    dfwmalloc_assert (cb->free_count < cb->used_count);
    dfwmalloc_cb_insert_free_block (cb, u);
#ifdef WITH_ACCOUNTS
    cbhs->total_alloced -= global->cb_heaps_info.page_info[cb->head_index].round_up;
    cbhs->cb_total_blocks--;
    cbhs->total_free++;
#endif

    if (__builtin_expect (cb->free_count == cb->used_count, 0)) {
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
        cb_int_t age = 0;
#define SHRINK_AGE      20
/* a single malloc/free should not cause a relatively expensive
 * dfwmalloc_cb_rankheap_insert/dfwmalloc_cb_rankheap_remove operation. However
 * if the user has explicitly called dfwmalloc_truncate_heap() then we make
 * an exception. ( See, for example, 17m22s "Toward the end of 2006..." ---
 * https://www.youtube.com/watch?u=RcWp5vwGlYU ) */
        if ((cb->cb_magic_flags & CB_FLAG_FLUSH) != cbhs->flush) {
            cb->cb_magic_flags ^= CB_FLAG_FLUSH;
            age = SHRINK_AGE + 1;
        } else {
            age = GET_CREATE_TIME - cb->create_time;
        }
        if (cb->used_count > (cb->total_count >> 2) || age > 3) {
#ifdef WITH_ACCOUNTS
            cbhs->total_free -= cb->free_count;
#endif
            dfwmalloc_cb_rankheap_insert (&cbhs->heaps[heap_index].rankheap, cb);
            UNLINK_CB (&cbhs->cb_heads[cb->head_index], cb);
            cb->head_index = 0;
            if (age > SHRINK_AGE)
                dfwmalloc_cb_shrink_heap (&cbhs->heaps[heap_index].heap, cbhs, (char *) dfwmalloc_cb_unlink_heap_tail (cbhs, heap_index));
            __sync_synchronize ();
            cbhs->cb_refs--;
            return;
        }
#endif
        deref = 1;
    }

    if (!(cb->cb_magic_flags & CB_FLAG_PARTIAL_LINKED)) {
        LINK_CB (cbh, cb);
    }

#ifndef NO_HEAP_TRUNCATION
/* Avoid fragmentation by performing one iteration of a bubble sort.
 * This moves the most-full blocks closer to the head of the list so
 * that the next malloc will likely fill up a block, say, that is
 * almost full. */
    dfwmalloc_cb_bubblesort (cbh, cb);
#endif

    if (deref) {
        __sync_synchronize ();
        cbhs->cb_refs--;
    }
}


STATICFN LONG dfwmalloc_cb_check (struct cb_heaps *cbhs, const void *const u)
{
    struct control_block_head *cbh;
    struct control_block *cb;
    ULONG64 m;
    int heap_index;
    (void) cbh;
    heap_index = CB_HEAP_GET_INDEX (u);
    m = ((ULONG64) HEAP_INDEX_TO_CHUNKSIZE (heap_index)) - 1;
    cb = (struct control_block *) (void *) ((ULONG64) u & ~m);
    dfwmalloc_assert (CB_MAGIC_CHK (cb));
    dfwmalloc_assert (CB_TAIL_P(cb, heap_index)[0] == CB_TAIL_MAGIC);
    dfwmalloc_assert (cb->free_count < cb->used_count);
    cbh = &cbhs->cb_heads[cb->head_index];
    dfwmalloc_assert (!cbh->partial_blocks_first || CB_MAGIC_CHK (cbh->partial_blocks_first));
    dfwmalloc_assert (!cbh->partial_blocks_first || CB_TAIL_P(cbh->partial_blocks_first, heap_index)[0] == CB_TAIL_MAGIC);
    return global->cb_heaps_info.page_info[cb->head_index].round_up;
}

#ifdef USE_TLS

/* @cb_size_t_to_ulong@start@ */

STATICFN ULONG cb_size_t_to_ulong (const cb_size_t s)
{
    if (SIZEOF_CB_SIZE == 2) {
        LONG s0, s1;
        s0 = s.d[0];
        s1 = s.d[1];
        return (s1 << 8) | (s0 << 0);
    } else if (SIZEOF_CB_SIZE == 3) {
        LONG s0, s1, s2;
        s0 = s.d[0];
        s1 = s.d[1];
        s2 = s.d[2];
        return (s2 << 16) | (s1 << 8) | (s0 << 0);
#if defined(__x86_64__)
    } else if (SIZEOF_CB_SIZE == 4 && sizeof (unsigned int) == 4) {
        return (ULONG) (unsigned int) ((unsigned int *) &s)[0];
#endif
    } else if (SIZEOF_CB_SIZE == 4) {
        LONG s0, s1, s2, s3;
        s0 = s.d[0];
        s1 = s.d[1];
        s2 = s.d[2];
        s3 = s.d[3];
        return (s3 << 24) | (s2 << 16) | (s1 << 8) | (s0 << 0);
#if defined(__x86_64__)
    } else if (SIZEOF_CB_SIZE == 5 && sizeof (unsigned int) == 4) {
        return ((ULONG) (unsigned int) ((unsigned int *) &s)[0]) | ((ULONG) s.d[4] << 32);
#endif
    } else if (SIZEOF_CB_SIZE == 5) {
        LONG s0, s1, s2, s3, s4;
        s0 = s.d[0];
        s1 = s.d[1];
        s2 = s.d[2];
        s3 = s.d[3];
        s4 = s.d[4];
        return (s4 << 32) | (s3 << 24) | (s2 << 16) | (s1 << 8) | (s0 << 0);
#if defined(__x86_64__)
    } else if (SIZEOF_CB_SIZE == 6 && sizeof (unsigned int) == 4) {
        return ((ULONG) (unsigned int) ((unsigned int *) &s)[0]) | ((ULONG) s.d[4] << 32) | ((ULONG) s.d[5] << 40);
#endif
    } else if (SIZEOF_CB_SIZE >= 6) {
        LONG s0, s1, s2, s3, s4, s5;
        s0 = s.d[0];
        s1 = s.d[1];
        s2 = s.d[2];
        s3 = s.d[3];
        s4 = s.d[4];
        s5 = s.d[5];
        return (s5 << 40) | (s4 << 32) | (s3 << 24) | (s2 << 16) | (s1 << 8) | (s0 << 0);
    } else {
        abort ();               /* implement your own for other sizes */
    }
}

STATICFN void ulong_to_cb_size_t (cb_size_t * s, const ULONG u)
{
    if (SIZEOF_CB_SIZE == 2) {
        s->d[0] = (u >> 0);
        s->d[1] = (u >> 8);
    } else if (SIZEOF_CB_SIZE == 3) {
        s->d[0] = (u >> 0);
        s->d[1] = (u >> 8);
        s->d[2] = (u >> 16);
#if defined(__x86_64__)
    } else if (SIZEOF_CB_SIZE == 4 && sizeof (unsigned int) == 4) {
        unsigned int *p;
        p = (unsigned int *) s;
        *p = (unsigned int) u;
#endif
    } else if (SIZEOF_CB_SIZE == 4) {
        s->d[0] = (u >> 0);
        s->d[1] = (u >> 8);
        s->d[2] = (u >> 16);
        s->d[3] = (u >> 24);
#if defined(__x86_64__)
    } else if (SIZEOF_CB_SIZE == 5 && sizeof (unsigned int) == 4) {
        unsigned int *p;
        p = (unsigned int *) s;
        *p = (unsigned int) u;
        s->d[4] = (u >> 32);
#endif
    } else if (SIZEOF_CB_SIZE == 5) {
        s->d[0] = (u >> 0);
        s->d[1] = (u >> 8);
        s->d[2] = (u >> 16);
        s->d[3] = (u >> 24);
        s->d[4] = (u >> 32);
#if defined(__x86_64__)
    } else if (SIZEOF_CB_SIZE == 6 && sizeof (unsigned int) == 4) {
        unsigned int *p;
        p = (unsigned int *) s;
        *p = (unsigned int) u;
        s->d[4] = (u >> 32);
        s->d[5] = (u >> 40);
#endif
    } else if (SIZEOF_CB_SIZE == 6) {
        s->d[0] = (u >> 0);
        s->d[1] = (u >> 8);
        s->d[2] = (u >> 16);
        s->d[3] = (u >> 24);
        s->d[4] = (u >> 32);
        s->d[5] = (u >> 40);
    } else {
        abort ();               /* implement your own for other sizes */
    }
}

/* @cb_size_t_to_ulong@end@ */

STATICFN void dfwmalloc_cb_sharedfree (struct dfwmalloc *dfw)
{
    int heap_index;
    for (heap_index = 0; heap_index < (int) CB_SIZE_MAX; heap_index++) {
        char *p;
        struct control_block_heap *cbh;
        cbh = &dfw->cb_heaps.heaps[heap_index];
        if (!cbh->heap.start_of_heap)
            continue;
        while (memcmp (&cbh->first_sharedfree_item, &empty_cb_list[0], sizeof (cb_size_t))) {
            p = &cbh->heap.start_of_heap[cb_size_t_to_ulong (cbh->first_sharedfree_item)];
            memcpy (&cbh->first_sharedfree_item, p, sizeof (cb_size_t));
            dfwmalloc_cb_free (&dfw->cb_heaps, (void *) p);
            /* BARRIER; not needed since this is called from one place only */
            dfw->shared_free_count--;
        }
    }
}

STATICFN void dfwmalloc_cb_linkshared (struct dfwmalloc *dfw, char *p)
{
    struct control_block_heap *cbh;
    int heap_index;
    heap_index = CB_HEAP_GET_INDEX (p);
    cbh = &dfw->cb_heaps.heaps[heap_index];
    memcpy (p, &cbh->first_sharedfree_item, sizeof (cb_size_t));
    ulong_to_cb_size_t (&cbh->first_sharedfree_item, (ULONG) (p - cbh->heap.start_of_heap));
    dfw->shared_free_count++;
}

#endif

#endif

#endif

#ifdef USE_TLS

STATICFN int dfwmalloc_heap_init (struct heap *h, int arena_id)
{
    ULONG64 u;
    dfwmalloc_assert (arena_id >= 0);
#ifdef NO_CB_HEAPS
    POINTER_FROM_ARENA (u, arena_id, 0);
#else
    POINTER_FROM_ARENA (u, arena_id, CB_SIZE_MAX);
#endif
    dfwmalloc_assert (ARENA_FROM_POINTER(u) == arena_id);
    return dfwmalloc_mmap_heap_init (h, MMAP_PAGE_SIZE_BIG, u);
}

STATICFN void dfwmalloc_heap_free (struct heap *h)
{
    dfwmalloc_mmap_heap_free (h);
}

STATICFN void *dfwmalloc_heap_rawalloc_locked (struct heap *h, LONG n, int alignment, int *need_zero)
{
    return dfwmalloc_mmap_heap_rawalloc_locked (h, n, alignment, global->max_heap, need_zero);
}

STATICFN void dfwmalloc_heap_rawalloc_shrink_heap_locked (struct heap *h, LONG n, unsigned int *flush)
{
    dfwmalloc_mmap_heap_rawalloc_shrink_heap_locked (h, n, flush);
}

#else


STATICFN int dfwmalloc_heap_init (struct heap *h, int arena_id)
{
    (void) arena_id;
    dfwmalloc_assert (arena_id == 0);
    dfwmalloc_assert (!h->end_of_heap && !h->start_of_heap);
    h->nonzero_heap = h->end_of_heap = h->start_of_heap = (char *) sbrk (0);
    return 0;
}

/* main arena never free's -- used for non-tls design only */
#if 0
STATICFN void dfwmalloc_heap_free (struct heap *h)
{
    LONG l;
    dfwmalloc_assert (h->end_of_heap && h->start_of_heap);
    l = (LONG) (h->end_of_heap - h->start_of_heap);
    l = -l;
    sbrk (l);
    memset (h, '\0', sizeof (*h));
}
#endif

STATICFN void *dfwmalloc_heap_rawalloc_locked (struct heap *h, LONG n, int alignment, int *need_zero)
{
    void *r, *old;
    int a;
    dfwmalloc_assert (n > 0);
    dfwmalloc_assert (h->end_of_heap && h->start_of_heap);
    a = dfwmalloc_get_alignment_adjustment (h->end_of_heap, alignment);
    old = h->end_of_heap;
    h->end_of_heap += a;
    r = h->end_of_heap;

    if (h->nonzero_heap <= h->end_of_heap)
        if (need_zero)
            *need_zero = 0;
    h->end_of_heap += n;

    if ((ULONG) (h->end_of_heap - h->start_of_heap) > global->max_heap) {
        h->end_of_heap = (char *) old;               /* revert */
        errno = ENOMEM;
        return NULL;
    }

    if (sbrk (a + n) == (void *) -1) {
        h->end_of_heap = (char *) old;     /* revert */
        return NULL;
    }
#ifdef FOR_VALGRIND
    memset_uninitialized (old, (long) (h->end_of_heap - (char *) old));
#endif

    if (h->nonzero_heap < h->end_of_heap)
        h->nonzero_heap = h->end_of_heap;

    return r;
}

STATICFN void dfwmalloc_heap_rawalloc_shrink_heap_locked (struct heap *h, LONG n, unsigned int *flush)
{
    void *old;
    (void) flush;
    dfwmalloc_assert (n >= 0);
    dfwmalloc_assert (h->end_of_heap && h->start_of_heap);
    dfwmalloc_assert ((LONG) (h->end_of_heap - h->start_of_heap) >= n);
    old = h->end_of_heap;
    h->end_of_heap -= n;
    if (sbrk (-n) == (void *) -1)
        h->end_of_heap = (char *) old;     /* revert */
}

#endif



#define GET_BUCKET(r, dfw, n) \
    do { \
        int __i; \
        __i = dfwmalloc_get_bucket ((dfw), (n)); \
    /* use lazy initialization to speed up startup times: */ \
        if (__builtin_expect ((dfw)->free_list_initialized <= __i, 0)) { \
            do { \
                dfwmalloc_zero_free_list_head ((dfw), (dfw)->free_list_initialized); \
                (dfw)->free_list_initialized++; \
            } while ((dfw)->free_list_initialized <= __i); \
        } \
        (r) = &(dfw)->free_list[__i]; \
    } while (0)

#ifndef NO_HEAP_TRUNCATION

STATICFNINLINE void dfwmalloc_unlink_locked (struct dfwmalloc *dfw, struct free_list *l, struct free_item *const p)
{
    (void) dfw;
    dfwmalloc_assert (l->fl_first);
    if (l->fl_first == p) {
        dfwmalloc_assert (!p->fl_prev);
        l->fl_first = GETP (dfw, p->fl_next);
        if (l->fl_first)
            l->fl_first->fl_prev = 0;
    } else {
        struct free_item *fl_prev;
        if (p->fl_next) {
            struct free_item *fl_next;
            fl_next = GETP (dfw, p->fl_next);
            fl_next->fl_prev = p->fl_prev;
        }
        fl_prev = GETP (dfw, p->fl_prev);
        fl_prev->fl_next = p->fl_next;
    }
}

STATICFNINLINE int dfwmalloc_unlink (struct dfwmalloc *dfw, struct free_item *const p, struct free_item **prev, struct calltrace *c)
{
    struct free_list *l;
    char *s;
    int is_free = 1;

    (void) c;

    if (!p)
        return 0;

    GET_BUCKET (l, dfw, LEN (p));         /* ($1$) */

#ifndef USE_TLS
    if (dfwmalloc_mutex_trylock (&l->bucket_mutex)) {       /* out of order use of locks means this could fail */
        *prev = NULL;
        return 0;
    }
#endif

    if (!IS_IN_USE (p)) {
        *prev = GETP (dfw, p->hp_prev);

        s = (char *) (void *) p;
        s += HEAD_SIZE;

        USER_ALERT (s[-1] == '{' && s[LEN (p)] == '}' && s[LEN (p) + 1] == '\0', "corrupted block on free list (truncate heap)");

#ifdef WITH_BTANDCHECK
        if (global->backtrace_enabled)
            dfwmalloc_poison_check_block (dfw, p, s, c);
        s[-1] = '$';
        s[0] = (char) 0xff;
        s[LEN (p)] = '*';
        p->len = 0;
#endif

        dfwmalloc_unlink_locked (dfw, l, p);
/* At this point p is guaranteed to not be referenced by anything at all. */

#ifdef WITH_ACCOUNTS
        l->count--;
#endif
    } else {
        *prev = NULL;
        is_free = 0;
    }

#ifndef USE_TLS
    dfwmalloc_mutex_unlock (&l->bucket_mutex);
#endif

    return is_free;
}

STATICFNINLINE void dfwmalloc_truncate_heap_locked (struct dfwmalloc *dfw, unsigned int *flush, struct calltrace *c)
{
    struct free_item *p = NULL, *q = NULL, *r = NULL;
    (void) c;
    p = dfw->last_block;
    if (p && !IS_IN_USE (p)) {
        for (;;) {
            r = q;
            q = p;
/* We are now unlocked w.r.t. any free list, so blocks could be changed
 * here, thus resulting in an immediate return of dfwmalloc_unlink() with
 * zero. In this situation r could be zero. */
            if (!dfwmalloc_unlink (dfw, p, &p, c))
                break;
        }
        if (r) {
            dfw->last_block = q;
            dfwmalloc_heap_rawalloc_shrink_heap_locked (dfw->heap, dfw->heap->end_of_heap - (char *) r, flush);
        }
    }
}
#endif


STATICFN void *dfwmalloc_heap_rawalloc (struct dfwmalloc *dfw, LONG n, int alignment, int *need_zero)
{
    void *r;
    dfwmalloc_assert (n > 0);
#ifndef USE_TLS
    dfwmalloc_mutex_lock (&dfw->heap->heap_mutex);
#endif
    r = dfwmalloc_heap_rawalloc_locked (dfw->heap, n, alignment, need_zero);
#ifndef NO_HEAP_TRUNCATION
    dfw->last_block = NULL;
#endif
#ifndef USE_TLS
    dfwmalloc_mutex_unlock (&dfw->heap->heap_mutex);
#endif
    return r;
}

#if !defined(USE_PRECOMPILED_STATIC_DATA) || defined(WITH_BTANDCHECK)

STATICFN void *dfwmalloc_raw_calloc (struct dfwmalloc *dfw, LONG n)
{
    void *r;
    int need_zero = 1;
    r = dfwmalloc_heap_rawalloc (dfw, n, POINTER_ALIGNMENT, &need_zero);
    if (r && need_zero)
        memset (r, '\0', n);
    return r;
}

#endif

#ifdef WITH_BTANDCHECK

STATICFN struct backtrace_item *dfwmalloc_add_backtrace_locked (struct dfwmalloc *dfw, struct backtrace *g,
                                                            ULONG hash1, ULONG hash2, void **addr, int n)
{
    struct backtrace_item *b, **b_;

    b = (struct backtrace_item *) dfwmalloc_raw_calloc (dfw, sizeof (*b) + sizeof (void *) * n);
    b->n = n;
    b->hash2 = hash2;
    memcpy (b->addr, addr, n * sizeof (void *));

    dfwmalloc_assert (g->backtrace_id < (1L << (BACKTRACE_ID_HIGH + BACKTRACE_ID_LOW)));
    b->backtrace_id = (backtrace_id_t) g->backtrace_id;
    /* link: */
    b->bt_next = g->backtrace_head[hash1].bt_first;

/* order here is important since these arrays are accessed unlocked by the backtrace dumper */
    __sync_synchronize ();
    dfwmalloc_backtrace_array (dfw, g, b->backtrace_id, &b_);
    *b_ = b;
    __sync_synchronize ();
    g->backtrace_id++;
    __sync_synchronize ();
    g->backtrace_head[hash1].bt_first = b;

    return b;
}

STATICFN struct backtrace_item *dfwmalloc_find_backtrace (struct backtrace *g, ULONG hash1, ULONG hash2, void *addr, int n)
{
    struct backtrace_item *b;
    for (b = g->backtrace_head[hash1].bt_first; b; b = b->bt_next)
        if (hash2 == b->hash2)
            if (n == b->n && !memcmp (b->addr, addr, n * sizeof (void *)))
                return b;
    return NULL;
}

STATICFN ULONG dfwmalloc_hash_address (void **addr, int n)
{
    int i;
    ULONG hash1 = 51929;
    ULONG hash2 = 24543;

    for (i = 0; i < n; i++) {
        ULONG c1, c2;
        hash1 ^= (ULONG) addr[i];
        hash2 ^= (ULONG) addr[i] >> 11;
        c1 = hash1 % 3259;
        c2 = hash2 % 3259;
        hash1 ^= (((c1 + 9) * (c1 + 2) * 401) >> 1);
        hash2 ^= (((c2 + 9) * (c2 + 2) * 401) >> 1);
    }

    return (hash1 ^ hash2);
}

#ifdef WITH_BTANDCHECK
STATICFN void dfwmalloc_backtrace_account (struct dfwmalloc *dfw, backtrace_id_t backtrace_id, LONG alloced, LONG blocks)
{
    (void) dfw;
    if (global->backtrace_enabled && backtrace_id != INVALID_BACKTRACE_ID) {
        struct backtrace_item **b;
        dfwmalloc_backtrace_array (dfw, &dfw->backtrace, backtrace_id, &b);
        dfwmalloc_assert (*b);
        dfwmalloc_synchronous_add (&(*b)->bt_alloced, alloced);
        dfwmalloc_synchronous_add (&(*b)->bt_blocks, blocks);
    }
}

#ifdef USE_TLS
STATICFN void dfwmalloc_backtrace_account_otherarena (struct dfwmalloc *dfw, struct backtrace *g, backtrace_id_t backtrace_id, LONG alloced, LONG blocks)
{
    if (global->backtrace_enabled && backtrace_id != INVALID_BACKTRACE_ID) {
        struct backtrace_item **b;
        dfwmalloc_backtrace_array (dfw, g, backtrace_id, &b);
        dfwmalloc_assert (*b);
        dfwmalloc_mutex_lock (&g->backtrace_mutex);
        (*b)->bt_alloced_otherarena += alloced;
        (*b)->bt_blocks_otherarena += blocks;
        dfwmalloc_mutex_unlock (&g->backtrace_mutex);
    }
}
#endif
#endif

STATICFN backtrace_id_t dfwmalloc_backtrace (struct dfwmalloc *dfw, struct backtrace *g, struct calltrace *c)
{
    ULONG hash, hash1, hash2;
    struct backtrace_item *b;

    if (!c->n)
        return INVALID_BACKTRACE_ID;

    hash = dfwmalloc_hash_address (c->addr, c->n);
    hash1 = hash % BACKTRACE_HEADS;
    hash2 = (hash / BACKTRACE_HEADS) & 0xffff;
    b = dfwmalloc_find_backtrace (g, hash1, hash2, c->addr, c->n);
    if (!b) {
/* this lock only happens for *new* backtrace items, so it will have little impact on performance */
        dfwmalloc_mutex_lock (&g->backtrace_mutex);
        b = dfwmalloc_add_backtrace_locked (dfw, g, hash1, hash2, c->addr, c->n);
        dfwmalloc_mutex_unlock (&g->backtrace_mutex);
    }
    return b->backtrace_id;
}

#endif


STATICFNINLINE int dfwmalloc_get_bucket (struct dfwmalloc *dfw, LONG n)
{
    int bits, shift = 0;
    LONG g;
    (void) dfw;
    g = 1L + (n >> WANTED_ALIGNMENT);
    bits = __builtin_clzl (g);
    if (bits < 64 - BUCKET_SMOOTHNESS)
        shift = 64 - BUCKET_SMOOTHNESS - bits;
    g >>= shift;
    return (int) (g + ((1 << (BUCKET_SMOOTHNESS - 1)) * shift));
}

#ifdef WITH_UNITTESTS

int __dfwmalloc_get_bucket (struct dfwmalloc *dfw, LONG n)
{
    return dfwmalloc_get_bucket (dfw, n);
}

#endif

#ifndef USE_PRECOMPILED_STATIC_DATA
/* 
 *  Perform an iterative calculation to make it true that
 * 
 *    dfwmalloc_get_bucket (dfw, alloc_size + 1) == dfwmalloc_get_bucket (dfw, alloc_size) + 1
 * 
 *  for all buckets.
 */
STATICFN LONG calc_bucket (struct dfwmalloc *dfw, int k, LONG r)
{
    long inc;

    inc = 1;
    while (dfwmalloc_get_bucket (dfw, r) < k + 1) {
        r += inc;
        inc = (inc + 1) * 3 / 2;
    }

    for (; inc; inc >>= 1)
        while (dfwmalloc_get_bucket (dfw, r - inc + 1) > k)
            r -= inc;

    dfwmalloc_assert (dfwmalloc_get_bucket (dfw, r + 1) == dfwmalloc_get_bucket (dfw, r) + 1);
    dfwmalloc_assert (k == dfwmalloc_get_bucket (dfw, r));

    return r;
}
#endif

STATICFN int dfwmalloc_init_global_free_lists (struct dfwmalloc *dfw)
{
#ifndef USE_PRECOMPILED_STATIC_DATA
    LONG u;
    int k, n_freelists = 0;

#ifdef FOR_VALGRIND
    dfwmalloc_assert (BUCKET_SMOOTHNESS <= 4);
#endif

    for (u = 1, k = 1;; k++) {
        u = calc_bucket (NULL, k, u);
        if (u >= (LONG) dfw->max_alloc) {
            n_freelists = k + 1;
            break;
        }
    }
    global->bucket_size = (malloc_size_t *) dfwmalloc_raw_calloc (dfw, n_freelists * sizeof (malloc_size_t));       /* the first "arena" is never free'd */
    if (!global->bucket_size)
        return 1;

    dfwmalloc_always_assert (global->bucket_size);

    global->n_freelists = n_freelists;
    global->bucket_size[0] = 0;

    for (u = 1, k = 1; k < n_freelists; k++) {
        u = calc_bucket (NULL, k, u);
        if (u >= (LONG) dfw->max_alloc) {
            global->bucket_size[k] = dfw->max_alloc;
            break;
        }
        global->bucket_size[k] = u;
    }
#else
    (void) dfw;
    global->bucket_size = global_free_list_sizes;
    global->n_freelists = n_global_free_list_sizes;
#endif
    return 0;
}

STATICFN void dfwmalloc_zero_free_list_head (struct dfwmalloc *dfw, int k)
{
#ifdef WITH_ACCOUNTS
    dfw->free_list[k].count = 0;
#endif
    dfw->free_list[k].fl_first = NULL;
#ifndef USE_TLS
    dfwmalloc_mutex_init (&dfw->free_list[k].bucket_mutex, NULL);
#endif
}

STATICFN int dfwmalloc_init_free_lists (struct dfwmalloc *dfw)
{
    int need_zero = 1;

#ifdef USER_MAX_MALLOC
    dfw->max_alloc = USER_MAX_MALLOC;
#else
    if ((ULONG64) MALLOC_MAX < (ULONG64) MALLOC_MAX_64_BIT || sizeof (size_t) <= 4)
        dfw->max_alloc = LEN_MASK;
    else
        dfw->max_alloc = MALLOC_MAX_64_BIT;    /* 16 terabytes */
    if (dfw->max_alloc > MAX_HEAP)
        dfw->max_alloc = MAX_HEAP;
#ifdef USE_TLS
    if (dfw->max_alloc > MAX_TLS_HEAP)
        dfw->max_alloc = MAX_TLS_HEAP;
#endif
    dfw->max_alloc -= (HEAD_SIZE + TAIL_SIZE);
#endif
#ifdef FOR_VALGRIND
    dfw->max_alloc = 150000;
#endif

#ifdef WITH_BTANDCHECK
    dfw->backtrace.backtrace_id = 1L;
#endif

#ifndef USE_TLS
    dfwmalloc_assert (!global->n_freelists);
#endif

/* first time this is calculated, and thereafter assigned in
 * dfwmalloc_get_bucket_(). this makes for faster thread startup times: */
    if (!global->n_freelists)
        if (dfwmalloc_init_global_free_lists (dfw))     /* the first arena never deletes its heap, so this is ok */
            return 1;

    dfw->free_list = (struct free_list *) dfwmalloc_heap_rawalloc (dfw, global->n_freelists * sizeof (dfw->free_list[0]), POINTER_ALIGNMENT, &need_zero);
    (void) need_zero;
    if (!dfw->free_list)
        return 1;
    dfwmalloc_zero_free_list_head (dfw, 0);
#ifdef USE_TLS
    dfw->free_list_initialized = need_zero ? 1 : global->n_freelists;
#else
    dfw->free_list_initialized = 1;
#endif
    return 0;
}

STATICFN int dfwmalloc_startup (struct dfwmalloc *dfw, struct heap *heap, int arena_id)
{
    memset (dfw, '\0', sizeof (*dfw));
    dfw->arena_id = arena_id;
    dfw->heap = heap;
#ifdef USE_TLS
    dfwmalloc_mutex_init (&dfw->sharedfree_mutex, NULL);
#endif
#ifdef WITH_BTANDCHECK
    dfwmalloc_mutex_init (&dfw->backtrace.backtrace_mutex, NULL);
#endif
    if (dfwmalloc_init_free_lists (dfw))
        return 1;
#if !defined(NO_LOCKFREEVISITOR) && defined(USE_TLS)
    lfv_init (&dfw->lfv);
#endif
#ifndef NO_CB_HEAPS
    dfwmalloc_cb_init (&dfw->cb_heaps);
#endif
    __sync_synchronize ();
    return 0;
}

#ifdef WITH_BTANDCHECK
STATICFN void dfwmalloc_poison_check_block (struct dfwmalloc *dfw, const struct free_item *p, const char *s, struct calltrace *c)
{
    LONG *w;
    int g;
    if (backtrace_id_to_long (&p->malloc_trace) == INVALID_BACKTRACE_ID)        /* This means the malloc happened before backtracing was enabled. */
        return;
    w = (LONG *) s;
    if (LEN (p) >= sizeof (LONG) * 2) {
        g = -(w[0]);
        USER_ALERT (w[1 + g] == -1, "user wrote to freed block (1)");
    } else if (LEN (p) >= sizeof (LONG)) {
        USER_ALERT (w[0] == -1, "user wrote to freed block (2)");
    } else {
        USER_ALERT (s[0] == (char) 0x7f, "user wrote to freed block (3)");
    }
}

STATICFN void dfwmalloc_poison_free_block (struct free_item *p, char *s)
{
    int g;
    LONG *w;
    w = (LONG *) s;
    if (LEN (p) >= sizeof (LONG) * 2) {
        static int cc = 0;
        cc++;
        g = cc % ((LEN (p) - sizeof (LONG)) / sizeof (LONG));
        w[0] = -g;
        w[1 + g] = -1;
    } else if (LEN (p) >= sizeof (LONG)) {
        w[0] = -1;
    } else {
        s[0] = (char) 0x7f;
    }
}
#endif

STATICFNINLINE void *dfwmalloc_locked (struct dfwmalloc *dfw, struct free_list *l, const malloc_size_t n, int alignment, int *need_zero, struct calltrace *c)
{
    char *s;
    struct free_item *p;
#ifdef WITH_BTANDCHECK
    backtrace_id_t malloc_trace;
/* backtrace items cannot be free'd, so allocate this before the
 * rawalloc below. this allows the rawalloc below to be amenable to heap
 * truncation. */
    malloc_trace = dfwmalloc_backtrace (dfw, &dfw->backtrace, c);
#endif
    (void) c;
    dfwmalloc_assert (n <= BUCKET_SIZE (dfw, l));
    if ((p = l->fl_first)) {
        s = (char *) (void *) p;
        s += HEAD_SIZE;

/* There could be something on the free list that was allocated
 * with alignment requirements less strict than the current call.
 * Reuse would return a block with bad alignment. */
        if (__builtin_expect(alignment != NO_ALIGNMENT && ((ULONG64) s & (alignment - 1)), 0))
            goto here;

/* unlink first element from free list: */
        l->fl_first = GETP (dfw, p->fl_next);
#ifndef NO_HEAP_TRUNCATION
        if (l->fl_first)
            l->fl_first->fl_prev = 0;
#endif
#ifdef WITH_ACCOUNTS
        l->count--;
#endif

        USER_ALERT (s[-1] == '{' && s[LEN (p)] == '}' && s[LEN (p) + 1] == '\0', "corrupted block on free list");
#ifdef WITH_BTANDCHECK
        if (global->backtrace_enabled)
            dfwmalloc_poison_check_block (dfw, p, s, c);
#endif
        p->len = n;     /* This is mathematically guaranteed not to change to a different free list. See for example ($2$). */
        dfwmalloc_assert (s + p->len + TAIL_SIZE <= dfw->heap->end_of_heap);
    } else {
        malloc_size_t len;
#ifndef NO_HEAP_TRUNCATION
        struct free_item *last_block;
#endif
      here:
        len = HEAD_SIZE + BUCKET_SIZE (dfw, l) + TAIL_SIZE;
        if (alignment < (1 << WANTED_ALIGNMENT))
            alignment = (1 << WANTED_ALIGNMENT);
#ifndef USE_TLS
        dfwmalloc_mutex_lock (&dfw->heap->heap_mutex);
#endif
#ifndef NO_HEAP_TRUNCATION
        last_block = dfw->last_block;
#endif
        p = (struct free_item *) dfwmalloc_heap_rawalloc_locked (dfw->heap, len, alignment, need_zero);
        if (p) {
#ifndef NO_HEAP_TRUNCATION
            dfw->last_block = p;
            p->hp_prev = SETP (dfw, last_block);
#endif
            p->len = n; /* must be inside the heap lock so that ($1$) can get the lock on the correct list. this is pretty subtle */
        }
#ifndef USE_TLS
        dfwmalloc_mutex_unlock (&dfw->heap->heap_mutex);
#endif
        if (!p)
            return NULL;
        s = (char *) (void *) p;
        s += HEAD_SIZE;
#ifdef WITH_BTANDCHECK
        s[-1] = '{';
#endif
        dfwmalloc_assert (s + p->len + TAIL_SIZE <= dfw->heap->end_of_heap);
    }
#ifdef WITH_BTANDCHECK
    backtrace_id_from_long (&p->malloc_trace, malloc_trace);
    dfwmalloc_backtrace_account (dfw, malloc_trace, (LONG) n, 1L);
    backtrace_id_from_long (&p->free_trace, 0L);
#endif

    dfwmalloc_assert (s + p->len + TAIL_SIZE <= dfw->heap->end_of_heap);
#ifdef WITH_BTANDCHECK
    s[p->len] = '~';
    s[p->len + 1] = '\0';
#endif
    return s;
}

STATICFNINLINE void *dfwmalloc_with_truncation (struct dfwmalloc *dfw, struct free_list *l, const malloc_size_t n, int alignment, int *need_zero, struct calltrace *c)
{
    void *u;
#ifndef NO_HEAP_TRUNCATION
    if (!l->fl_first) {            /* there seems to be no point in incurring the work of truncation if the free list has something immediately on it */
#ifndef USE_TLS
        dfwmalloc_mutex_lock (&dfw->heap->heap_mutex);
#endif
        dfwmalloc_truncate_heap_locked (dfw, &dfw->flush, c);
#ifndef USE_TLS
        dfwmalloc_mutex_unlock (&dfw->heap->heap_mutex);
#endif
    }
#endif
#ifndef USE_TLS
    dfwmalloc_mutex_lock (&l->bucket_mutex);
#endif
    u = dfwmalloc_locked (dfw, l, n, alignment, need_zero, c);
#ifndef USE_TLS
    dfwmalloc_mutex_unlock (&l->bucket_mutex);
#endif
    return u;
}

STATICFNINLINE void dfwfree__ (struct dfwmalloc *dfw, struct free_list *l, struct free_item *p)
{
    (void) dfw;
#ifndef USE_TLS
    dfwmalloc_mutex_lock (&l->bucket_mutex);
#endif
#ifdef WITH_ACCOUNTS
    l->count++;
#endif
    dfwmalloc_assert (IS_IN_USE (p));

/* link with free element of list: */
#ifndef NO_HEAP_TRUNCATION
    if (l->fl_first)
        l->fl_first->fl_prev = SETP (dfw, p);
#endif
    p->fl_next = SETP (dfw, l->fl_first);
#ifndef NO_HEAP_TRUNCATION
    p->fl_prev = 0;
#endif
    l->fl_first = p;

    SET_FREE (p);
#ifndef USE_TLS
    dfwmalloc_mutex_unlock (&l->bucket_mutex);
#endif
}

STATICFN void *dfwrealloc (struct dfwmalloc *dfw, void *u, const malloc_size_t n, struct calltrace *c)
{
    struct free_list *lold, *lnew;
    struct free_item *p;
    char *s;
    LONG delta;
#ifdef WITH_BTANDCHECK
    LONG malloc_trace;
#endif

    s = (char *) u;
    p = (struct free_item *) (s - HEAD_SIZE);
    USER_ALERT (!(s[p->len] == '}' || s[p->len] == '@'), "caller trying to realloc already-freed block");
    USER_ALERT (s[-1] == '{' && s[p->len] == '~' && s[p->len + 1] == '\0', "corrupted block on realloc");
#ifdef WITH_BTANDCHECK
    s[p->len] = '}';
    malloc_trace = backtrace_id_to_long (&p->malloc_trace);
#endif

    delta = (LONG) n - (LONG) p->len;

    GET_BUCKET (lold, dfw, p->len);
    GET_BUCKET (lnew, dfw, n);

    dfwmalloc_assert (n <= BUCKET_SIZE (dfw, lnew));

#ifdef WITH_ACCOUNTS
#ifdef WITH_BTANDCHECK
    if (global->backtrace_enabled)
        dfwmalloc_synchronous_add (&dfw->total_alloced_bt, delta);
    else
#endif
        dfwmalloc_synchronous_add (&dfw->total_alloced, delta);
#endif

    if (lold == lnew) {
        p->len = n;             /* ($2$) */
#ifdef WITH_BTANDCHECK
        s[p->len] = '~';
        s[p->len + 1] = '\0';
        dfwmalloc_backtrace_account (dfw, malloc_trace, delta, 0L);
#endif
    } else {
        char *end_of_heap;
        int adjust_heap_tail_done = 0;
        char *t;

        end_of_heap = ((char *) p) + HEAD_SIZE + BUCKET_SIZE (dfw, lold) + TAIL_SIZE;
        if (end_of_heap == dfw->heap->end_of_heap) {
#ifndef USE_TLS
            dfwmalloc_mutex_lock (&dfw->heap->heap_mutex);
#endif
            if (end_of_heap == dfw->heap->end_of_heap) {
                if (BUCKET_SIZE (dfw, lnew) > BUCKET_SIZE (dfw, lold)) {
                    void *discard;
                    discard = dfwmalloc_heap_rawalloc_locked (dfw->heap, (LONG) BUCKET_SIZE (dfw, lnew) - (LONG) BUCKET_SIZE (dfw, lold), 1, NULL);
                    /* dfw->last_block does not change */
                    if (!discard) {
                        s = NULL;
                        goto out;
                    }
                } else {
                    dfwmalloc_heap_rawalloc_shrink_heap_locked (dfw->heap, (LONG) BUCKET_SIZE (dfw, lold) - (LONG) BUCKET_SIZE (dfw, lnew), &dfw->flush);
                }

                p->len = ((LONG) p->len + delta);       /* heap_mutex must be locked while we update len due to ($1$) */

#ifdef WITH_BTANDCHECK
                s[p->len] = '~';
                s[p->len + 1] = '\0';
                dfwmalloc_backtrace_account (dfw, malloc_trace, delta, 0L);
#endif

              out:
                adjust_heap_tail_done = 1;
            }
#ifndef USE_TLS
            dfwmalloc_mutex_unlock (&dfw->heap->heap_mutex);
#endif
        }

        if (!adjust_heap_tail_done) {
            s = (char *) dfwmalloc_with_truncation (dfw, lnew, n, NO_ALIGNMENT, NULL, c);

            t = (char *) u;
            if (!s) {
#ifdef WITH_ACCOUNTS
#ifdef WITH_BTANDCHECK
                if (global->backtrace_enabled)
                    dfwmalloc_synchronous_add (&dfw->total_alloced_bt, -delta);
                else
#endif
                    dfwmalloc_synchronous_add (&dfw->total_alloced, -delta);
#endif
                return NULL;  /* Linux realloc man page: "If realloc() fails, the original block is left untouched; it is not freed or moved." */
            }
            memcpy (s, t, p->len < n ? p->len : n);
#ifdef WITH_BTANDCHECK
            if (global->backtrace_enabled)
                dfwmalloc_poison_free_block (p, t);
#endif
/* you don't want to backtrace after the free because another thread
 * could pick up the free block. See the same at (*4*) */
#ifdef WITH_BTANDCHECK
            backtrace_id_from_long (&p->free_trace, dfwmalloc_backtrace (dfw, &dfw->backtrace, c));
            dfwmalloc_backtrace_account (dfw, malloc_trace, -((LONG) LEN (p)), -1L);
#endif
            dfwfree__ (dfw, lold, p);
        }
    }
    return s;
}

STATICFNINLINE void *dfwmalloc_ (struct dfwmalloc *dfw, const malloc_size_t n, int alignment, int *need_zero, struct calltrace *c)
{
    struct free_list *l;
    void *u;

    if (alignment != NO_ALIGNMENT)
        goto here;

#ifndef NO_CB_HEAPS
    if (__builtin_expect (
#ifdef WITH_BTANDCHECK
        !global->backtrace_enabled && 
#endif
        n <= CB_N_SIZES, 1)) {     /* no compact heaps with backtracing since compact heaps don't support this */
        void *r;
#ifndef USE_TLS
        dfwmalloc_mutex_lock (&dfw->cb_heaps.mutex);
#endif
        r = dfwmalloc_cb_malloc (&dfw->cb_heaps, dfw->arena_id, n);
#ifndef USE_TLS
        dfwmalloc_mutex_unlock (&dfw->cb_heaps.mutex);
#endif
        return r;
    }
#endif

  here:

    if (n > dfw->max_alloc) {
        errno = ENOMEM;
        return NULL;
    }

#ifdef WITH_ACCOUNTS
#ifdef WITH_BTANDCHECK
    if (global->backtrace_enabled)
        dfwmalloc_synchronous_add (&dfw->total_alloced_bt, (LONG) n);
    else
#endif
        dfwmalloc_synchronous_add (&dfw->total_alloced, (LONG) n);
#endif

#if defined(WITH_ACCOUNTS) || defined(USE_TLS)
#ifdef WITH_BTANDCHECK
    if (global->backtrace_enabled)
        dfwmalloc_synchronous_add (&dfw->total_blocks_bt, 1L);
    else
#endif
        dfwmalloc_synchronous_add (&dfw->total_blocks, 1L);
#endif

    GET_BUCKET (l, dfw, n);

    u = dfwmalloc_with_truncation (dfw, l, n, alignment, need_zero, c);
    if (__builtin_expect (u == NULL, 0)) {

#ifdef WITH_ACCOUNTS
#ifdef WITH_BTANDCHECK
        if (global->backtrace_enabled)
            dfwmalloc_synchronous_add (&dfw->total_alloced_bt, -((LONG) n));
        else
#endif
            dfwmalloc_synchronous_add (&dfw->total_alloced, -((LONG) n));
#endif

        __sync_synchronize ();

#if defined(WITH_ACCOUNTS) || defined(USE_TLS)
#ifdef WITH_BTANDCHECK
        if (global->backtrace_enabled)
            dfwmalloc_synchronous_add (&dfw->total_blocks_bt, -1L);
        else
#endif
            dfwmalloc_synchronous_add (&dfw->total_blocks, -1L);
#endif
    }

    return u;
}

STATICFNINLINE void dfwfree_ (struct dfwmalloc *dfw, struct free_item *p)
{
    struct free_list *l;
#ifdef WITH_ACCOUNTS
    LONG len;
#endif

#ifndef NO_CB_HEAPS
    dfwmalloc_assert (!CB_HEAP_CHECK (p));
#endif

#ifdef WITH_ACCOUNTS
    len = (LONG) p->len;
#endif

    GET_BUCKET (l, dfw, p->len);
    dfwfree__ (dfw, l, p);

#ifdef WITH_ACCOUNTS
#ifdef WITH_BTANDCHECK
    if (global->backtrace_enabled && backtrace_id_to_long (&p->malloc_trace) != INVALID_BACKTRACE_ID)
        dfwmalloc_synchronous_add (&dfw->total_alloced_bt, -len);
    else
#endif
        dfwmalloc_synchronous_add (&dfw->total_alloced, -len);
#endif

    __sync_synchronize ();

#if defined(WITH_ACCOUNTS) || defined(USE_TLS)
#ifdef WITH_BTANDCHECK
    if (global->backtrace_enabled && backtrace_id_to_long (&p->malloc_trace) != INVALID_BACKTRACE_ID)
        dfwmalloc_synchronous_add (&dfw->total_blocks_bt, -1L);
    else
#endif
        dfwmalloc_synchronous_add (&dfw->total_blocks, -1L);
#endif
}

#ifdef USE_TLS
/* call free on any blocks that were free'd by other threads */
STATICFNNOINLINE void dfwmalloc_free_shared_pointers_ (struct dfwmalloc *dfw, struct calltrace *c)
{
    struct free_item *p;
    (void) c;
#ifndef NO_LOCKFREEVISITOR
    dfwmalloc_lfv_free_shared (dfw);
    if (!dfw->shared_free_count)
        return; /* don't waist energy on a lock */
    __sync_synchronize ();
#endif
    dfwmalloc_mutex_lock (&dfw->sharedfree_mutex);
    dfw->shared_free_count++;
    __sync_synchronize ();
#ifndef NO_CB_HEAPS
    dfwmalloc_cb_sharedfree (dfw);
#endif
    p = dfw->sharedfree_list;
    while (p) {
        struct free_item *sh_next;
        char *s;
        sh_next = GETP (dfw, p->sh_next);
        s = (char *) (void *) p;
        s += HEAD_SIZE;
        dfwmalloc_assert (s + p->len + TAIL_SIZE <= dfw->heap->end_of_heap);
        USER_ALERT (s[-1] == '{' && s[p->len] == '@' && s[p->len + 1] == '\0', "corrupted block found on free (shared)");
#ifdef WITH_BTANDCHECK
        s[p->len] = '}';
        if (global->backtrace_enabled)
            dfwmalloc_poison_check_block (dfw, p, s, c);
#endif
        dfwfree_ (dfw, p);
        p = sh_next;
        dfw->shared_free_count--;
    }
    dfw->sharedfree_list = NULL;
    __sync_synchronize ();
    dfw->shared_free_count--;
    dfwmalloc_mutex_unlock (&dfw->sharedfree_mutex);
}

/* call free on any blocks that were free'd by other threads */
STATICFN void dfwmalloc_free_shared_pointers (struct dfwmalloc *dfw, struct calltrace *c)
{
#ifndef NO_LOCKFREEVISITOR
    if (!dfw->lfv.lfv_d && dfw->shared_free_count > 100) {
        struct lfitem *p;
        int need_zero = 1;
        p = (struct lfitem *) dfwmalloc_heap_rawalloc (dfw, sizeof (struct lfitem) * N_LFV_ITEMS, POINTER_ALIGNMENT, &need_zero);
        (void) need_zero;
        __sync_synchronize ();
        dfw->lfv.lfv_d = p;
    }
#endif
    dfwmalloc_free_shared_pointers_ (dfw, c);
}
#endif

STATICFNINLINE void dfwfree (struct dfwmalloc *dfw, void *u, struct calltrace *c)
{
    struct free_item *p;
    char *s;

#ifndef NO_CB_HEAPS
    if (__builtin_expect (CB_HEAP_CHECK (u), 1)) {      /* (*5*) */
#ifndef USE_TLS
        dfwmalloc_mutex_lock (&dfw->cb_heaps.mutex);
#endif
        dfwmalloc_cb_free (&dfw->cb_heaps, u);
#ifndef USE_TLS
        dfwmalloc_mutex_unlock (&dfw->cb_heaps.mutex);
#endif
        return;
    }
#endif

    (void) c;
    s = (char *) u;
    p = (struct free_item *) (s - HEAD_SIZE);
/* since this block is not free'd, can we can use p->len instead of
 * LEN(p)? ans: no, we have to account for possible double free: */
    dfwmalloc_assert (s + LEN (p) + TAIL_SIZE <= dfw->heap->end_of_heap);
    USER_ALERT (!(s[LEN (p)] == '}' || s[LEN (p)] == '@'), "caller trying to free already-freed block");
    USER_ALERT (s[-1] == '{' && s[LEN (p)] == '~' && s[LEN (p) + 1] == '\0', "corrupted block found on free");

/* we want this regardless of build options: */
    if (__builtin_expect (!IS_IN_USE (p), 0)) {
        dfwmalloc_writestr ("DFWMalloc double free detected, aborting\n");
        abort ();
    }

#ifdef WITH_BTANDCHECK
    s[LEN (p)] = '}';
    if (global->backtrace_enabled)
        dfwmalloc_poison_free_block (p, s);
#endif

/* you don't want to backtrace after the free because another thread
 * could pick up the free block. See the same at (*4*) */
#ifdef WITH_BTANDCHECK
    backtrace_id_from_long (&p->free_trace, dfwmalloc_backtrace (dfw, &dfw->backtrace, c));
    dfwmalloc_backtrace_account (dfw, backtrace_id_to_long (&p->malloc_trace), -((LONG) LEN (p)), -1L);
#endif
    dfwfree_ (dfw, p);
}

#if !defined(NO_LOCKFREEVISITOR) && defined(USE_TLS)
STATICFNNOINLINE void dfwmalloc_lfv_free_shared (struct dfwmalloc *dfw)
{
    struct calltrace c;
    void *u;
    while ((u = lfv_pop (&dfw->lfv, &c)))
        dfwfree (dfw, u, &c);
}
#endif

STATICFN LONG dfwcheck (struct dfwmalloc *dfw, const void *u, struct calltrace *c)
{
    const struct free_item *p;
    const char *s;
    (void) dfw;

#ifndef NO_CB_HEAPS
    if (CB_HEAP_CHECK (u)) {
        int r;
#ifndef USE_TLS
        dfwmalloc_mutex_lock (&dfw->cb_heaps.mutex);
#endif
        r = dfwmalloc_cb_check (&dfw->cb_heaps, u);
#ifndef USE_TLS
        dfwmalloc_mutex_unlock (&dfw->cb_heaps.mutex);
#endif
        return r;
    }
#endif

    (void) c;
    s = (const char *) u;
    p = (const struct free_item *) (s - HEAD_SIZE);
    dfwmalloc_assert (s + p->len + TAIL_SIZE <= dfw->heap->end_of_heap);
    USER_ALERT (!(s[p->len] == '}' || s[p->len] == '@'), "caller trying to free already-freed block");
    USER_ALERT (s[-1] == '{' && s[p->len] == '~' && s[p->len + 1] == '\0', "corrupted block found on free");

    return (LONG) p->len;
}

#ifdef USE_TLS

STATICFN void dfwfree_delayed (struct dfwmalloc *this_arena, struct dfwmalloc *dfw, void *u, struct calltrace *c)
{
    struct free_item *p;
    char *s;
    (void) this_arena;

#ifndef NO_CB_HEAPS
    if (CB_HEAP_CHECK (u)) {
        dfwmalloc_mutex_lock (&dfw->sharedfree_mutex);
        dfwmalloc_cb_linkshared (dfw, (char *) u);
        dfwmalloc_mutex_unlock (&dfw->sharedfree_mutex);
        return;
    }
#endif

    (void) c;
    s = (char *) u;
    p = (struct free_item *) (s - HEAD_SIZE);
    dfwmalloc_assert (s + p->len + TAIL_SIZE <= dfw->heap->end_of_heap);
    USER_ALERT (!(s[p->len] == '}' || s[p->len] == '@'), "caller trying to free already-freed block (delayed)");
    USER_ALERT (s[-1] == '{' && s[p->len] == '~' && s[p->len + 1] == '\0', "corrupted block found on free");
#ifdef WITH_BTANDCHECK
    s[p->len] = '@';
    if (global->backtrace_enabled)
        dfwmalloc_poison_free_block (p, s);
#endif

    dfwmalloc_mutex_lock (&dfw->sharedfree_mutex);
    p->sh_next = SETP (dfw, dfw->sharedfree_list);
    dfw->sharedfree_list = p;
    dfw->shared_free_count++;

    dfwmalloc_mutex_unlock (&dfw->sharedfree_mutex);

#ifdef WITH_BTANDCHECK
    backtrace_id_from_long (&p->free_trace, dfwmalloc_backtrace (this_arena, &dfw->backtrace, c)); /* (*11*) */
    dfwmalloc_backtrace_account_otherarena (this_arena, &dfw->backtrace, backtrace_id_to_long (&p->malloc_trace), -((LONG) p->len), -1L);
#endif
}

STATICFNNOINLINE void dfwfree_shared_pointer (struct dfwmalloc *this_arena, void *u, struct calltrace *c)
{
    struct dfwmalloc *dfw;
    dfw = dfwmalloc_find_pointer (global, u);
    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
    if (dfw) {
#ifndef NO_LOCKFREEVISITOR
        if (lfv_push (this_arena->arena_id, &dfw->lfv, u, c))
            dfwfree_delayed (this_arena, dfw, u, c);
#else
        dfwfree_delayed (this_arena, dfw, u, c);
#endif
        if (__builtin_expect (dfw->released, 0))
            dfwmalloc_try_free_dfwmalloc (global, dfw, c);
    } else {
#ifdef WITH_BTANDCHECK
        dfwmalloc_user_alert (NULL, c, "caller trying to free block in a different arena that cannot be found", NULL);
#else
        abort ();
#endif
    }
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);
}

#endif

#ifndef NO_CB_HEAPS
STATICFNINLINE void dfwmalloc_cb_heaps_free_ (const struct dfwmalloc *const f_, void *u, struct calltrace *c)
{
#ifdef USE_TLS
    struct dfwmalloc *dfw;
    dfw = dfwmalloc_find_pointer (global, u);
    if (!dfw) {
#ifdef WITH_BTANDCHECK
        dfwmalloc_user_alert (NULL, c, "caller trying to free small block that cannot be found", NULL);
#else
        abort ();
#endif
    }
    dfwmalloc_cb_free (&dfw->cb_heaps, u);
    if (dfw != f_) {
        dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
        if (__builtin_expect (dfw->released, 0))
            dfwmalloc_try_free_dfwmalloc (global, dfw, c);
        dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);
    }
#else
    (void) c;
    (void) f_;
    dfwmalloc_mutex_lock (&dfwmalloc_tls->cb_heaps.mutex);
    dfwmalloc_cb_free (&dfwmalloc_tls->cb_heaps, u);
    dfwmalloc_mutex_unlock (&dfwmalloc_tls->cb_heaps.mutex);
#endif
}
#endif

#if defined(USE_TLS) && !defined(NO_INTERNAL_EXPOSE)
struct dfwmalloc *dfwmalloc_new (void)
{
    struct dfwmalloc *dfw;
    struct calltrace c;

    INIT (dfw, c, return NULL);   /* <== ensure that arena 0 is allocated -- we never want to loose arena 0 because its heap holds bucket sizes */

    dfwmalloc_backtrace_if_enabled (&c);

    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
    dfw = dfwmalloc_get_dfwmalloc (global, &c);
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);

    return dfw;
}

/* returns non-zero if there are leaks */
int dfwmalloc_destroy (struct dfwmalloc *dfw)
{
    int r;
    struct calltrace c;

    dfwmalloc_backtrace_if_enabled (&c);

    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
    r = dfwmalloc_try_free_dfwmalloc (global, dfw, &c);
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);

    return r;
}
#endif

STATICFNINLINE void *malloc_dfwmalloc (size_t n)
{
    struct calltrace c;
    struct dfwmalloc *dfw;
    if (!n)
        return MALLOC_ZERO;
    INIT_SELDOMSHRDFREE (dfw, c, return NULL);
    return dfwmalloc_ (dfw, (malloc_size_t) n, NO_ALIGNMENT, NULL, &c);
}

#if (defined(WANT_PRINTFSIMPLE) && defined(WITH_UNITTESTS)) || (defined(HAVE_PTHREAD) && defined(USE_TLS) && !defined(NO_PTHREAD_EXIT_OVERRIDE)) || (defined(PRECOMPILE_STATIC_DATA))
/* non-inline malloc used within this library */
STATICFNNOINLINE void *malloc_internal (size_t n)
{
    return malloc_dfwmalloc (n);
}
#endif

/* Note that malloc,free,realloc,calloc are not used within this library: internal functions
 * are used instead. This is to avoid possible optimization of these functions. */
void *malloc (size_t n)
{
    return malloc_dfwmalloc (n);
}

#if defined(USE_TLS) && !defined(NO_INTERNAL_EXPOSE)
void *dfwmalloc_malloc (struct dfwmalloc *dfw, size_t n)
{
    struct calltrace c;

    INITF (dfw, c, return NULL);

    dfwmalloc_assert (n > 0);

    return dfwmalloc_ (dfw, (malloc_size_t) n, NO_ALIGNMENT, NULL, &c);
}
#endif

STATICFNINLINE void dfwmalloc_free_ (struct dfwmalloc *dfw, void *u, struct calltrace *c)
{
#ifdef USE_TLS
    if (__builtin_expect (!dfw || ARENA_FROM_POINTER (u) != dfw->arena_id, 0)) {
        dfwfree_shared_pointer (dfw, u, c);
        return;
    }
#endif
#ifdef NO_CB_HEAPS
    dfwmalloc_assert ((char *) u >= dfw->heap->start_of_heap);
    dfwmalloc_assert ((char *) u < dfw->heap->end_of_heap);
#endif
    dfwfree (dfw, u, c);
}

STATICFN LONG dfwmalloc_check_ (struct dfwmalloc *dfw, void *u, struct calltrace *c)
{
#ifdef USE_TLS
    if (ARENA_FROM_POINTER (u) != dfw->arena_id)
        return -1;
#endif
#ifdef NO_CB_HEAPS
    dfwmalloc_assert ((char *) u >= dfw->heap->start_of_heap);
    dfwmalloc_assert ((char *) u < dfw->heap->end_of_heap);
#endif
    return dfwcheck (dfw, u, c);
}

STATICFNINLINE void free_dfwmalloc (void *u)
{
    struct calltrace c;
    struct dfwmalloc *dfw = NULL;
    if (!u)
        return;
    if (u == MALLOC_ZERO)
        return;
    INIT_NOCHK (dfw, c, return);
    dfwmalloc_free_ (dfw, u, &c);
}

/* non-inline free used within this library */
STATICFNNOINLINE void free_internal (void *u)
{
    free_dfwmalloc (u);
}

/* Note that malloc,free,realloc,calloc are not used within this library: internal functions
 * are used instead. This is to avoid possible optimization of these functions. */
void free (void *u)
{
    free_dfwmalloc (u);
}

#if defined(USE_TLS) && !defined(NO_INTERNAL_EXPOSE)
void dfwmalloc_free (struct dfwmalloc *dfw, void *u)
{
    struct calltrace c;

    dfwmalloc_assert (u != NULL);
    dfwmalloc_assert (u != MALLOC_ZERO);

    INITF (dfw, c, return);

    dfwmalloc_free_ (dfw, u, &c);
}
#endif

LONG dfwmalloc_check (void *u)
{
    struct calltrace c;
    struct dfwmalloc *dfw;

    if (!u)
        return 0;

    if (u == MALLOC_ZERO)
        return 0;

    INIT (dfw, c, return -1L);

    return dfwmalloc_check_ (dfw, u, &c);
}

void dfwmalloc_free_foreign (void)
{
    struct calltrace c;
    struct dfwmalloc *dfw;

    INIT (dfw, c, return);
    (void) dfw;
}

#if defined(USE_TLS) && !defined(NO_INTERNAL_EXPOSE)
LONG dfwmalloc_dfwcheck (struct dfwmalloc *dfw, void *u)
{
    struct calltrace c;

    dfwmalloc_assert (u != NULL);
    dfwmalloc_assert (u != MALLOC_ZERO);

    INITF (dfw, c, return -1L);

    return dfwmalloc_check_ (dfw, u, &c);
}
#endif

/* In theory this should check released and then do dfwmalloc_try_free_dfwmalloc */
STATICFNINLINE void *dfwmalloc_realloc_ (struct dfwmalloc *dfw, void *u, malloc_size_t n, struct calltrace *c)
{
    if (!u || u == MALLOC_ZERO)
        return dfwmalloc_ (dfw, n, NO_ALIGNMENT, NULL, c);

#ifndef NO_CB_HEAPS
    if (CB_HEAP_CHECK (u)) {
        void *r;
        malloc_size_t size;
        size = dfwmalloc_cb_get_size (u);
        r = dfwmalloc_ (dfw, n, NO_ALIGNMENT, NULL, c);
        if (r) {
            memcpy (r, u, size < n ? size : n);
#ifdef USE_TLS
            if (ARENA_FROM_POINTER (u) != dfw->arena_id) {
                dfwfree_shared_pointer (dfw, u, c);
                return r;
            }
#endif
            dfwmalloc_cb_heaps_free_ (dfw, u, c);
        }
        return r;
    }
#endif

#ifdef USE_TLS
    if (ARENA_FROM_POINTER (u) != dfw->arena_id) {
        void *r;
        char *s;
        struct free_item *p;

        s = (char *) u;
        p = (struct free_item *) (s - HEAD_SIZE);
#ifdef WITH_BTANDCHECK
        if (s[p->len] == '}' || s[p->len] == '@')
            dfwmalloc_user_alert (NULL, c, "caller trying to realloc already-freed block from another arena", p);
#endif
        r = dfwmalloc_ (dfw, n, NO_ALIGNMENT, NULL, c);
        if (r) {
            memcpy (r, u, p->len < n ? p->len : n);
            dfwfree_shared_pointer (dfw, u, c);
        }
        return r;
    }

    return dfwrealloc (dfw, u, n, c);
#else
    dfwmalloc_assert ((char *) u >= dfw->heap->start_of_heap);
    dfwmalloc_assert ((char *) u < dfw->heap->end_of_heap);

    return dfwrealloc (dfw, u, n, c);
#endif
}

#if defined(USE_TLS) && !defined(NO_INTERNAL_EXPOSE)
void *dfwmalloc_realloc (struct dfwmalloc *dfw, void *u, size_t n)
{
    struct calltrace c;

    INITF (dfw, c, return NULL);

    dfwmalloc_assert (n > 0);

    return dfwmalloc_realloc_ (dfw, u, n, &c);
}
#endif

/* Note that malloc,free,realloc,calloc are not used within this library: internal functions
 * are used instead. This is to avoid possible optimization of these functions. */
void *realloc (void *u, size_t n)
{
    struct calltrace c;
    struct dfwmalloc *dfw;

    if (!n) {
        free_internal (u);
        return MALLOC_ZERO;
    }

    INIT (dfw, c, return NULL);

    return dfwmalloc_realloc_ (dfw, u, (malloc_size_t) n, &c);
}

/* Note that malloc,free,realloc,calloc are not used within this library: internal functions
 * are used instead. This is to avoid possible optimization of these functions. */
void *calloc (size_t nmemb, size_t size)
{
    struct calltrace c;
    size_t n;
    struct dfwmalloc *dfw;
    void *r;
    int need_zero;

    if (!size || !nmemb)
        return MALLOC_ZERO;

    /* glibc just allocates nmemb * size */
    n = nmemb * size;
    if (n / size != nmemb) {
        errno = ENOMEM;
        return NULL;
    }

    INIT (dfw, c, return NULL);

/* FIXME: this should call all the way through to mmap() with
 * MAP_UNINITIALIZED off so that the memory is zero. */
    need_zero = 1;
    r = dfwmalloc_ (dfw, (malloc_size_t) n, (1 << WANTED_ALIGNMENT), &need_zero, &c);
    if (r && need_zero)
        memset (r, '\0', n);
    return r;
}

#ifndef NO_HEAP_TRUNCATION
STATICFN void dfwmalloc_truncate_heap_ (struct dfwmalloc *dfw, struct calltrace *c)
{
    dfw->flush ^= HEAP_FLAG_FLUSH;
#ifndef USE_TLS
    dfwmalloc_mutex_lock (&dfw->heap->heap_mutex);
#endif
    dfwmalloc_truncate_heap_locked (dfw, &dfw->flush, c);
#ifndef USE_TLS
    dfwmalloc_mutex_unlock (&dfw->heap->heap_mutex);
#endif

#ifndef NO_CB_HEAPS
#ifndef USE_TLS
    dfwmalloc_mutex_lock (&dfw->cb_heaps.mutex);
#endif
    dfwmalloc_cb_truncate (&dfw->cb_heaps);
#ifndef USE_TLS
    dfwmalloc_mutex_unlock (&dfw->cb_heaps.mutex);
#endif
#endif
}
#endif

STATICFN LONG dfwmalloc_efficiency_ (struct dfwmalloc *dfw, int arena)
{
    LONG r = -1L;
    (void) dfw;
    (void) arena;
#ifndef NO_CB_HEAPS
#ifndef USE_TLS
    dfwmalloc_mutex_lock (&dfw->cb_heaps.mutex);
#endif
    r = dfwmalloc_cb_efficiency (&dfw->cb_heaps, arena);
#ifndef USE_TLS
    dfwmalloc_mutex_unlock (&dfw->cb_heaps.mutex);
#endif
#endif
    return r;
}

#if defined(USE_TLS) && !defined(NO_INTERNAL_EXPOSE)
void dfwmalloc_dfwtruncate_heap (struct dfwmalloc *dfw)
{
#ifndef NO_HEAP_TRUNCATION
    struct calltrace c;

    INITF (dfw, c, return);

    dfwmalloc_truncate_heap_ (dfw, &c);
#endif
    (void) dfw;
}

LONG dfwmalloc_dfwefficiency (struct dfwmalloc *dfw)
{
    const char *head;
    struct calltrace c;
    (void) c;

    INITF (dfw, c, return -1L);

    head = "output version 1";
    DFWMALLOC_LOG (head, strlen (head));

    head = "arena heap cb size n used total";
    DFWMALLOC_LOG (head, strlen (head));

    return dfwmalloc_efficiency_ (dfw, -1);
}
#endif

void dfwmalloc_truncate_heap (void)
{
#ifndef NO_HEAP_TRUNCATION
    struct calltrace c;
    struct dfwmalloc *dfw;

    INIT (dfw, c, return);

    dfwmalloc_truncate_heap_ (dfw, &c);
#endif
}

LONG dfwmalloc_efficiency (void)
{
    const char *head;
    LONG total = 0L;
    struct dfwmalloc *dfw;
#ifdef USE_TLS
    int t;
#endif

    head = "output version 1";
    DFWMALLOC_LOG (head, strlen (head));

    head = "arena heap cb size n used total";
    DFWMALLOC_LOG (head, strlen (head));
#ifdef USE_TLS
    for (t = 0; t < global->n_arenas; t++) {
        if (global->arena[t].occupied != OCCUPIED_USED)
            continue;
        dfw = global->arena[t].dfwmalloc;
        total += dfwmalloc_efficiency_ (dfw, t);
#else
        dfw = global->arena.dfwmalloc;
        total += dfwmalloc_efficiency_ (dfw, -1);
#endif
#ifdef USE_TLS
    }
#endif
    return total;
}

void dfwmalloc_verify (void)
{
    struct dfwmalloc *dfw;
    int k;
#ifdef USE_TLS
    int t;
    for (t = 0; t < global->n_arenas; t++) {
        if (global->arena[t].occupied != OCCUPIED_USED)
            continue;
        dfw = global->arena[t].dfwmalloc;
#else
        dfw = global->arena.dfwmalloc;
#endif
        for (k = 1; k < global->n_freelists; k++) {
            struct free_list *l;
            struct free_item *p;
            l = &dfw->free_list[k];
            if ((p = l->fl_first)) {
                do {
#if 0
/* do checks here */
                    dfwmalloc_assert (p-> ...);
#endif
                    p = GETP (dfw, p->fl_next);
                } while (p);
            }
        }
#ifdef USE_TLS
    }
#endif
}

const char *dfwmalloc_version_str = DFWMALLOC_VERSION_STR;

LONG dfwmalloc_version (void)
{
    return DFWMALLOC_VERSION;
}

/* less common functions: */
int malloc_trim (size_t pad)
{
    (void) pad;
    dfwmalloc_truncate_heap ();
    return 1;
}

STATICFNINLINE void *dfwmalloc_memalign (size_t alignment, size_t n_)
{
    struct calltrace c;
    malloc_size_t n = (malloc_size_t) n_;
    struct dfwmalloc *dfw;

    /* power of two test: */
    if (((((alignment - 1) ^ alignment) + 1) >> 1) != alignment) {
        errno = EINVAL;
        return NULL;
    }

    if (!n)
        return MALLOC_ZERO;

    INIT (dfw, c, return NULL);

    return dfwmalloc_ (dfw, n, (int) alignment, NULL, &c);
}

void *memalign (size_t alignment, size_t n)
{
    return dfwmalloc_memalign (alignment, n);
}

int posix_memalign (void **r, size_t alignment, size_t n)
{
    void *u;
    if ((alignment % POINTER_ALIGNMENT))
        return EINVAL;
    u = dfwmalloc_memalign (alignment, n);
    if (!u)
        return errno;
    *r = u;
    return 0;
}

void malloc_stats (void)
{
    struct dfwmalloc_stats st;
    int r = 0;
    char s[256];

    memset (&st, '\0', sizeof (st));

    dfwmalloc_stats (&st);

#define PRTFLD(x,m) \
    do { \
        s[0] = '\0'; \
        strlimcat (s, sizeof (s), #x ": "); \
        strlimcat (s, sizeof (s), itoa_base (st.x, 1, 10)); \
        strlimcat (s, sizeof (s), "  ("); \
        strlimcat (s, sizeof (s), m); \
        strlimcat (s, sizeof (s), ")\n"); \
        r += write (2, s, strlen (s)); \
    } while(0)

#define PRTINT(msg1,i,msg2) \
    do { \
        s[0] = '\0'; \
        strlimcat (s, sizeof (s), msg1); \
        strlimcat (s, sizeof (s), itoa_base (i, 1, 10)); \
        strlimcat (s, sizeof (s), msg2); \
        r += write (2, s, strlen (s)); \
    } while(0)

    PRTFLD (alloced, "bytes in used large blocks");
    PRTFLD (blocks, "number of used large blocks");
    PRTFLD (alloced_bt, "bytes in used blocks with backtraces");
    PRTFLD (blocks_bt, "number of used blocks with backtraces");
    PRTFLD (alloced_small, "bytes in used small blocks");
    PRTFLD (blocks_small, "number of used small blocks");
    PRTFLD (heap, "heap size for large blocks");
    PRTFLD (heap_small, "heap size for small blocks");
    PRTFLD (free_blocks, "number of large free blocks");
    PRTFLD (free_blocks_small, "number of small free blocks");
    PRTFLD (max_heap, "max heap size for large blocks");
    PRTFLD (max_small_area, "max heap size for each area of small blocks");
    PRTFLD (max_cb_size, "maximum size of block that will be handled by control blocks arenas");

    r += write (2, "\n", 1);
    PRTINT ("The size of the per-thread support structure is ", sizeof (struct dfwmalloc), " bytes.\n");
    r += write (2, "The total number of allocated bytes in all blocks is:  alloced + alloced_small + alloced_bt.\n", 93);
    r += write (2, "The total number of allocated blocks is:  blocks + blocks_small + blocks_bt.\n", 77);
    r += write (2, "The total number of free blocks is:  free_blocks + free_blocks_small.\n", 70);
    r += write (2, "The total heap size is calculated from the end-most pointer to a used or free block, but the number of bytes in mapped pages may be very slightly larger if a truncate is pending.\n", 179);
    r += write (2, "With backtracing enabled small blocks are not used.\n", 52);
    (void) r;
}

#ifndef __FreeBSD__
struct mallinfo mallinfo (void)
{
    struct mallinfo r;
    memset (&r, '\0', sizeof (r));
    r.uordblks = dfwmalloc_alloced_bytes ();
    return r;
}
#endif

int mallopt(int param, int value)
{
    (void) param;
    (void) value;
    return 1;   /* 1 means Success */
}

size_t malloc_usable_size (void *u)
{
    LONG n;
    if (!u || u == MALLOC_ZERO)
        return 0;
    n = dfwmalloc_check (u);
    if (n < 0)
        return 0;
    return (size_t) n;
}

#if 0
char *strdup (const char *s)
{
    char *r;
    int l;
    l = strlen (s);
    r = (char *) malloc_internal (l + 1);
    memcpy (r, s, l + 1);
    return r;
}
#endif

/* obsolete functions: */
void cfree (void *u)
{
    free_internal (u);
}

/* 
 * 
 * TODO: possibly implement:
 *  __libc_calloc
 *  __libc_free
 *  __libc_malloc
 *  __libc_memalign
 *  __libc_realloc
 *  __libc_valloc
 * 
 */

void *aligned_alloc (size_t alignment, size_t size)
{
    return dfwmalloc_memalign (alignment, size);
}

void *valloc (size_t size)
{
    return dfwmalloc_memalign (MMAP_PAGE_SIZE_SMALL, size);
}

void *pvalloc (size_t size)
{
    return dfwmalloc_memalign (MMAP_PAGE_SIZE_SMALL, size + ALIGN_ADJUST (size, MMAP_PAGE_SIZE_SMALL - 1, 0));
}

/* for libc < 2.25 */
void *__libc_memalign (size_t alignment, size_t n)
{
    return dfwmalloc_memalign (alignment, n);
}


/* C++ mangled names: new operators */

/* void *operator new (size_t size); */
void *_Znwm (size_t n)
{
    void *u;
    u = malloc_dfwmalloc (n);
    if (!u)
        dfwmalloc_do_throw ();
    return u;
}

/* void *operator new[] (size_t size); */
void *_Znam (size_t n)
{
    void *u;
    u = malloc_dfwmalloc (n);
    if (!u)
        dfwmalloc_do_throw ();
    return u;
}

/* void *operator new (size_t size, std::align_val_t align); */
void *_ZnwmSt11align_val_t (size_t size, int align)
{
    void *u;
    u = dfwmalloc_memalign (align, size);
    if (!u)
        dfwmalloc_do_throw ();
    return u;
}

/* void *operator new[] (size_t size, std::align_val_t align); */
void *_ZnamSt11align_val_t (size_t size, int align)
{
    void *u;
    u = dfwmalloc_memalign (align, size);
    if (!u)
        dfwmalloc_do_throw ();
    return u;
}


/* void *operator new (size_t size, std::nothrow_t const &); */
void *_ZnwmRKSt9nothrow_t (size_t size)
{
    return malloc_dfwmalloc (size);
}

/* void *operator new[] (size_t size, std::nothrow_t const &); */
void *_ZnamRKSt9nothrow_t (size_t size)
{
    return malloc_dfwmalloc (size);
}

/* void *operator new (size_t size, std::align_val_t align, std::nothrow_t const &); */
void *_ZnwmSt11align_val_tRKSt9nothrow_t (size_t size, int align)
{
    return dfwmalloc_memalign (align, size);
}

/* void *operator new[] (size_t size, std::align_val_t align, std::nothrow_t const &); */
void *_ZnamSt11align_val_tRKSt9nothrow_t (size_t size, int align)
{
    return dfwmalloc_memalign (align, size);
}


/* C++ mangled names: delete operators */

/* void operator delete(void*); */
void _ZdlPv (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete[](void*); */
void _ZdaPv (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete(void*, std::size_t); */
void _ZdlPvm (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete[](void*, std::size_t); */
void _ZdaPvm (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete(void*, const std::nothrow_t&); */
void _ZdlPvRKSt9nothrow_t (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete[](void*, const std::nothrow_t&); */
void _ZdaPvRKSt9nothrow_t (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete(void*, std::align_val_t); */
void _ZdlPvSt11align_val_t (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete(void*, std::align_val_t, const std::nothrow_t&); */
void _ZdlPvSt11align_val_tRKSt9nothrow_t (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete[](void*, std::align_val_t); */
void _ZdaPvSt11align_val_t (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete[](void*, std::align_val_t, const std::nothrow_t&); */
void _ZdaPvSt11align_val_tRKSt9nothrow_t (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete(void*, std::size_t, std::align_val_t); */
void _ZdlPvmSt11align_val_t (void *u)
{
    free_dfwmalloc (u);
}

/* void operator delete[](void*, std::size_t, std::align_val_t); */
void _ZdaPvmSt11align_val_t (void *u)
{
    free_dfwmalloc (u);
}




STATICFN void dfwmalloc_stats_ (struct dfwmalloc_stats *s, struct dfwmalloc *dfw)
{
#ifdef WITH_ACCOUNTS
    int j;
    LONG n;
#endif

#ifdef WITH_ACCOUNTS
    s->alloced += dfw->total_alloced;
#ifdef WITH_BTANDCHECK
    s->alloced_bt += dfw->total_alloced_bt;
#endif
#endif

#if defined(WITH_ACCOUNTS) || defined(USE_TLS)
    s->blocks += dfw->total_blocks;
#ifdef WITH_BTANDCHECK
    s->blocks_bt += dfw->total_blocks_bt;
#endif
#endif

#if !defined(NO_CB_HEAPS) && defined(WITH_ACCOUNTS)
    s->free_blocks_small += dfw->cb_heaps.total_free;
    s->blocks_small += dfw->cb_heaps.cb_total_blocks;
    s->alloced_small += dfw->cb_heaps.total_alloced;
#endif
#ifndef NO_CB_HEAPS
    s->heap_small += dfw->cb_heaps.total_heap;
#endif

    s->heap += (LONG) (dfw->heap->end_of_heap - dfw->heap->start_of_heap);

#ifdef WITH_ACCOUNTS
    for (n = 0L, j = 1; j < dfw->free_list_initialized; j++)
        n += dfw->free_list[j].count;

    s->free_blocks += n;
#endif
}

#if defined(USE_TLS) && !defined(NO_INTERNAL_EXPOSE)
void dfwmalloc_dfwstats (struct dfwmalloc *dfw, struct dfwmalloc_stats *s)
{
    memset (s, '\0', sizeof (*s));
    dfwmalloc_stats_ (s, dfw);

#ifndef NO_CB_HEAPS
    s->max_small_area = global->max_cb_heap;
#endif
    s->max_heap = global->max_heap;
#ifndef NO_CB_HEAPS
    s->max_cb_size = CB_N_SIZES;
#endif
}
#endif

void dfwmalloc_stats (struct dfwmalloc_stats *s)
{
    int i;
    memset (s, '\0', sizeof (*s));
    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
#ifdef USE_TLS
    for (i = 0; i < global->n_arenas; i++) {
        if (global->arena[i].occupied == OCCUPIED_USED) {
            dfwmalloc_stats_ (s, global->arena[i].dfwmalloc);
        }
    }
#else
    (void) i;
    dfwmalloc_stats_ (s, global->arena.dfwmalloc);
#endif
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);

#ifndef NO_CB_HEAPS
    s->max_small_area = global->max_cb_heap;
#endif
    s->max_heap = global->max_heap;
#ifndef NO_CB_HEAPS
    s->max_cb_size = CB_N_SIZES;
#endif
}

LONG dfwmalloc_heap_size (void)
{
    struct dfwmalloc *dfw;
    int i;
    LONG r;
    (void) dfw;
    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
#ifdef USE_TLS
    r = 0L;
    for (i = 0; i < global->n_arenas; i++) {
        if (global->arena[i].occupied == OCCUPIED_USED) {
            dfw = global->arena[i].dfwmalloc;
            r += (LONG) (dfw->heap->end_of_heap - dfw->heap->start_of_heap);
#ifndef NO_CB_HEAPS
            r += dfw->cb_heaps.total_heap;
#endif
        }
    }
#else
    (void) i;
    dfw = global->arena.dfwmalloc;
    r = (long) (dfw->heap->end_of_heap - dfw->heap->start_of_heap);
#ifndef NO_CB_HEAPS
    r += dfw->cb_heaps.total_heap;
#endif
#endif
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);
    return r;
}

LONG dfwmalloc_alloced_bytes (void)
{
#ifdef WITH_ACCOUNTS
    struct dfwmalloc *dfw;
    int i;
    LONG r;
    (void) dfw;
    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
    r = 0L;
#ifdef USE_TLS
    for (i = 0; i < global->n_arenas; i++) {
        if (global->arena[i].occupied == OCCUPIED_USED) {
            dfw = global->arena[i].dfwmalloc;
            r += dfw->total_alloced;
#ifdef WITH_BTANDCHECK
            r += dfw->total_alloced_bt;
#endif
#if !defined(NO_CB_HEAPS) && defined(WITH_ACCOUNTS)
            r += dfw->cb_heaps.total_alloced;
#endif
        }
    }
#else
    (void) i;
    dfw = global->arena.dfwmalloc;
    r += dfw->total_alloced;
#ifdef WITH_BTANDCHECK
    r += dfw->total_alloced_bt;
#endif
#if !defined(NO_CB_HEAPS) && defined(WITH_ACCOUNTS)
    r += dfw->cb_heaps.total_alloced;
#endif
#endif
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);
    return r;
#else
    return 0L;
#endif
}

void dfwmalloc_set_lfv_enablement (int enabled)
{
#if !defined(NO_LOCKFREEVISITOR) && defined(USE_TLS)
    struct dfwmalloc *dfw;
    struct calltrace c;

    INIT (dfw, c, return);

    global->lfv_enabled = enabled;
#else
    (void) enabled;
#endif
}

#ifdef WITH_UNITTESTS

int __dfwmalloc_get_n_arenas (void)
{
#ifdef USE_TLS
    return global_.n_arenas;
#else
    return 1;
#endif
}

int __dfwmalloc_get_max_arenas (void)
{
#ifdef USE_TLS
    return N_ARENAS;
#else
    return 1;
#endif
}

#endif

#ifdef WITH_UNITTESTS
#ifndef NO_CB_HEAPS

long __dfwmalloc_cb_refs (struct dfwmalloc *dfw)
{
    return (long) dfw->cb_heaps.cb_refs;
}

long __dfwmalloc_cb_heap_index_to_chunk_size (int i)
{
    return (long) HEAP_INDEX_TO_CHUNKSIZE (i);
}

long __dfwmalloc_cb_heap_index_from_pointer (void *u)
{
    return (long) CB_HEAP_GET_INDEX (u);
}

long __dfwmalloc_cb_max_head (void)
{
    return (long) global->cb_heaps_info.max_head;
}

struct cb_page_info __dfwmalloc_cb_page_info (int c)
{
    dfwmalloc_assert (c <= global->cb_heaps_info.max_head);
    return global->cb_heaps_info.page_info[c];
}

long __dfwmalloc_cb_head_index (int c)
{
    dfwmalloc_assert (c <= CB_N_SIZES);
    return global->cb_heaps_info.head_index[c];
}

#endif
#endif

LONG dfwmalloc_alloced_blocks (void)
{
#ifdef WITH_ACCOUNTS
    struct dfwmalloc *dfw;
    int i;
    LONG r;
    (void) dfw;
    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
    r = 0L;
#ifdef USE_TLS
    for (i = 0; i < global->n_arenas; i++) {
        if (global->arena[i].occupied == OCCUPIED_USED) {
            dfw = global->arena[i].dfwmalloc;
            r += dfw->total_blocks;
#ifdef WITH_BTANDCHECK
            r += dfw->total_blocks_bt;
#endif
#if !defined(NO_CB_HEAPS) && defined(WITH_ACCOUNTS)
            r += dfw->cb_heaps.cb_total_blocks;
#endif
        }
    }
#else
    (void) i;
    dfw = global->arena.dfwmalloc;
    r += dfw->total_blocks;
#ifdef WITH_BTANDCHECK
    r += dfw->total_blocks_bt;
#endif
#if !defined(NO_CB_HEAPS) && defined(WITH_ACCOUNTS)
    r += dfw->cb_heaps.cb_total_blocks;
#endif
#endif
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);
    return r;
#else
    return 0L;
#endif
}

#ifdef WITH_BTANDCHECK
STATICFN void dfwmalloc_alloc_backtrace_head (struct dfwmalloc *dfw)
{
    dfw->backtrace.backtrace_head = (struct backtrace_head *) dfwmalloc_raw_calloc (dfw, BACKTRACE_HEADS * sizeof (dfw->backtrace.backtrace_head[0]));
    dfw->backtrace.backtrace_array = (struct backtrace_item ***) dfwmalloc_raw_calloc (dfw, sizeof (struct backtrace_item **) * ((1L << BACKTRACE_ID_HIGH) + 1L));
}
#endif

#ifdef WITH_BTANDCHECK
static struct dfwmalloc_stats st_adjust;
#endif

#ifdef WITH_UNITTESTS

void __dfwmalloc_get_stat_adjust (struct dfwmalloc_stats *s)
{
#ifdef WITH_BTANDCHECK
    *s = st_adjust;
#else
    memset (s, '\0', sizeof (*s));
#endif
}

void __dfwmalloc_print_vitals (void)
{
#ifdef WANT_PRINTFSIMPLE
    int k, r = 1;
    struct dfwmalloc  *dfw;
    char *p;
    malloc_size_t n;
    int u;
    struct free_list *l;

    printf_simple ("\nVital Statistics ==>\n\n");

    p = (char *) malloc_internal (1);
    (void) p;

    dfw = dfwmalloc_tls;

#ifdef USE_PRECOMPILED_STATIC_DATA
    printf_simple ("using precompiled data : global_free_list_sizes = %ld\n", (long) n_global_free_list_sizes);
#endif

    printf_simple ("max_heap = %ld (%ld MB)\n", (long) global_.max_heap, (long) global_.max_heap >> 20);
#ifndef NO_CB_HEAPS
    printf_simple ("max_cb_heap = %ld (%ld MB)\n", (long) global_.max_cb_heap, (long) global_.max_cb_heap >> 20);
#ifdef USE_TLS
    printf_simple ("HEAD_SIZE = %d, max_head = %d, CB_N_SIZES = %d, CB_SIZE_MAX = %d, SIZEOF_CB_SIZE = %d\n", (int) HEAD_SIZE,
            global->cb_heaps_info.max_head, (int) CB_N_SIZES, (int) CB_SIZE_MAX, (int) SIZEOF_CB_SIZE);
#endif
    printf_simple ("sizeof(struct control_block) = %d\n", (int) sizeof (struct control_block));
    {
        int y;
         y = 47 - N_ARENAS_BITS - CB_SIZE_BITS;
         printf_simple ("Size of each arena for CB allocs\n");
         printf_simple ("47 - N_ARENAS_BITS(%d) - CB_SIZE_BITS(%d) = %d (%d MBytes)\n", (int) N_ARENAS_BITS, (int) CB_SIZE_BITS, (int) y, (1 << (y - 20)));
    }
#endif

    for (k = 0; k < 5; k++) {
        struct free_list *l;
        l = &dfw->free_list[k];
        printf_simple ("bucket %d  => %ld\n", k, (long) BUCKET_SIZE (dfw, l));
    }
    printf_simple ("\n");
    for (k = global->n_freelists - 5; k < global->n_freelists; k++) {
        struct free_list *l;
        l = &dfw->free_list[k];
        printf_simple ("bucket %d  => %ld\n", k, (long) BUCKET_SIZE (dfw, l));
    }
    printf_simple ("\n");

    for (k = 0;; k++) {
        const char *s = " kMGTP";
        u = r >= global->n_freelists ? global->n_freelists - 1 : r;
        if (r < 10000)
            r *= 2;
        else {
            r = r * 256 / 220;
            r -= r % 1000;
        }
        l = &dfw->free_list[u];
        n = BUCKET_SIZE (dfw, l);
        while (n > 10240) {
            n /= 1024;
            s++;
        }
        printf_simple ("free bucket %d: %ld%c\n", u, (long) n, *s);
        if (u == global->n_freelists - 1)
            break;
    }
    u = global->n_freelists - 1;
    l = &dfw->free_list[u];
    n = BUCKET_SIZE (dfw, l);
    printf_simple ("last free bucket %d: %ld\n", u, (long) n);
#endif
}

#endif

void dfwmalloc_enable_backtrace (void)
{
#ifdef WITH_BTANDCHECK
    struct calltrace c;
    struct dfwmalloc *dfw;
    struct dfwmalloc_stats st;

/* the design of this is that you cannot turn it off once you have turned it on: */
    dfwmalloc_assert (!global->backtrace_enabled);

    INIT (dfw, c, return);

    memset (&st, '\0', sizeof (st));
    dfwmalloc_stats (&st);

    global->backtrace_enabled = 1;
    dfwmalloc_backtrace_if_enabled (&c);

    memset (&st_adjust, '\0', sizeof (st_adjust));
    dfwmalloc_stats (&st_adjust);

    st_adjust.alloced -= st.alloced;
    st_adjust.blocks -= st.blocks;
    st_adjust.alloced_small -= st.alloced_small;
    st_adjust.blocks_small -= st.blocks_small;

    dfwmalloc_alloc_backtrace_head (dfw);
#endif
}

STATICFN void dfwmalloc_dump_leaks_ (struct dfwmalloc_leak_info *info, int arena_id, struct dfwmalloc *dfw, void (*cb) (void *, struct dfwmalloc_leak *), void *hook)
{
#ifdef WITH_BTANDCHECK
    int i;

    (void) info;

    for (i = 1; i < dfw->backtrace.backtrace_id; i++) {
        struct backtrace_item **b_, *b;
        struct dfwmalloc_leak leak;
        LONG blocks;

        dfwmalloc_backtrace_array (dfw, &dfw->backtrace, i, &b_);
        b = *b_;

        if (ARENA_FROM_POINTER (b) != arena_id)         /* free's in another arena that would contain a block count of zero */
            continue;

        dfwmalloc_assert (b);

        blocks = b->bt_blocks;
#ifdef USE_TLS
        blocks += b->bt_blocks_otherarena;
#endif

        if (blocks) {
            memset (&leak, '\0', sizeof (leak));

            leak.arena_id = arena_id;
            leak.backtrace_id = i;
            leak.alloced = b->bt_alloced;
#ifdef USE_TLS
            leak.alloced += b->bt_alloced_otherarena;
#endif
            leak.blocks = blocks;
            leak.addr = &b->addr[0];
            leak.n = b->n;

            (*cb) (hook, &leak);
        }
    }
#else
    (void) info;
    (void) arena_id;
    (void) dfw;
    (void) cb;
    (void) hook;
#endif
}


#ifdef WITH_BTANDCHECK
STATICFN void leak_cb (void *dummy, struct dfwmalloc_leak *l)
{
    char buf[2048];
    int buflen, i;
    char *p;
    const char *t;

    (void) dummy;

    p = &buf[0];
    buflen = sizeof (buf);

#define adds(s) \
    do { \
        t = s; \
        strcpy(p, t); \
        buflen -= strlen(t); \
        p += strlen(t); \
    } while (0)

    adds ("leak: arena ");
    adds (itoa_base (l->arena_id, 0, 10));
    adds (", ");
    adds (itoa_base (l->alloced, 0, 10));
    adds (" bytes, ");
    adds (itoa_base (l->blocks, 0, 10));
    adds (" blocks  [ ");

    for (i = 0; i < l->n; i++) {
        adds ("0x");
        adds (itoa_base ((ULONG64) l->addr[i], 0, 16));
        adds (" ");
        if (buflen < 32) {
            adds ("... ");
            break;
        }
    }

    adds ("]");

    DFWMALLOC_LOG (buf, (int) (p - &buf[0]));
}
#endif

void dfwmalloc_dump_leaks (struct dfwmalloc_leak_info *info, void (*cb) (void *, struct dfwmalloc_leak *), void *hook)
{
    int i;
    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
#ifdef USE_TLS
    for (i = 0; i < global->n_arenas; i++) {
        if (global->arena[i].occupied == OCCUPIED_USED) {
            dfwmalloc_dump_leaks_ (info, i, global->arena[i].dfwmalloc, cb, hook);
        }
    }
#else
    (void) i;
    dfwmalloc_dump_leaks_ (info, 0, global->arena.dfwmalloc, cb, hook);
#endif
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);
}

void dfwmalloc_log_leaks (void)
{
#ifdef WITH_BTANDCHECK
    DFWMALLOC_LOG ("DFWMalloc leaks: ", 16);
    dfwmalloc_dump_leaks (NULL, leak_cb, NULL);
#endif
}

int dfwmalloc_dfwget_backtrace_id (struct dfwmalloc *dfw)
{
#ifdef WITH_BTANDCHECK
    return dfw->backtrace.backtrace_id;
#else
    (void) dfw;
    return 0;
#endif
}

int dfwmalloc_get_backtrace_id (void)
{
#ifdef WITH_BTANDCHECK
    int r;
    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
    if (dfwmalloc_tls)
        r = dfwmalloc_dfwget_backtrace_id (dfwmalloc_tls);
    else
        r = 0;
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);
    return r;
#else
    return 0;
#endif
}





#ifdef HAVE_PTHREAD
#ifdef USE_TLS

void dfwmalloc_pthread_exit (void)
{
    struct calltrace c;
    struct dfwmalloc *dfw;

    if (!dfwmalloc_tls)
        return;

    INIT_NOCHK (dfw, c, return);

    dfwmalloc_mutex_lock (&dfwmalloc_global_lock);
    if (!dfwmalloc_try_free_dfwmalloc (global, dfw, &c))
        dfwmalloc_tls = NULL;
    dfwmalloc_mutex_unlock (&dfwmalloc_global_lock);
}

#ifndef NO_PTHREAD_EXIT_OVERRIDE

typedef void *(*pthread_start_fn_t) (void *);
typedef int (*pthread_create_fn_t) (pthread_t *, pthread_attr_t const *, pthread_start_fn_t, void *);
typedef void (*pthread_exit_fn_t) (void *);
typedef int (*pthread_join_fn_t) (pthread_t, void **);

struct pthread_override_funcs {
    pthread_create_fn_t pt_cr;
    pthread_exit_fn_t pt_ex;
    pthread_join_fn_t pt_jn;
};

struct start_redirect {
    unsigned int start_redirect_magic;
#define START_REDIRECT_MAGIC    0xaf947d10
    pthread_start_fn_t original_start_fn;
    void *original_arg;
    void *exit_result;
    int detached_state;
    void *reference_alloc;
};

__thread struct start_redirect *start_redirect = NULL;
static struct pthread_override_funcs pthread_override_funcs;
static int pthread_override_funcs_initialized = 0;

STATICFN struct start_redirect *dfwmalloc_get_start_redirect (void)
{
    struct start_redirect *r;
    r = start_redirect;
    start_redirect = NULL;
    return r;
}

STATICFN void dfwmalloc_thread_cancel (void *arg)
{
    struct start_redirect *s;
    (void) arg;
    s = dfwmalloc_get_start_redirect ();
    dfwmalloc_assert (start_redirect == NULL);
    if (!s)
        return;
    dfwmalloc_assert (s->start_redirect_magic == START_REDIRECT_MAGIC);
    free_internal (s->reference_alloc);
    free_internal (s);
    dfwmalloc_pthread_exit ();
}

STATICFN void *dfwmalloc_redirect_start_fn (void *arg)
{
    struct start_redirect *s;
    start_redirect = (struct start_redirect *) arg;
    dfwmalloc_assert (start_redirect->start_redirect_magic == START_REDIRECT_MAGIC);
    start_redirect->reference_alloc = malloc_internal (1 << WANTED_ALIGNMENT);

    pthread_cleanup_push (dfwmalloc_thread_cancel, NULL);
    start_redirect->exit_result = (start_redirect->original_start_fn) (start_redirect->original_arg);
    pthread_cleanup_pop (0);
    s = dfwmalloc_get_start_redirect ();
    dfwmalloc_assert (start_redirect == NULL);
    if (s->detached_state == PTHREAD_CREATE_DETACHED) {
        free_internal (s->reference_alloc);
        free_internal (s);
    }
    dfwmalloc_pthread_exit ();
    return arg;
}

__attribute__ ((__visibility__ ("default"), noreturn))
void pthread_exit (void *u)
{
    struct start_redirect *s = NULL;
    s = dfwmalloc_get_start_redirect ();
    dfwmalloc_assert (start_redirect == NULL);
    dfwmalloc_assert (s != NULL);
    dfwmalloc_assert (s->start_redirect_magic == START_REDIRECT_MAGIC);
    if (s->detached_state == PTHREAD_CREATE_DETACHED) {
        free_internal (s->reference_alloc);
        free_internal (s);
        dfwmalloc_pthread_exit ();
        (*pthread_override_funcs.pt_ex) (NULL);
    } else {
        s->exit_result = u;
        dfwmalloc_pthread_exit ();
        (*pthread_override_funcs.pt_ex) ((void *) s);
    }
    abort ();
}

__attribute__ ((__visibility__ ("default")))
int pthread_join (pthread_t thread_id, void **retval)
{
    struct start_redirect *s;
    void *dummy_retval;
    void *retval_ = NULL;
    int r;
    if (!retval)
        retval = &dummy_retval;
    *retval = NULL;     /* defensive */
/* someone could try exit the thread neither by returning nor by calling pthread_exit(): */
    r = (*pthread_override_funcs.pt_jn) (thread_id, &retval_);
    if (r || !retval_ || retval_ == PTHREAD_CANCELED) {
        *retval = retval_;
        return r;
    }
    s = (struct start_redirect *) retval_;
    dfwmalloc_assert (s->start_redirect_magic == START_REDIRECT_MAGIC);
    *retval = s->exit_result;

/* since dfwmalloc_pthread_exit() would have already been called at this point
 * this free will drop the reference count to zero and then release the heap: */
    if (s->reference_alloc)
        free_internal (s->reference_alloc);
    free_internal (s);
    return 0;
}

STATICFN int dfwmalloc_pthread_override_initialize (void)
{
    if (!pthread_override_funcs_initialized) {
        pthread_override_funcs.pt_cr = (pthread_create_fn_t) dfwmalloc_dlsym ("pthread_create");
        if (!pthread_override_funcs.pt_cr)
            return 1;
        pthread_override_funcs.pt_ex = (pthread_exit_fn_t) dfwmalloc_dlsym ("pthread_exit");
        if (!pthread_override_funcs.pt_ex)
            return 1;
        pthread_override_funcs.pt_jn = (pthread_join_fn_t) dfwmalloc_dlsym ("pthread_join");
        if (!pthread_override_funcs.pt_jn)
            return 1;
        __sync_synchronize ();
        pthread_override_funcs_initialized = 1;
    }
    return 0;
}

__attribute__ ((__visibility__ ("default")))
int pthread_create (pthread_t * thread_id, pthread_attr_t const *attr, void *(*start_fn) (void *), void *arg)
{
    struct start_redirect *s;
    if (dfwmalloc_pthread_override_initialize ())
        return ENOENT;
    if (!(s = (struct start_redirect *) malloc_internal (sizeof (*s))))
        return ENOMEM;
    memset (s, '\0', sizeof (*s));
    s->start_redirect_magic = START_REDIRECT_MAGIC;
    s->original_start_fn = start_fn;
    s->original_arg = arg;
    s->detached_state = PTHREAD_CREATE_JOINABLE;
    if (attr)
        pthread_attr_getdetachstate (attr, &s->detached_state);
    return (*pthread_override_funcs.pt_cr) (thread_id, attr, dfwmalloc_redirect_start_fn, (void *) s);
}

#endif  /* !NO_PTHREAD_EXIT_OVERRIDE */
#endif
#endif




#ifdef __cplusplus
}
#endif



#ifdef PRECOMPILE_STATIC_DATA

#include <stdio.h>

static const int max_len = 100;

static void formatln (int *line_length, char *s, int comma)
{
    if (*line_length + (int) strlen (s) + comma == max_len) {
        *line_length = 4;
        printf ("%s%s\n    ", s, comma ? "," : "");
    } else if (*line_length + (int) strlen (s) + comma * 2 > max_len) {
        *line_length = 4 + strlen (s) + comma * 2;
        printf ("\n    %s%s", s, comma ? ", " : "");
    } else {
        *line_length += strlen (s) + comma * 2;
        printf ("%s%s", s, comma ? ", " : "");
    }
}

int main(int argc, char **argv)
{
    char s[256];
    int line_length = 0;
    int k;
    char *p, *q;
    (void) argc;
    (void) argv;
    p = (char *) malloc_internal (32);
    q = (char *) malloc_internal (1024 * 1024);
    (void) p;
    (void) q;

    printf ("\n");
    printf ("/* This file is autogenerated. Do not edit. */\n");
    printf ("\n");


    printf ("/* The following tests check that the generated and non-generated source were built with the same options: */\n");
    printf ("\n");

/* value macros */
#ifdef N_ARENAS_BITS
    printf ("#if !defined(N_ARENAS_BITS) || N_ARENAS_BITS != %d\n#error N_ARENAS_BITS should be defined as %d\n#endif\n\n", N_ARENAS_BITS, N_ARENAS_BITS);
#else
    printf ("#ifdef N_ARENAS_BITS\n#error N_ARENAS_BITS should not be defined\n#endif\n\n");
#endif
#ifdef MIN_ALIGN
    printf ("#if !defined(MIN_ALIGN) || MIN_ALIGN != %d\n#error MIN_ALIGN should be defined as %d\n#endif\n\n", MIN_ALIGN, MIN_ALIGN);
#else
    printf ("#ifdef MIN_ALIGN\n#error MIN_ALIGN should not be defined\n#endif\n\n");
#endif
#ifdef CB_SIZE_BITS
    printf ("#if !defined(CB_SIZE_BITS) || CB_SIZE_BITS != %d\n#error CB_SIZE_BITS should be defined as %d\n#endif\n\n", CB_SIZE_BITS, CB_SIZE_BITS);
#else
    printf ("#ifdef CB_SIZE_BITS\n#error CB_SIZE_BITS should not be defined\n#endif\n\n");
#endif
#ifdef CHUNK_POWER
    printf ("#if !defined(CHUNK_POWER) || CHUNK_POWER != %d\n#error CHUNK_POWER should be defined as %d\n#endif\n\n", CHUNK_POWER, CHUNK_POWER);
#else
    printf ("#ifdef CHUNK_POWER\n#error CHUNK_POWER should not be defined\n#endif\n\n");
#endif
#ifdef ESTIMATED_MIN_BLOCKS
    printf ("#if !defined(ESTIMATED_MIN_BLOCKS) || ESTIMATED_MIN_BLOCKS != %d\n#error ESTIMATED_MIN_BLOCKS should be defined as %d\n#endif\n\n", ESTIMATED_MIN_BLOCKS, ESTIMATED_MIN_BLOCKS);
#else
    printf ("#ifdef ESTIMATED_MIN_BLOCKS\n#error ESTIMATED_MIN_BLOCKS should not be defined\n#endif\n\n");
#endif
#ifdef ALLOCATION_SIZE_RATIO
    printf ("#if !defined(ALLOCATION_SIZE_RATIO) || ALLOCATION_SIZE_RATIO != %d\n#error ALLOCATION_SIZE_RATIO should be defined as %d\n#endif\n\n", ALLOCATION_SIZE_RATIO, ALLOCATION_SIZE_RATIO);
#else
    printf ("#ifdef ALLOCATION_SIZE_RATIO\n#error ALLOCATION_SIZE_RATIO should not be defined\n#endif\n\n");
#endif
#ifdef WANTED_ALIGNMENT
    printf ("#if !defined(WANTED_ALIGNMENT) || WANTED_ALIGNMENT != %d\n#error WANTED_ALIGNMENT should be defined as %d\n#endif\n\n", WANTED_ALIGNMENT, WANTED_ALIGNMENT);
#else
    printf ("#ifdef WANTED_ALIGNMENT\n#error WANTED_ALIGNMENT should not be defined\n#endif\n\n");
#endif
#ifdef USER_MAX_MALLOC
    printf ("#if !defined(USER_MAX_MALLOC) || USER_MAX_MALLOC != %d\n#error USER_MAX_MALLOC should be defined as %d\n#endif\n\n", USER_MAX_MALLOC, USER_MAX_MALLOC);
#else
    printf ("#ifdef USER_MAX_MALLOC\n#error USER_MAX_MALLOC should not be defined\n#endif\n\n");
#endif
#ifdef BUCKET_SMOOTHNESS
    printf ("#if !defined(BUCKET_SMOOTHNESS) || BUCKET_SMOOTHNESS != %d\n#error BUCKET_SMOOTHNESS should be defined as %d\n#endif\n\n", BUCKET_SMOOTHNESS, BUCKET_SMOOTHNESS);
#else
    printf ("#ifdef BUCKET_SMOOTHNESS\n#error BUCKET_SMOOTHNESS should not be defined\n#endif\n\n");
#endif

/* macro switches */
#ifdef WITH_BTANDCHECK
    printf ("#ifndef WITH_BTANDCHECK\n#error WITH_BTANDCHECK should be defined\n#endif\n\n");
#else
    printf ("#ifdef WITH_BTANDCHECK\n#error WITH_BTANDCHECK should not be defined\n#endif\n\n");
#endif
#ifdef WITH_ACCOUNTS
    printf ("#ifndef WITH_ACCOUNTS\n#error WITH_ACCOUNTS should be defined\n#endif\n\n");
#else
    printf ("#ifdef WITH_ACCOUNTS\n#error WITH_ACCOUNTS should not be defined\n#endif\n\n");
#endif
#ifdef HAVE_PTHREAD
    printf ("#ifndef HAVE_PTHREAD\n#error HAVE_PTHREAD should be defined\n#endif\n\n");
#else
    printf ("#ifdef HAVE_PTHREAD\n#error HAVE_PTHREAD should not be defined\n#endif\n\n");
#endif
#ifdef USE_TLS
    printf ("#ifndef USE_TLS\n#error USE_TLS should be defined\n#endif\n\n");
#else
    printf ("#ifdef USE_TLS\n#error USE_TLS should not be defined\n#endif\n\n");
#endif
/* 
 * #ifdef NO_ASSERT_CHECKS
 *     printf ("#ifndef NO_ASSERT_CHECKS\n#error NO_ASSERT_CHECKS should be defined\n#endif\n\n");
 * #else
 *     printf ("#ifdef NO_ASSERT_CHECKS\n#error NO_ASSERT_CHECKS should not be defined\n#endif\n\n");
 * #endif
 */
#ifdef NO_HEAP_TRUNCATION
    printf ("#ifndef NO_HEAP_TRUNCATION\n#error NO_HEAP_TRUNCATION should be defined\n#endif\n\n");
#else
    printf ("#ifdef NO_HEAP_TRUNCATION\n#error NO_HEAP_TRUNCATION should not be defined\n#endif\n\n");
#endif
#ifdef LIGHT
    printf ("#ifndef LIGHT\n#error LIGHT should be defined\n#endif\n\n");
#else
    printf ("#ifdef LIGHT\n#error LIGHT should not be defined\n#endif\n\n");
#endif
#ifdef FEATHERWEIGHT
    printf ("#ifndef FEATHERWEIGHT\n#error FEATHERWEIGHT should be defined\n#endif\n\n");
#else
    printf ("#ifdef FEATHERWEIGHT\n#error FEATHERWEIGHT should not be defined\n#endif\n\n");
#endif
#ifdef NO_CB_HEAPS
    printf ("#ifndef NO_CB_HEAPS\n#error NO_CB_HEAPS should be defined\n#endif\n\n");
#else
    printf ("#ifdef NO_CB_HEAPS\n#error NO_CB_HEAPS should not be defined\n#endif\n\n");
#endif
#ifdef COMPACT_MODE
    printf ("#ifndef COMPACT_MODE\n#error COMPACT_MODE should be defined\n#endif\n\n");
#else
    printf ("#ifdef COMPACT_MODE\n#error COMPACT_MODE should not be defined\n#endif\n\n");
#endif
#ifdef NO_BLOCK_ORDERING
    printf ("#ifndef NO_BLOCK_ORDERING\n#error NO_BLOCK_ORDERING should be defined\n#endif\n\n");
#else
    printf ("#ifdef NO_BLOCK_ORDERING\n#error NO_BLOCK_ORDERING should not be defined\n#endif\n\n");
#endif

    printf ("static const long n_global_free_list_sizes = %ldL;\n\n", (long) global->n_freelists);
    snprintf (s, sizeof(s), "static const malloc_size_t global_free_list_sizes[] = {");
    formatln (&line_length, s, 0);

    for (k = 0; k < global->n_freelists; k++) {
        snprintf (s, sizeof(s), "%ldL", (long) global->bucket_size[k]);
        formatln (&line_length, s, k < global->n_freelists - 1);
    }
    printf ("};\n");
    printf ("\n");

#ifndef NO_CB_HEAPS
    line_length = 0;
    snprintf (s, sizeof (s), "static const int global_max_heads = %d;", global->cb_heaps_info.max_head);
    formatln (&line_length, s, 0);
    printf ("\n\n");
    line_length = 0;
    for (k = 0; k < CB_SIZE_MAX; k++) {
        snprintf (s, sizeof (s), "%s%d: %ld bytes", k ? "" : "/* Heap index sizes ---  ", k, (long) HEAP_INDEX_TO_CHUNKSIZE (k));
        formatln (&line_length, s, k != CB_SIZE_MAX - 1);
    }
    formatln (&line_length, " */", 0);
    printf ("\n\n");
    line_length = 0;
    printf ("/* n_blocks, heap_index, mask_packing, first_block, round_up */\n");
    snprintf (s, sizeof (s), "static const struct cb_page_info global_page_info[%d] = {", global->cb_heaps_info.max_head + 1);
    formatln (&line_length, s, 0);

    for (k = 0; k < global->cb_heaps_info.max_head + 1; k++) {
        struct cb_page_info *i;
        i = &global->cb_heaps_info.page_info[k];
        if (k == 0)
            snprintf (s, sizeof (s), "{%d, %d, %d, %d, %d}", 0, 0, 0, 0, 0);
        else
            snprintf (s, sizeof (s), "{%d, %d, %d, %d, %d}", (int) i->n_blocks, (int) i->heap_index, (int) i->mask_packing, (int) i->first_block, (int) i->round_up);
        formatln (&line_length, s, k != global->cb_heaps_info.max_head);
    }
    printf ("\n};\n\n");

    printf ("/* CB_N_SIZES = %d */\n\n", CB_N_SIZES);
    line_length = 0;
    snprintf (s, sizeof (s), "static const cb_headint_t global_head_index[CB_N_SIZES + 1] = {");
    formatln (&line_length, s, 0);

    for (k = 0; k < CB_N_SIZES + 1; k++) {
        if (k == 0)
            snprintf (s, sizeof (s), "%d", 0);
        else
            snprintf (s, sizeof (s), "%d", (int) global->cb_heaps_info.head_index[k]);
        formatln (&line_length, s, k < CB_N_SIZES);
    }

    printf ("\n};\n\n");
#endif

    exit (0);
}

#endif



/* gcc seems to ignore -fno-inline, so do this to trick the optimizer: */
#ifdef DISABLE_INLINE

struct dfwmalloc_noopt {
#ifdef WITH_BTANDCHECK
    backtrace_id_t (*dfwmalloc_backtrace) (struct dfwmalloc *dfw, struct backtrace *g, struct calltrace *c);
#endif
    char *(*itoa_base) (ULONG64 i, int issigned, int base);
    void (*dfwmalloc_assert_) (const char *file, int line, const char *msg);
    int (*dfwmalloc_get_alignment_adjustment) (char *p, int alignment);
    int (*dfwmalloc_get_bucket) (struct dfwmalloc *dfw, long n);
    int (*dfwmalloc_heap_init) (struct heap *h, int arena_id);
    void (*dfwmalloc_print_backtrace) (const char *msg, void **addr, int n);
#if defined(WITH_BTANDCHECK) || defined(WANT_PRINTFSIMPLE)
    int (*snprintf_simple) (char *out, int outlen, const char *fmt, ...);
    int (*vsnprintf_simple) (char *out, int outlen, const char *s, va_list ap);
#endif
    long (*calc_bucket) (struct dfwmalloc *dfw, int k, long r);
#ifdef WITH_BTANDCHECK
    struct backtrace_item *(*dfwmalloc_add_backtrace_locked) (struct dfwmalloc *dfw, struct backtrace *g, ULONG hash1, ULONG hash2, void **addr, int n);
    struct backtrace_item *(*dfwmalloc_find_backtrace) (struct backtrace *g, ULONG hash1, ULONG hash2, void *addr, int n);
#endif
    struct dfwmalloc *(*dfwmalloc_find_pointer) (struct global *g, void *u);
    struct arena *(*dfwmalloc_get_arena) (struct global *g);
    ULONG (*dfwmalloc_hash_address) (void **addr, int n);
    void (*dfwfree_delayed) (struct dfwmalloc *dfw, struct dfwmalloc *other_arena, void *u, struct calltrace *c);
    void (*dfwfree_shared_pointer) (struct dfwmalloc *dfw, void *u, struct calltrace *c);
    void (*dfwfree_) (struct dfwmalloc *dfw, struct free_item *p);
    void (*dfwfree) (struct dfwmalloc *dfw, void *u, struct calltrace *c);
    void (*dfwmalloc_alloc_backtrace_head) (struct dfwmalloc *dfw);
#ifdef WITH_BTANDCHECK
    void (*dfwmalloc_backtrace_account) (struct dfwmalloc *dfw, backtrace_id_t backtrace_id, long alloced, long blocks);
#ifdef USE_TLS
    void (*dfwmalloc_backtrace_account_otherarena) (struct dfwmalloc *dfw, struct backtrace *g, backtrace_id_t backtrace_id, long alloced, long blocks);
#endif
#endif
    void (*dfwmalloc_backtrace_if_enabled) (struct calltrace *c);
    void (*dfwmalloc_dump_leaks_) (struct dfwmalloc_leak_info *info, int arena_id, struct dfwmalloc *dfw, void (*cb) (void *, struct dfwmalloc_leak *), void *hook);
#ifndef NO_CB_HEAPS
#ifndef NO_HEAP_TRUNCATION
    void (*dfwmalloc_cb_bubblesort) (struct control_block_head *cbh, struct control_block *cb);
#endif
    void (*dfwmalloc_cb_insert_free_block) (struct control_block *cb, void *u);
    void *(*dfwmalloc_cb_get_min_block) (struct control_block *cb);
    void (*dfwmalloc_cb_free) (struct cb_heaps *cbhs, void *u);
    int (*dfwmalloc_cb_get_size) (void *u);
    void *(*dfwmalloc_cb_malloc) (struct cb_heaps *cbhs, int arena_id, malloc_size_t n);
    void (*dfwmalloc_cb_destroy_heap) (struct cb_heaps *cbhs);
    void (*dfwmalloc_cb_init_page_info) (struct cb_heaps_info *cb_heaps_info);
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
    void (*dfwmalloc_cb_rankheap_remove) (struct rankheap *rh, struct control_block *cb_);
    struct control_block *(*dfwmalloc_cb_rankheap_pop) (struct rankheap *rh);
    struct control_block *(*dfwmalloc_cb_rankheap_merge) (struct control_block *a, struct control_block *b);
    void (*dfwmalloc_cb_rankheap_insert) (struct rankheap *rh, struct control_block *cb);
    struct control_block *(*dfwmalloc_cb_rankheap_merge_right) (struct control_block *a);
    struct control_block *(*dfwmalloc_cb_rankheap_merge_left) (struct control_block *a);
    struct control_block *(*dfwmalloc_cb_rankheap_merge_subheaps) (struct control_block *a);
    void (*detach_subheap) (struct control_block *cb);
#endif
#ifndef NO_HEAP_TRUNCATION
    struct control_block *(*dfwmalloc_cb_unlink_heap_tail) (struct cb_heaps *cbhs, int heap_index);
    void (*dfwmalloc_cb_shrink_heap) (struct heap *heap, struct cb_heaps *cbhs, char *p);
    void (*dfwmalloc_cb_truncate) (struct cb_heaps *cbhs);
#endif
    struct control_block *(*dfwmalloc_cb_add) (struct cb_heaps *cbhs, int arena_id, malloc_size_t n);
#endif
    void (*dfwmalloc_free_shared_pointers) (struct dfwmalloc *dfw, struct calltrace *c);
    void (*dfwmalloc_free_) (struct dfwmalloc *dfw, void *u, struct calltrace *c);
    void (*dfwmalloc_heap_free) (struct heap *h);
    void *(*dfwmalloc_heap_rawalloc_locked) (struct heap *h, long n, int alignment, int *need_zero);
    void *(*dfwmalloc_heap_rawalloc) (struct dfwmalloc *dfw, long n, int alignment, int *need_zero);
    int (*dfwmalloc_init_free_lists) (struct dfwmalloc *dfw);
    int (*dfwmalloc_init_global_free_lists) (struct dfwmalloc *dfw);
    void *(*dfwmalloc_locked) (struct dfwmalloc *dfw, struct free_list *l, const malloc_size_t n, int alignment, int *need_zero, struct calltrace *c);
#if !defined(USE_PRECOMPILED_STATIC_DATA) || defined(WITH_BTANDCHECK)
    void *(*dfwmalloc_raw_calloc) (struct dfwmalloc *dfw, long n);
#endif
    int (*dfwmalloc_try_free_dfwmalloc) (struct global *g, struct dfwmalloc *dfw, struct calltrace *c);
    void (*dfwmalloc_release_arena) (struct arena *t);
    int (*dfwmalloc_startup) (struct dfwmalloc *dfw, struct heap *heap, int arena_id);
    void (*dfwmalloc_stats_) (struct dfwmalloc_stats *s, struct dfwmalloc *dfw);
    void *(*dfwmalloc_with_truncation) (struct dfwmalloc *dfw, struct free_list *l, const malloc_size_t n, int alignment, int *need_zero, struct calltrace *c);
    void *(*dfwmalloc_) (struct dfwmalloc *dfw, const malloc_size_t n, int alignment, int *need_zero, struct calltrace *c);
    void (*dfwmalloc_test) (struct dfwmalloc *dfw);
#ifndef NO_HEAP_TRUNCATION
    void (*dfwmalloc_truncate_heap_locked) (struct dfwmalloc *dfw, unsigned int *flush, struct calltrace *c);
#endif
    void (*dfwmalloc_user_alert) (struct dfwmalloc *dfw, struct calltrace *c, const char *msg, const struct free_item *p);
    void (*dfwmalloc_zero_free_list_head) (struct dfwmalloc *dfw, int k);
    void *(*dfwrealloc) (struct dfwmalloc *dfw, void *u, const malloc_size_t n, struct calltrace *c);
    void (*dfwfree__) (struct dfwmalloc *dfw, struct free_list *l, struct free_item *p);
    void (*printf_simple) (const char *fmt, ...);
    void (*perror_abort) (const char *msg);
    void (*speed_test) (void);
    void *(*tf2) (void *arg);
    void *(*tf3) (void *arg);
    void *(*tf) (void *arg);
};

void dfwmalloc_implementation (struct dfwmalloc_noopt *s)
{
#ifdef WITH_BTANDCHECK
    s->dfwmalloc_backtrace = dfwmalloc_backtrace;
#endif
    s->itoa_base = itoa_base;
#ifdef WITH_BTANDCHECK
    s->dfwmalloc_assert_ = dfwmalloc_assert_;
#endif
    s->dfwmalloc_get_alignment_adjustment = dfwmalloc_get_alignment_adjustment;
    s->dfwmalloc_get_bucket = dfwmalloc_get_bucket;
    s->dfwmalloc_heap_init = dfwmalloc_heap_init;
#ifdef WITH_BTANDCHECK
    s->dfwmalloc_print_backtrace = dfwmalloc_print_backtrace;
#endif
#if defined(WITH_BTANDCHECK) || defined(WANT_PRINTFSIMPLE)
    s->snprintf_simple = snprintf_simple;
    s->vsnprintf_simple = vsnprintf_simple;
#endif
    s->calc_bucket = calc_bucket;
#ifdef WITH_BTANDCHECK
    s->dfwmalloc_add_backtrace_locked = dfwmalloc_add_backtrace_locked;
    s->dfwmalloc_find_backtrace = dfwmalloc_find_backtrace;
#endif
#ifdef USE_TLS
    s->dfwmalloc_find_pointer = dfwmalloc_find_pointer;
    s->dfwmalloc_get_arena = dfwmalloc_get_arena;
#endif
#ifdef WITH_BTANDCHECK
    s->dfwmalloc_hash_address = dfwmalloc_hash_address;
#endif
#ifdef USE_TLS
    s->dfwfree_delayed = dfwfree_delayed;
    s->dfwfree_shared_pointer = dfwfree_shared_pointer;
#endif
    s->dfwfree_ = dfwfree_;
    s->dfwfree = dfwfree;
#ifdef WITH_BTANDCHECK
    s->dfwmalloc_alloc_backtrace_head = dfwmalloc_alloc_backtrace_head;
    s->dfwmalloc_backtrace_account = dfwmalloc_backtrace_account;
#ifdef USE_TLS
    s->dfwmalloc_backtrace_account_otherarena = dfwmalloc_backtrace_account_otherarena;
#endif
    s->dfwmalloc_backtrace_if_enabled = dfwmalloc_backtrace_if_enabled;
#endif
    s->dfwmalloc_dump_leaks_ = dfwmalloc_dump_leaks_;
#ifndef NO_CB_HEAPS
#ifndef NO_HEAP_TRUNCATION
    s->dfwmalloc_cb_bubblesort = dfwmalloc_cb_bubblesort;
#endif
    s->dfwmalloc_cb_insert_free_block = dfwmalloc_cb_insert_free_block;
    s->dfwmalloc_cb_get_min_block = dfwmalloc_cb_get_min_block;
    s->dfwmalloc_cb_free = dfwmalloc_cb_free;
    s->dfwmalloc_cb_get_size = dfwmalloc_cb_get_size;
    s->dfwmalloc_cb_malloc = dfwmalloc_cb_malloc;
    s->dfwmalloc_cb_destroy_heap = dfwmalloc_cb_destroy_heap;
    s->dfwmalloc_cb_init_page_info = dfwmalloc_cb_init_page_info;
#if !defined(NO_HEAP_TRUNCATION) && !defined(NO_RANKHEAP)
    s->dfwmalloc_cb_rankheap_remove = dfwmalloc_cb_rankheap_remove;
    s->dfwmalloc_cb_rankheap_pop = dfwmalloc_cb_rankheap_pop;
    s->dfwmalloc_cb_rankheap_merge = dfwmalloc_cb_rankheap_merge;
    s->dfwmalloc_cb_rankheap_insert = dfwmalloc_cb_rankheap_insert;
    s->dfwmalloc_cb_rankheap_merge_right = dfwmalloc_cb_rankheap_merge_right;
    s->dfwmalloc_cb_rankheap_merge_left = dfwmalloc_cb_rankheap_merge_left;
    s->dfwmalloc_cb_rankheap_merge_subheaps = dfwmalloc_cb_rankheap_merge_subheaps;
    s->detach_subheap = detach_subheap;
#endif
#ifndef NO_HEAP_TRUNCATION
    s->dfwmalloc_cb_unlink_heap_tail = dfwmalloc_cb_unlink_heap_tail;
    s->dfwmalloc_cb_shrink_heap = dfwmalloc_cb_shrink_heap;
    s->dfwmalloc_cb_truncate = dfwmalloc_cb_truncate;
#endif
    s->dfwmalloc_cb_add = dfwmalloc_cb_add;
#endif
#ifdef USE_TLS
    s->dfwmalloc_free_shared_pointers = dfwmalloc_free_shared_pointers;
    s->dfwmalloc_heap_free = dfwmalloc_heap_free;
#endif
    s->dfwmalloc_free_ = dfwmalloc_free_;
    s->dfwmalloc_heap_rawalloc_locked = dfwmalloc_heap_rawalloc_locked;
    s->dfwmalloc_heap_rawalloc = dfwmalloc_heap_rawalloc;
    s->dfwmalloc_init_free_lists = dfwmalloc_init_free_lists;
    s->dfwmalloc_init_global_free_lists = dfwmalloc_init_global_free_lists;
    s->dfwmalloc_locked = dfwmalloc_locked;
#if !defined(USE_PRECOMPILED_STATIC_DATA) || defined(WITH_BTANDCHECK)
    s->dfwmalloc_raw_calloc = dfwmalloc_raw_calloc;
#endif
#ifdef USE_TLS
    s->dfwmalloc_try_free_dfwmalloc = dfwmalloc_try_free_dfwmalloc;
    s->dfwmalloc_release_arena = dfwmalloc_release_arena;
#endif
    s->dfwmalloc_startup = dfwmalloc_startup;
    s->dfwmalloc_stats_ = dfwmalloc_stats_;
#ifndef NO_HEAP_TRUNCATION
    s->dfwmalloc_truncate_heap_locked = dfwmalloc_truncate_heap_locked;
#endif
    s->dfwmalloc_with_truncation = dfwmalloc_with_truncation;
    s->dfwmalloc_ = dfwmalloc_;
    s->dfwmalloc_zero_free_list_head = dfwmalloc_zero_free_list_head;
    s->dfwrealloc = dfwrealloc;
    s->dfwfree__ = dfwfree__;
#if defined(WITH_BTANDCHECK) || defined(WANT_PRINTFSIMPLE)
    s->printf_simple = printf_simple;
#endif
#ifdef USE_TLS
    s->perror_abort = perror_abort;
#endif
};

#endif


