#line 1 "numpy/core/src/npysort/timsort.c.src"

/*
 *****************************************************************************
 **       This file was autogenerated from a template  DO NOT EDIT!!!!      **
 **       Changes should be made to the original source (.src) file         **
 *****************************************************************************
 */

#line 1
/* -*- c -*- */

/*
 * The purpose of this module is to add faster sort functions
 * that are type-specific.  This is done by altering the
 * function table for the builtin descriptors.
 *
 * These sorting functions are copied almost directly from numarray
 * with a few modifications (complex comparisons compare the imaginary
 * part if the real parts are equal, for example), and the names
 * are changed.
 *
 * The original sorting code is due to Charles R. Harris who wrote
 * it for numarray.
 */

/*
 * Quick sort is usually the fastest, but the worst case scenario can
 * be slower than the merge and heap sorts.  The merge sort requires
 * extra memory and so for large arrays may not be useful.
 *
 * The merge sort is *stable*, meaning that equal components
 * are unmoved from their entry versions, so it can be used to
 * implement lexigraphic sorting on multiple keys.
 *
 * The heap sort is included for completeness.
 */


/* For details of Timsort, refer to
 * https://github.com/python/cpython/blob/3.7/Objects/listsort.txt
 */

#define NPY_NO_DEPRECATED_API NPY_API_VERSION

#include "npy_sort.h"
#include "npysort_common.h"
#include <stdlib.h>

/* enough for 32 * 1.618 ** 128 elements */
#define TIMSORT_STACK_SIZE 128



static npy_intp compute_min_run(npy_intp num)
{
    npy_intp r = 0;

    while (64 < num) {
        r |= num & 1;
        num >>= 1;
    }

    return num + r;
}

typedef struct {
    npy_intp s; /* start pointer */
    npy_intp l; /* length */
} run;


/* buffer for argsort. Declared here to avoid multiple declarations. */
typedef struct {
    npy_intp *pw;
    npy_intp size;
} buffer_intp;


/* buffer method */
static NPY_INLINE int
resize_buffer_intp(buffer_intp *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_intp));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_intp));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}

/*
 *****************************************************************************
 **                            NUMERIC SORTS                                **
 *****************************************************************************
 */


#line 113


typedef struct {
    npy_bool * pw;
    npy_intp size;
} buffer_bool;


static NPY_INLINE int
resize_buffer_bool(buffer_bool *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_bool));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_bool));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_bool(npy_bool *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_bool vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!BOOL_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !BOOL_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && BOOL_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            BOOL_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && BOOL_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_bool(npy_bool *p1, npy_intp l1, npy_bool *p2, npy_intp l2,
                  npy_bool *p3)
{
    npy_bool *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_bool) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (BOOL_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_bool) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_bool(npy_bool *p1, npy_intp l1, npy_bool *p2, npy_intp l2,
                   npy_bool *p3)
{
    npy_intp ofs;
    npy_bool *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_bool) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (BOOL_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_bool) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_bool(const npy_bool *arr, const npy_intp size, const npy_bool key)
{
    npy_intp last_ofs, ofs, m;

    if (BOOL_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (BOOL_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (BOOL_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_bool(const npy_bool *arr, const npy_intp size, const npy_bool key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (BOOL_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (BOOL_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (BOOL_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_bool(npy_bool *arr, const run *stack, const npy_intp at,
                buffer_bool *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_bool *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_bool(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_bool(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_bool(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_bool(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_bool(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_bool(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_bool(npy_bool *arr, run *stack, npy_intp *stack_ptr,
                    buffer_bool *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_bool(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_bool(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_bool(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_bool(npy_bool *arr, run *stack, npy_intp *stack_ptr,
                      buffer_bool *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_bool(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_bool(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_bool(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_bool(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_bool buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_bool(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_bool(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_bool(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_bool(npy_bool *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_bool vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!BOOL_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !BOOL_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && BOOL_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && BOOL_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_bool(const npy_bool *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_bool key)
{
    npy_intp last_ofs, ofs, m;

    if (BOOL_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (BOOL_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (BOOL_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_bool(const npy_bool *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_bool key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (BOOL_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (BOOL_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (BOOL_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_bool(npy_bool *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (BOOL_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_bool(npy_bool *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (BOOL_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_bool(npy_bool *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_bool(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_bool(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_bool(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_bool(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_bool(npy_bool *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_bool(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_bool(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_bool(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_bool(npy_bool *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_bool(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_bool(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_bool(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_bool(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_bool(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_bool(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_bool(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_byte * pw;
    npy_intp size;
} buffer_byte;


static NPY_INLINE int
resize_buffer_byte(buffer_byte *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_byte));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_byte));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_byte(npy_byte *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_byte vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!BYTE_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !BYTE_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && BYTE_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            BYTE_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && BYTE_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_byte(npy_byte *p1, npy_intp l1, npy_byte *p2, npy_intp l2,
                  npy_byte *p3)
{
    npy_byte *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_byte) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (BYTE_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_byte) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_byte(npy_byte *p1, npy_intp l1, npy_byte *p2, npy_intp l2,
                   npy_byte *p3)
{
    npy_intp ofs;
    npy_byte *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_byte) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (BYTE_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_byte) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_byte(const npy_byte *arr, const npy_intp size, const npy_byte key)
{
    npy_intp last_ofs, ofs, m;

    if (BYTE_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (BYTE_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (BYTE_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_byte(const npy_byte *arr, const npy_intp size, const npy_byte key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (BYTE_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (BYTE_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (BYTE_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_byte(npy_byte *arr, const run *stack, const npy_intp at,
                buffer_byte *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_byte *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_byte(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_byte(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_byte(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_byte(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_byte(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_byte(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_byte(npy_byte *arr, run *stack, npy_intp *stack_ptr,
                    buffer_byte *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_byte(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_byte(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_byte(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_byte(npy_byte *arr, run *stack, npy_intp *stack_ptr,
                      buffer_byte *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_byte(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_byte(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_byte(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_byte(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_byte buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_byte(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_byte(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_byte(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_byte(npy_byte *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_byte vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!BYTE_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !BYTE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && BYTE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && BYTE_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_byte(const npy_byte *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_byte key)
{
    npy_intp last_ofs, ofs, m;

    if (BYTE_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (BYTE_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (BYTE_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_byte(const npy_byte *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_byte key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (BYTE_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (BYTE_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (BYTE_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_byte(npy_byte *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (BYTE_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_byte(npy_byte *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (BYTE_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_byte(npy_byte *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_byte(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_byte(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_byte(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_byte(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_byte(npy_byte *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_byte(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_byte(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_byte(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_byte(npy_byte *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_byte(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_byte(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_byte(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_byte(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_byte(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_byte(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_byte(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_ubyte * pw;
    npy_intp size;
} buffer_ubyte;


static NPY_INLINE int
resize_buffer_ubyte(buffer_ubyte *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_ubyte));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_ubyte));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_ubyte(npy_ubyte *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_ubyte vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!UBYTE_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !UBYTE_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && UBYTE_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            UBYTE_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && UBYTE_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_ubyte(npy_ubyte *p1, npy_intp l1, npy_ubyte *p2, npy_intp l2,
                  npy_ubyte *p3)
{
    npy_ubyte *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_ubyte) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (UBYTE_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_ubyte) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_ubyte(npy_ubyte *p1, npy_intp l1, npy_ubyte *p2, npy_intp l2,
                   npy_ubyte *p3)
{
    npy_intp ofs;
    npy_ubyte *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_ubyte) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (UBYTE_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_ubyte) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_ubyte(const npy_ubyte *arr, const npy_intp size, const npy_ubyte key)
{
    npy_intp last_ofs, ofs, m;

    if (UBYTE_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (UBYTE_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (UBYTE_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_ubyte(const npy_ubyte *arr, const npy_intp size, const npy_ubyte key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (UBYTE_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (UBYTE_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (UBYTE_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_ubyte(npy_ubyte *arr, const run *stack, const npy_intp at,
                buffer_ubyte *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_ubyte *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_ubyte(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_ubyte(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_ubyte(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_ubyte(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_ubyte(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_ubyte(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_ubyte(npy_ubyte *arr, run *stack, npy_intp *stack_ptr,
                    buffer_ubyte *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_ubyte(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_ubyte(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_ubyte(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_ubyte(npy_ubyte *arr, run *stack, npy_intp *stack_ptr,
                      buffer_ubyte *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_ubyte(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_ubyte(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_ubyte(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_ubyte(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_ubyte buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_ubyte(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_ubyte(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_ubyte(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_ubyte(npy_ubyte *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_ubyte vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!UBYTE_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !UBYTE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && UBYTE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && UBYTE_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_ubyte(const npy_ubyte *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_ubyte key)
{
    npy_intp last_ofs, ofs, m;

    if (UBYTE_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (UBYTE_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (UBYTE_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_ubyte(const npy_ubyte *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_ubyte key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (UBYTE_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (UBYTE_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (UBYTE_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_ubyte(npy_ubyte *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (UBYTE_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_ubyte(npy_ubyte *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (UBYTE_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_ubyte(npy_ubyte *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_ubyte(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_ubyte(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_ubyte(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_ubyte(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_ubyte(npy_ubyte *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_ubyte(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_ubyte(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_ubyte(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_ubyte(npy_ubyte *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_ubyte(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_ubyte(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_ubyte(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_ubyte(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_ubyte(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_ubyte(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_ubyte(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_short * pw;
    npy_intp size;
} buffer_short;


static NPY_INLINE int
resize_buffer_short(buffer_short *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_short));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_short));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_short(npy_short *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_short vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!SHORT_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !SHORT_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && SHORT_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            SHORT_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && SHORT_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_short(npy_short *p1, npy_intp l1, npy_short *p2, npy_intp l2,
                  npy_short *p3)
{
    npy_short *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_short) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (SHORT_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_short) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_short(npy_short *p1, npy_intp l1, npy_short *p2, npy_intp l2,
                   npy_short *p3)
{
    npy_intp ofs;
    npy_short *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_short) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (SHORT_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_short) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_short(const npy_short *arr, const npy_intp size, const npy_short key)
{
    npy_intp last_ofs, ofs, m;

    if (SHORT_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (SHORT_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (SHORT_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_short(const npy_short *arr, const npy_intp size, const npy_short key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (SHORT_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (SHORT_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (SHORT_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_short(npy_short *arr, const run *stack, const npy_intp at,
                buffer_short *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_short *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_short(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_short(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_short(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_short(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_short(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_short(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_short(npy_short *arr, run *stack, npy_intp *stack_ptr,
                    buffer_short *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_short(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_short(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_short(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_short(npy_short *arr, run *stack, npy_intp *stack_ptr,
                      buffer_short *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_short(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_short(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_short(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_short(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_short buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_short(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_short(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_short(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_short(npy_short *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_short vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!SHORT_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !SHORT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && SHORT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && SHORT_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_short(const npy_short *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_short key)
{
    npy_intp last_ofs, ofs, m;

    if (SHORT_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (SHORT_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (SHORT_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_short(const npy_short *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_short key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (SHORT_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (SHORT_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (SHORT_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_short(npy_short *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (SHORT_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_short(npy_short *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (SHORT_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_short(npy_short *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_short(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_short(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_short(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_short(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_short(npy_short *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_short(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_short(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_short(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_short(npy_short *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_short(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_short(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_short(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_short(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_short(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_short(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_short(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_ushort * pw;
    npy_intp size;
} buffer_ushort;


static NPY_INLINE int
resize_buffer_ushort(buffer_ushort *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_ushort));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_ushort));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_ushort(npy_ushort *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_ushort vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!USHORT_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !USHORT_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && USHORT_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            USHORT_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && USHORT_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_ushort(npy_ushort *p1, npy_intp l1, npy_ushort *p2, npy_intp l2,
                  npy_ushort *p3)
{
    npy_ushort *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_ushort) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (USHORT_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_ushort) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_ushort(npy_ushort *p1, npy_intp l1, npy_ushort *p2, npy_intp l2,
                   npy_ushort *p3)
{
    npy_intp ofs;
    npy_ushort *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_ushort) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (USHORT_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_ushort) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_ushort(const npy_ushort *arr, const npy_intp size, const npy_ushort key)
{
    npy_intp last_ofs, ofs, m;

    if (USHORT_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (USHORT_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (USHORT_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_ushort(const npy_ushort *arr, const npy_intp size, const npy_ushort key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (USHORT_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (USHORT_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (USHORT_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_ushort(npy_ushort *arr, const run *stack, const npy_intp at,
                buffer_ushort *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_ushort *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_ushort(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_ushort(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_ushort(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_ushort(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_ushort(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_ushort(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_ushort(npy_ushort *arr, run *stack, npy_intp *stack_ptr,
                    buffer_ushort *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_ushort(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_ushort(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_ushort(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_ushort(npy_ushort *arr, run *stack, npy_intp *stack_ptr,
                      buffer_ushort *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_ushort(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_ushort(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_ushort(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_ushort(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_ushort buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_ushort(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_ushort(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_ushort(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_ushort(npy_ushort *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_ushort vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!USHORT_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !USHORT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && USHORT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && USHORT_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_ushort(const npy_ushort *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_ushort key)
{
    npy_intp last_ofs, ofs, m;

    if (USHORT_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (USHORT_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (USHORT_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_ushort(const npy_ushort *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_ushort key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (USHORT_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (USHORT_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (USHORT_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_ushort(npy_ushort *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (USHORT_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_ushort(npy_ushort *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (USHORT_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_ushort(npy_ushort *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_ushort(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_ushort(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_ushort(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_ushort(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_ushort(npy_ushort *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_ushort(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_ushort(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_ushort(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_ushort(npy_ushort *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_ushort(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_ushort(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_ushort(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_ushort(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_ushort(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_ushort(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_ushort(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_int * pw;
    npy_intp size;
} buffer_int;


static NPY_INLINE int
resize_buffer_int(buffer_int *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_int));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_int));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_int(npy_int *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_int vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!INT_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !INT_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && INT_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INT_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && INT_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_int(npy_int *p1, npy_intp l1, npy_int *p2, npy_intp l2,
                  npy_int *p3)
{
    npy_int *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_int) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (INT_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_int) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_int(npy_int *p1, npy_intp l1, npy_int *p2, npy_intp l2,
                   npy_int *p3)
{
    npy_intp ofs;
    npy_int *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_int) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (INT_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_int) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_int(const npy_int *arr, const npy_intp size, const npy_int key)
{
    npy_intp last_ofs, ofs, m;

    if (INT_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (INT_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (INT_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_int(const npy_int *arr, const npy_intp size, const npy_int key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (INT_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (INT_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (INT_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_int(npy_int *arr, const run *stack, const npy_intp at,
                buffer_int *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_int *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_int(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_int(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_int(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_int(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_int(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_int(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_int(npy_int *arr, run *stack, npy_intp *stack_ptr,
                    buffer_int *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_int(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_int(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_int(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_int(npy_int *arr, run *stack, npy_intp *stack_ptr,
                      buffer_int *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_int(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_int(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_int(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_int(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_int buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_int(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_int(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_int(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_int(npy_int *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_int vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!INT_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !INT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && INT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && INT_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_int(const npy_int *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_int key)
{
    npy_intp last_ofs, ofs, m;

    if (INT_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (INT_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (INT_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_int(const npy_int *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_int key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (INT_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (INT_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (INT_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_int(npy_int *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (INT_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_int(npy_int *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (INT_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_int(npy_int *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_int(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_int(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_int(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_int(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_int(npy_int *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_int(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_int(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_int(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_int(npy_int *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_int(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_int(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_int(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_int(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_int(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_int(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_int(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_uint * pw;
    npy_intp size;
} buffer_uint;


static NPY_INLINE int
resize_buffer_uint(buffer_uint *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_uint));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_uint));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_uint(npy_uint *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_uint vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!UINT_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !UINT_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && UINT_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            UINT_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && UINT_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_uint(npy_uint *p1, npy_intp l1, npy_uint *p2, npy_intp l2,
                  npy_uint *p3)
{
    npy_uint *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_uint) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (UINT_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_uint) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_uint(npy_uint *p1, npy_intp l1, npy_uint *p2, npy_intp l2,
                   npy_uint *p3)
{
    npy_intp ofs;
    npy_uint *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_uint) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (UINT_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_uint) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_uint(const npy_uint *arr, const npy_intp size, const npy_uint key)
{
    npy_intp last_ofs, ofs, m;

    if (UINT_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (UINT_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (UINT_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_uint(const npy_uint *arr, const npy_intp size, const npy_uint key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (UINT_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (UINT_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (UINT_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_uint(npy_uint *arr, const run *stack, const npy_intp at,
                buffer_uint *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_uint *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_uint(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_uint(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_uint(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_uint(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_uint(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_uint(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_uint(npy_uint *arr, run *stack, npy_intp *stack_ptr,
                    buffer_uint *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_uint(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_uint(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_uint(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_uint(npy_uint *arr, run *stack, npy_intp *stack_ptr,
                      buffer_uint *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_uint(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_uint(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_uint(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_uint(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_uint buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_uint(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_uint(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_uint(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_uint(npy_uint *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_uint vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!UINT_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !UINT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && UINT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && UINT_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_uint(const npy_uint *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_uint key)
{
    npy_intp last_ofs, ofs, m;

    if (UINT_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (UINT_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (UINT_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_uint(const npy_uint *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_uint key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (UINT_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (UINT_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (UINT_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_uint(npy_uint *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (UINT_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_uint(npy_uint *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (UINT_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_uint(npy_uint *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_uint(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_uint(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_uint(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_uint(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_uint(npy_uint *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_uint(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_uint(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_uint(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_uint(npy_uint *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_uint(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_uint(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_uint(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_uint(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_uint(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_uint(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_uint(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_long * pw;
    npy_intp size;
} buffer_long;


static NPY_INLINE int
resize_buffer_long(buffer_long *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_long));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_long));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_long(npy_long *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_long vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!LONG_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !LONG_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && LONG_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            LONG_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && LONG_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_long(npy_long *p1, npy_intp l1, npy_long *p2, npy_intp l2,
                  npy_long *p3)
{
    npy_long *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_long) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (LONG_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_long) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_long(npy_long *p1, npy_intp l1, npy_long *p2, npy_intp l2,
                   npy_long *p3)
{
    npy_intp ofs;
    npy_long *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_long) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (LONG_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_long) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_long(const npy_long *arr, const npy_intp size, const npy_long key)
{
    npy_intp last_ofs, ofs, m;

    if (LONG_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (LONG_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (LONG_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_long(const npy_long *arr, const npy_intp size, const npy_long key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (LONG_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (LONG_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (LONG_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_long(npy_long *arr, const run *stack, const npy_intp at,
                buffer_long *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_long *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_long(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_long(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_long(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_long(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_long(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_long(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_long(npy_long *arr, run *stack, npy_intp *stack_ptr,
                    buffer_long *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_long(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_long(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_long(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_long(npy_long *arr, run *stack, npy_intp *stack_ptr,
                      buffer_long *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_long(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_long(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_long(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_long(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_long buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_long(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_long(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_long(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_long(npy_long *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_long vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!LONG_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !LONG_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && LONG_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && LONG_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_long(const npy_long *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_long key)
{
    npy_intp last_ofs, ofs, m;

    if (LONG_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (LONG_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (LONG_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_long(const npy_long *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_long key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (LONG_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (LONG_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (LONG_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_long(npy_long *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (LONG_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_long(npy_long *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (LONG_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_long(npy_long *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_long(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_long(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_long(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_long(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_long(npy_long *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_long(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_long(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_long(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_long(npy_long *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_long(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_long(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_long(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_long(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_long(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_long(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_long(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_ulong * pw;
    npy_intp size;
} buffer_ulong;


static NPY_INLINE int
resize_buffer_ulong(buffer_ulong *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_ulong));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_ulong));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_ulong(npy_ulong *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_ulong vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!ULONG_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !ULONG_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && ULONG_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            ULONG_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && ULONG_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_ulong(npy_ulong *p1, npy_intp l1, npy_ulong *p2, npy_intp l2,
                  npy_ulong *p3)
{
    npy_ulong *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_ulong) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (ULONG_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_ulong) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_ulong(npy_ulong *p1, npy_intp l1, npy_ulong *p2, npy_intp l2,
                   npy_ulong *p3)
{
    npy_intp ofs;
    npy_ulong *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_ulong) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (ULONG_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_ulong) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_ulong(const npy_ulong *arr, const npy_intp size, const npy_ulong key)
{
    npy_intp last_ofs, ofs, m;

    if (ULONG_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (ULONG_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (ULONG_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_ulong(const npy_ulong *arr, const npy_intp size, const npy_ulong key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (ULONG_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (ULONG_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (ULONG_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_ulong(npy_ulong *arr, const run *stack, const npy_intp at,
                buffer_ulong *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_ulong *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_ulong(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_ulong(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_ulong(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_ulong(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_ulong(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_ulong(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_ulong(npy_ulong *arr, run *stack, npy_intp *stack_ptr,
                    buffer_ulong *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_ulong(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_ulong(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_ulong(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_ulong(npy_ulong *arr, run *stack, npy_intp *stack_ptr,
                      buffer_ulong *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_ulong(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_ulong(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_ulong(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_ulong(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_ulong buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_ulong(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_ulong(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_ulong(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_ulong(npy_ulong *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_ulong vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!ULONG_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !ULONG_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && ULONG_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && ULONG_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_ulong(const npy_ulong *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_ulong key)
{
    npy_intp last_ofs, ofs, m;

    if (ULONG_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (ULONG_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (ULONG_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_ulong(const npy_ulong *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_ulong key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (ULONG_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (ULONG_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (ULONG_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_ulong(npy_ulong *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (ULONG_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_ulong(npy_ulong *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (ULONG_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_ulong(npy_ulong *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_ulong(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_ulong(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_ulong(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_ulong(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_ulong(npy_ulong *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_ulong(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_ulong(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_ulong(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_ulong(npy_ulong *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_ulong(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_ulong(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_ulong(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_ulong(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_ulong(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_ulong(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_ulong(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_longlong * pw;
    npy_intp size;
} buffer_longlong;


static NPY_INLINE int
resize_buffer_longlong(buffer_longlong *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_longlong));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_longlong));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_longlong(npy_longlong *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_longlong vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!LONGLONG_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !LONGLONG_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && LONGLONG_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            LONGLONG_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && LONGLONG_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_longlong(npy_longlong *p1, npy_intp l1, npy_longlong *p2, npy_intp l2,
                  npy_longlong *p3)
{
    npy_longlong *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_longlong) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (LONGLONG_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_longlong) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_longlong(npy_longlong *p1, npy_intp l1, npy_longlong *p2, npy_intp l2,
                   npy_longlong *p3)
{
    npy_intp ofs;
    npy_longlong *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_longlong) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (LONGLONG_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_longlong) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_longlong(const npy_longlong *arr, const npy_intp size, const npy_longlong key)
{
    npy_intp last_ofs, ofs, m;

    if (LONGLONG_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (LONGLONG_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (LONGLONG_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_longlong(const npy_longlong *arr, const npy_intp size, const npy_longlong key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (LONGLONG_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (LONGLONG_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (LONGLONG_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_longlong(npy_longlong *arr, const run *stack, const npy_intp at,
                buffer_longlong *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_longlong *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_longlong(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_longlong(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_longlong(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_longlong(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_longlong(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_longlong(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_longlong(npy_longlong *arr, run *stack, npy_intp *stack_ptr,
                    buffer_longlong *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_longlong(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_longlong(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_longlong(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_longlong(npy_longlong *arr, run *stack, npy_intp *stack_ptr,
                      buffer_longlong *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_longlong(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_longlong(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_longlong(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_longlong(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_longlong buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_longlong(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_longlong(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_longlong(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_longlong(npy_longlong *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_longlong vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!LONGLONG_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !LONGLONG_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && LONGLONG_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && LONGLONG_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_longlong(const npy_longlong *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_longlong key)
{
    npy_intp last_ofs, ofs, m;

    if (LONGLONG_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (LONGLONG_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (LONGLONG_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_longlong(const npy_longlong *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_longlong key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (LONGLONG_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (LONGLONG_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (LONGLONG_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_longlong(npy_longlong *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (LONGLONG_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_longlong(npy_longlong *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (LONGLONG_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_longlong(npy_longlong *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_longlong(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_longlong(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_longlong(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_longlong(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_longlong(npy_longlong *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_longlong(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_longlong(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_longlong(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_longlong(npy_longlong *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_longlong(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_longlong(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_longlong(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_longlong(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_longlong(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_longlong(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_longlong(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_ulonglong * pw;
    npy_intp size;
} buffer_ulonglong;


static NPY_INLINE int
resize_buffer_ulonglong(buffer_ulonglong *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_ulonglong));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_ulonglong));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_ulonglong(npy_ulonglong *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_ulonglong vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!ULONGLONG_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !ULONGLONG_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && ULONGLONG_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            ULONGLONG_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && ULONGLONG_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_ulonglong(npy_ulonglong *p1, npy_intp l1, npy_ulonglong *p2, npy_intp l2,
                  npy_ulonglong *p3)
{
    npy_ulonglong *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_ulonglong) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (ULONGLONG_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_ulonglong) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_ulonglong(npy_ulonglong *p1, npy_intp l1, npy_ulonglong *p2, npy_intp l2,
                   npy_ulonglong *p3)
{
    npy_intp ofs;
    npy_ulonglong *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_ulonglong) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (ULONGLONG_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_ulonglong) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_ulonglong(const npy_ulonglong *arr, const npy_intp size, const npy_ulonglong key)
{
    npy_intp last_ofs, ofs, m;

    if (ULONGLONG_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (ULONGLONG_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (ULONGLONG_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_ulonglong(const npy_ulonglong *arr, const npy_intp size, const npy_ulonglong key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (ULONGLONG_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (ULONGLONG_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (ULONGLONG_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_ulonglong(npy_ulonglong *arr, const run *stack, const npy_intp at,
                buffer_ulonglong *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_ulonglong *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_ulonglong(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_ulonglong(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_ulonglong(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_ulonglong(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_ulonglong(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_ulonglong(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_ulonglong(npy_ulonglong *arr, run *stack, npy_intp *stack_ptr,
                    buffer_ulonglong *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_ulonglong(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_ulonglong(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_ulonglong(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_ulonglong(npy_ulonglong *arr, run *stack, npy_intp *stack_ptr,
                      buffer_ulonglong *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_ulonglong(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_ulonglong(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_ulonglong(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_ulonglong(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_ulonglong buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_ulonglong(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_ulonglong(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_ulonglong(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_ulonglong(npy_ulonglong *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_ulonglong vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!ULONGLONG_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !ULONGLONG_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && ULONGLONG_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && ULONGLONG_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_ulonglong(const npy_ulonglong *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_ulonglong key)
{
    npy_intp last_ofs, ofs, m;

    if (ULONGLONG_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (ULONGLONG_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (ULONGLONG_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_ulonglong(const npy_ulonglong *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_ulonglong key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (ULONGLONG_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (ULONGLONG_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (ULONGLONG_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_ulonglong(npy_ulonglong *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (ULONGLONG_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_ulonglong(npy_ulonglong *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (ULONGLONG_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_ulonglong(npy_ulonglong *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_ulonglong(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_ulonglong(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_ulonglong(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_ulonglong(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_ulonglong(npy_ulonglong *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_ulonglong(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_ulonglong(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_ulonglong(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_ulonglong(npy_ulonglong *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_ulonglong(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_ulonglong(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_ulonglong(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_ulonglong(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_ulonglong(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_ulonglong(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_ulonglong(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_ushort * pw;
    npy_intp size;
} buffer_half;


static NPY_INLINE int
resize_buffer_half(buffer_half *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_ushort));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_ushort));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_half(npy_ushort *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_ushort vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!HALF_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !HALF_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && HALF_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            HALF_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && HALF_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_half(npy_ushort *p1, npy_intp l1, npy_ushort *p2, npy_intp l2,
                  npy_ushort *p3)
{
    npy_ushort *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_ushort) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (HALF_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_ushort) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_half(npy_ushort *p1, npy_intp l1, npy_ushort *p2, npy_intp l2,
                   npy_ushort *p3)
{
    npy_intp ofs;
    npy_ushort *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_ushort) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (HALF_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_ushort) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_half(const npy_ushort *arr, const npy_intp size, const npy_ushort key)
{
    npy_intp last_ofs, ofs, m;

    if (HALF_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (HALF_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (HALF_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_half(const npy_ushort *arr, const npy_intp size, const npy_ushort key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (HALF_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (HALF_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (HALF_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_half(npy_ushort *arr, const run *stack, const npy_intp at,
                buffer_half *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_ushort *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_half(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_half(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_half(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_half(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_half(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_half(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_half(npy_ushort *arr, run *stack, npy_intp *stack_ptr,
                    buffer_half *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_half(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_half(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_half(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_half(npy_ushort *arr, run *stack, npy_intp *stack_ptr,
                      buffer_half *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_half(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_half(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_half(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_half(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_half buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_half(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_half(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_half(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_half(npy_ushort *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_ushort vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!HALF_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !HALF_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && HALF_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && HALF_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_half(const npy_ushort *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_ushort key)
{
    npy_intp last_ofs, ofs, m;

    if (HALF_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (HALF_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (HALF_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_half(const npy_ushort *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_ushort key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (HALF_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (HALF_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (HALF_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_half(npy_ushort *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (HALF_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_half(npy_ushort *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (HALF_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_half(npy_ushort *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_half(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_half(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_half(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_half(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_half(npy_ushort *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_half(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_half(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_half(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_half(npy_ushort *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_half(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_half(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_half(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_half(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_half(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_half(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_half(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_float * pw;
    npy_intp size;
} buffer_float;


static NPY_INLINE int
resize_buffer_float(buffer_float *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_float));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_float));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_float(npy_float *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_float vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!FLOAT_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !FLOAT_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && FLOAT_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            FLOAT_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && FLOAT_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_float(npy_float *p1, npy_intp l1, npy_float *p2, npy_intp l2,
                  npy_float *p3)
{
    npy_float *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_float) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (FLOAT_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_float) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_float(npy_float *p1, npy_intp l1, npy_float *p2, npy_intp l2,
                   npy_float *p3)
{
    npy_intp ofs;
    npy_float *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_float) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (FLOAT_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_float) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_float(const npy_float *arr, const npy_intp size, const npy_float key)
{
    npy_intp last_ofs, ofs, m;

    if (FLOAT_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (FLOAT_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (FLOAT_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_float(const npy_float *arr, const npy_intp size, const npy_float key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (FLOAT_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (FLOAT_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (FLOAT_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_float(npy_float *arr, const run *stack, const npy_intp at,
                buffer_float *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_float *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_float(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_float(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_float(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_float(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_float(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_float(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_float(npy_float *arr, run *stack, npy_intp *stack_ptr,
                    buffer_float *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_float(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_float(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_float(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_float(npy_float *arr, run *stack, npy_intp *stack_ptr,
                      buffer_float *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_float(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_float(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_float(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_float(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_float buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_float(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_float(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_float(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_float(npy_float *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_float vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!FLOAT_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !FLOAT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && FLOAT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && FLOAT_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_float(const npy_float *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_float key)
{
    npy_intp last_ofs, ofs, m;

    if (FLOAT_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (FLOAT_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (FLOAT_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_float(const npy_float *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_float key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (FLOAT_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (FLOAT_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (FLOAT_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_float(npy_float *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (FLOAT_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_float(npy_float *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (FLOAT_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_float(npy_float *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_float(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_float(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_float(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_float(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_float(npy_float *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_float(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_float(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_float(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_float(npy_float *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_float(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_float(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_float(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_float(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_float(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_float(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_float(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_double * pw;
    npy_intp size;
} buffer_double;


static NPY_INLINE int
resize_buffer_double(buffer_double *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_double));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_double));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_double(npy_double *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_double vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!DOUBLE_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !DOUBLE_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && DOUBLE_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            DOUBLE_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && DOUBLE_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_double(npy_double *p1, npy_intp l1, npy_double *p2, npy_intp l2,
                  npy_double *p3)
{
    npy_double *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_double) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (DOUBLE_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_double) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_double(npy_double *p1, npy_intp l1, npy_double *p2, npy_intp l2,
                   npy_double *p3)
{
    npy_intp ofs;
    npy_double *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_double) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (DOUBLE_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_double) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_double(const npy_double *arr, const npy_intp size, const npy_double key)
{
    npy_intp last_ofs, ofs, m;

    if (DOUBLE_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (DOUBLE_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (DOUBLE_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_double(const npy_double *arr, const npy_intp size, const npy_double key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (DOUBLE_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (DOUBLE_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (DOUBLE_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_double(npy_double *arr, const run *stack, const npy_intp at,
                buffer_double *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_double *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_double(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_double(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_double(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_double(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_double(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_double(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_double(npy_double *arr, run *stack, npy_intp *stack_ptr,
                    buffer_double *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_double(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_double(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_double(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_double(npy_double *arr, run *stack, npy_intp *stack_ptr,
                      buffer_double *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_double(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_double(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_double(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_double(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_double buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_double(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_double(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_double(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_double(npy_double *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_double vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!DOUBLE_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !DOUBLE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && DOUBLE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && DOUBLE_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_double(const npy_double *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_double key)
{
    npy_intp last_ofs, ofs, m;

    if (DOUBLE_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (DOUBLE_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (DOUBLE_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_double(const npy_double *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_double key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (DOUBLE_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (DOUBLE_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (DOUBLE_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_double(npy_double *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (DOUBLE_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_double(npy_double *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (DOUBLE_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_double(npy_double *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_double(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_double(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_double(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_double(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_double(npy_double *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_double(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_double(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_double(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_double(npy_double *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_double(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_double(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_double(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_double(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_double(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_double(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_double(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_longdouble * pw;
    npy_intp size;
} buffer_longdouble;


static NPY_INLINE int
resize_buffer_longdouble(buffer_longdouble *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_longdouble));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_longdouble));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_longdouble(npy_longdouble *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_longdouble vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!LONGDOUBLE_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !LONGDOUBLE_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && LONGDOUBLE_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            LONGDOUBLE_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && LONGDOUBLE_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_longdouble(npy_longdouble *p1, npy_intp l1, npy_longdouble *p2, npy_intp l2,
                  npy_longdouble *p3)
{
    npy_longdouble *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_longdouble) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (LONGDOUBLE_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_longdouble) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_longdouble(npy_longdouble *p1, npy_intp l1, npy_longdouble *p2, npy_intp l2,
                   npy_longdouble *p3)
{
    npy_intp ofs;
    npy_longdouble *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_longdouble) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (LONGDOUBLE_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_longdouble) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_longdouble(const npy_longdouble *arr, const npy_intp size, const npy_longdouble key)
{
    npy_intp last_ofs, ofs, m;

    if (LONGDOUBLE_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (LONGDOUBLE_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (LONGDOUBLE_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_longdouble(const npy_longdouble *arr, const npy_intp size, const npy_longdouble key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (LONGDOUBLE_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (LONGDOUBLE_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (LONGDOUBLE_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_longdouble(npy_longdouble *arr, const run *stack, const npy_intp at,
                buffer_longdouble *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_longdouble *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_longdouble(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_longdouble(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_longdouble(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_longdouble(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_longdouble(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_longdouble(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_longdouble(npy_longdouble *arr, run *stack, npy_intp *stack_ptr,
                    buffer_longdouble *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_longdouble(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_longdouble(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_longdouble(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_longdouble(npy_longdouble *arr, run *stack, npy_intp *stack_ptr,
                      buffer_longdouble *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_longdouble(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_longdouble(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_longdouble(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_longdouble(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_longdouble buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_longdouble(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_longdouble(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_longdouble(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_longdouble(npy_longdouble *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_longdouble vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!LONGDOUBLE_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !LONGDOUBLE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && LONGDOUBLE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && LONGDOUBLE_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_longdouble(const npy_longdouble *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_longdouble key)
{
    npy_intp last_ofs, ofs, m;

    if (LONGDOUBLE_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (LONGDOUBLE_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (LONGDOUBLE_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_longdouble(const npy_longdouble *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_longdouble key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (LONGDOUBLE_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (LONGDOUBLE_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (LONGDOUBLE_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_longdouble(npy_longdouble *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (LONGDOUBLE_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_longdouble(npy_longdouble *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (LONGDOUBLE_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_longdouble(npy_longdouble *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_longdouble(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_longdouble(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_longdouble(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_longdouble(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_longdouble(npy_longdouble *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_longdouble(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_longdouble(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_longdouble(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_longdouble(npy_longdouble *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_longdouble(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_longdouble(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_longdouble(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_longdouble(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_longdouble(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_longdouble(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_longdouble(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_cfloat * pw;
    npy_intp size;
} buffer_cfloat;


static NPY_INLINE int
resize_buffer_cfloat(buffer_cfloat *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_cfloat));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_cfloat));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_cfloat(npy_cfloat *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_cfloat vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!CFLOAT_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !CFLOAT_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && CFLOAT_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            CFLOAT_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && CFLOAT_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_cfloat(npy_cfloat *p1, npy_intp l1, npy_cfloat *p2, npy_intp l2,
                  npy_cfloat *p3)
{
    npy_cfloat *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_cfloat) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (CFLOAT_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_cfloat) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_cfloat(npy_cfloat *p1, npy_intp l1, npy_cfloat *p2, npy_intp l2,
                   npy_cfloat *p3)
{
    npy_intp ofs;
    npy_cfloat *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_cfloat) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (CFLOAT_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_cfloat) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_cfloat(const npy_cfloat *arr, const npy_intp size, const npy_cfloat key)
{
    npy_intp last_ofs, ofs, m;

    if (CFLOAT_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (CFLOAT_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (CFLOAT_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_cfloat(const npy_cfloat *arr, const npy_intp size, const npy_cfloat key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (CFLOAT_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (CFLOAT_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (CFLOAT_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_cfloat(npy_cfloat *arr, const run *stack, const npy_intp at,
                buffer_cfloat *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_cfloat *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_cfloat(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_cfloat(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_cfloat(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_cfloat(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_cfloat(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_cfloat(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_cfloat(npy_cfloat *arr, run *stack, npy_intp *stack_ptr,
                    buffer_cfloat *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_cfloat(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_cfloat(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_cfloat(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_cfloat(npy_cfloat *arr, run *stack, npy_intp *stack_ptr,
                      buffer_cfloat *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_cfloat(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_cfloat(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_cfloat(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_cfloat(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_cfloat buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_cfloat(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_cfloat(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_cfloat(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_cfloat(npy_cfloat *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_cfloat vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!CFLOAT_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !CFLOAT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && CFLOAT_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && CFLOAT_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_cfloat(const npy_cfloat *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_cfloat key)
{
    npy_intp last_ofs, ofs, m;

    if (CFLOAT_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (CFLOAT_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (CFLOAT_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_cfloat(const npy_cfloat *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_cfloat key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (CFLOAT_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (CFLOAT_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (CFLOAT_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_cfloat(npy_cfloat *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (CFLOAT_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_cfloat(npy_cfloat *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (CFLOAT_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_cfloat(npy_cfloat *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_cfloat(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_cfloat(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_cfloat(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_cfloat(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_cfloat(npy_cfloat *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_cfloat(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_cfloat(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_cfloat(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_cfloat(npy_cfloat *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_cfloat(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_cfloat(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_cfloat(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_cfloat(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_cfloat(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_cfloat(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_cfloat(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_cdouble * pw;
    npy_intp size;
} buffer_cdouble;


static NPY_INLINE int
resize_buffer_cdouble(buffer_cdouble *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_cdouble));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_cdouble));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_cdouble(npy_cdouble *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_cdouble vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!CDOUBLE_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !CDOUBLE_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && CDOUBLE_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            CDOUBLE_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && CDOUBLE_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_cdouble(npy_cdouble *p1, npy_intp l1, npy_cdouble *p2, npy_intp l2,
                  npy_cdouble *p3)
{
    npy_cdouble *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_cdouble) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (CDOUBLE_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_cdouble) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_cdouble(npy_cdouble *p1, npy_intp l1, npy_cdouble *p2, npy_intp l2,
                   npy_cdouble *p3)
{
    npy_intp ofs;
    npy_cdouble *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_cdouble) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (CDOUBLE_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_cdouble) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_cdouble(const npy_cdouble *arr, const npy_intp size, const npy_cdouble key)
{
    npy_intp last_ofs, ofs, m;

    if (CDOUBLE_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (CDOUBLE_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (CDOUBLE_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_cdouble(const npy_cdouble *arr, const npy_intp size, const npy_cdouble key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (CDOUBLE_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (CDOUBLE_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (CDOUBLE_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_cdouble(npy_cdouble *arr, const run *stack, const npy_intp at,
                buffer_cdouble *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_cdouble *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_cdouble(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_cdouble(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_cdouble(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_cdouble(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_cdouble(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_cdouble(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_cdouble(npy_cdouble *arr, run *stack, npy_intp *stack_ptr,
                    buffer_cdouble *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_cdouble(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_cdouble(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_cdouble(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_cdouble(npy_cdouble *arr, run *stack, npy_intp *stack_ptr,
                      buffer_cdouble *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_cdouble(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_cdouble(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_cdouble(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_cdouble(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_cdouble buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_cdouble(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_cdouble(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_cdouble(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_cdouble(npy_cdouble *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_cdouble vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!CDOUBLE_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !CDOUBLE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && CDOUBLE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && CDOUBLE_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_cdouble(const npy_cdouble *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_cdouble key)
{
    npy_intp last_ofs, ofs, m;

    if (CDOUBLE_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (CDOUBLE_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (CDOUBLE_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_cdouble(const npy_cdouble *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_cdouble key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (CDOUBLE_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (CDOUBLE_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (CDOUBLE_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_cdouble(npy_cdouble *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (CDOUBLE_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_cdouble(npy_cdouble *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (CDOUBLE_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_cdouble(npy_cdouble *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_cdouble(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_cdouble(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_cdouble(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_cdouble(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_cdouble(npy_cdouble *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_cdouble(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_cdouble(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_cdouble(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_cdouble(npy_cdouble *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_cdouble(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_cdouble(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_cdouble(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_cdouble(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_cdouble(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_cdouble(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_cdouble(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_clongdouble * pw;
    npy_intp size;
} buffer_clongdouble;


static NPY_INLINE int
resize_buffer_clongdouble(buffer_clongdouble *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_clongdouble));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_clongdouble));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_clongdouble(npy_clongdouble *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_clongdouble vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!CLONGDOUBLE_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !CLONGDOUBLE_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && CLONGDOUBLE_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            CLONGDOUBLE_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && CLONGDOUBLE_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_clongdouble(npy_clongdouble *p1, npy_intp l1, npy_clongdouble *p2, npy_intp l2,
                  npy_clongdouble *p3)
{
    npy_clongdouble *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_clongdouble) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (CLONGDOUBLE_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_clongdouble) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_clongdouble(npy_clongdouble *p1, npy_intp l1, npy_clongdouble *p2, npy_intp l2,
                   npy_clongdouble *p3)
{
    npy_intp ofs;
    npy_clongdouble *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_clongdouble) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (CLONGDOUBLE_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_clongdouble) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_clongdouble(const npy_clongdouble *arr, const npy_intp size, const npy_clongdouble key)
{
    npy_intp last_ofs, ofs, m;

    if (CLONGDOUBLE_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (CLONGDOUBLE_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (CLONGDOUBLE_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_clongdouble(const npy_clongdouble *arr, const npy_intp size, const npy_clongdouble key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (CLONGDOUBLE_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (CLONGDOUBLE_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (CLONGDOUBLE_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_clongdouble(npy_clongdouble *arr, const run *stack, const npy_intp at,
                buffer_clongdouble *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_clongdouble *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_clongdouble(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_clongdouble(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_clongdouble(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_clongdouble(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_clongdouble(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_clongdouble(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_clongdouble(npy_clongdouble *arr, run *stack, npy_intp *stack_ptr,
                    buffer_clongdouble *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_clongdouble(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_clongdouble(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_clongdouble(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_clongdouble(npy_clongdouble *arr, run *stack, npy_intp *stack_ptr,
                      buffer_clongdouble *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_clongdouble(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_clongdouble(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_clongdouble(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_clongdouble(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_clongdouble buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_clongdouble(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_clongdouble(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_clongdouble(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_clongdouble(npy_clongdouble *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_clongdouble vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!CLONGDOUBLE_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !CLONGDOUBLE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && CLONGDOUBLE_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && CLONGDOUBLE_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_clongdouble(const npy_clongdouble *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_clongdouble key)
{
    npy_intp last_ofs, ofs, m;

    if (CLONGDOUBLE_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (CLONGDOUBLE_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (CLONGDOUBLE_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_clongdouble(const npy_clongdouble *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_clongdouble key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (CLONGDOUBLE_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (CLONGDOUBLE_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (CLONGDOUBLE_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_clongdouble(npy_clongdouble *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (CLONGDOUBLE_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_clongdouble(npy_clongdouble *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (CLONGDOUBLE_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_clongdouble(npy_clongdouble *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_clongdouble(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_clongdouble(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_clongdouble(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_clongdouble(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_clongdouble(npy_clongdouble *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_clongdouble(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_clongdouble(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_clongdouble(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_clongdouble(npy_clongdouble *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_clongdouble(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_clongdouble(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_clongdouble(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_clongdouble(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_clongdouble(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_clongdouble(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_clongdouble(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_datetime * pw;
    npy_intp size;
} buffer_datetime;


static NPY_INLINE int
resize_buffer_datetime(buffer_datetime *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_datetime));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_datetime));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_datetime(npy_datetime *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_datetime vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!DATETIME_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !DATETIME_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && DATETIME_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            DATETIME_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && DATETIME_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_datetime(npy_datetime *p1, npy_intp l1, npy_datetime *p2, npy_intp l2,
                  npy_datetime *p3)
{
    npy_datetime *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_datetime) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (DATETIME_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_datetime) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_datetime(npy_datetime *p1, npy_intp l1, npy_datetime *p2, npy_intp l2,
                   npy_datetime *p3)
{
    npy_intp ofs;
    npy_datetime *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_datetime) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (DATETIME_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_datetime) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_datetime(const npy_datetime *arr, const npy_intp size, const npy_datetime key)
{
    npy_intp last_ofs, ofs, m;

    if (DATETIME_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (DATETIME_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (DATETIME_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_datetime(const npy_datetime *arr, const npy_intp size, const npy_datetime key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (DATETIME_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (DATETIME_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (DATETIME_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_datetime(npy_datetime *arr, const run *stack, const npy_intp at,
                buffer_datetime *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_datetime *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_datetime(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_datetime(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_datetime(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_datetime(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_datetime(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_datetime(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_datetime(npy_datetime *arr, run *stack, npy_intp *stack_ptr,
                    buffer_datetime *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_datetime(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_datetime(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_datetime(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_datetime(npy_datetime *arr, run *stack, npy_intp *stack_ptr,
                      buffer_datetime *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_datetime(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_datetime(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_datetime(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_datetime(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_datetime buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_datetime(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_datetime(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_datetime(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_datetime(npy_datetime *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_datetime vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!DATETIME_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !DATETIME_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && DATETIME_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && DATETIME_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_datetime(const npy_datetime *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_datetime key)
{
    npy_intp last_ofs, ofs, m;

    if (DATETIME_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (DATETIME_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (DATETIME_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_datetime(const npy_datetime *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_datetime key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (DATETIME_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (DATETIME_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (DATETIME_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_datetime(npy_datetime *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (DATETIME_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_datetime(npy_datetime *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (DATETIME_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_datetime(npy_datetime *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_datetime(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_datetime(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_datetime(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_datetime(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_datetime(npy_datetime *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_datetime(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_datetime(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_datetime(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_datetime(npy_datetime *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_datetime(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_datetime(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_datetime(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_datetime(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_datetime(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_datetime(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_datetime(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}


#line 113


typedef struct {
    npy_timedelta * pw;
    npy_intp size;
} buffer_timedelta;


static NPY_INLINE int
resize_buffer_timedelta(buffer_timedelta *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(new_size * sizeof(npy_timedelta));
    } else {
        buffer->pw = realloc(buffer->pw, new_size * sizeof(npy_timedelta));
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_timedelta(npy_timedelta *arr, npy_intp l, npy_intp num, npy_intp minrun)
{
    npy_intp sz;
    npy_timedelta vc, *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l;

    /* (not strictly) ascending sequence */
    if (!TIMEDELTA_LT(*(pl + 1), *pl)) {
        for (pi = pl + 1; pi < arr + num - 1 && !TIMEDELTA_LT(*(pi + 1), *pi); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < arr + num - 1 && TIMEDELTA_LT(*(pi + 1), *pi); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            TIMEDELTA_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vc = *pi;
            pj = pi;

            while (pl < pj && TIMEDELTA_LT(vc, *(pj - 1))) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vc;
        }
    }

    return sz;
}


/* when the left part of the array (p1) is smaller, copy p1 to buffer
 * and merge from left to right
 */
static void
merge_left_timedelta(npy_timedelta *p1, npy_intp l1, npy_timedelta *p2, npy_intp l2,
                  npy_timedelta *p3)
{
    npy_timedelta *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_timedelta) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (TIMEDELTA_LT(*p2, *p3)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_timedelta) * (p2 - p1));
    }
}


/* when the right part of the array (p2) is smaller, copy p2 to buffer
 * and merge from right to left
 */
static void
merge_right_timedelta(npy_timedelta *p1, npy_intp l1, npy_timedelta *p2, npy_intp l2,
                   npy_timedelta *p3)
{
    npy_intp ofs;
    npy_timedelta *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_timedelta) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (TIMEDELTA_LT(*p3, *p1)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_timedelta) * ofs);
    }
}


/* Note: the naming convention of gallop functions are different from that of
 * CPython. For example, here gallop_right means gallop from left toward right,
 * whereas in CPython gallop_right means gallop
 * and find the right most element among equal elements
 */
static npy_intp
gallop_right_timedelta(const npy_timedelta *arr, const npy_intp size, const npy_timedelta key)
{
    npy_intp last_ofs, ofs, m;

    if (TIMEDELTA_LT(key, arr[0])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (TIMEDELTA_LT(key, arr[ofs])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs] <= key < arr[ofs] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (TIMEDELTA_LT(key, arr[m])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[ofs-1] <= key < arr[ofs] */
    return ofs;
}


static npy_intp
gallop_left_timedelta(const npy_timedelta *arr, const npy_intp size, const npy_timedelta key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (TIMEDELTA_LT(arr[size - 1], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (TIMEDELTA_LT(arr[size - ofs - 1], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[size-ofs-1] < key <= arr[size-last_ofs-1] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (TIMEDELTA_LT(arr[m], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[r-1] < key <= arr[r] */
    return r;
}


static int
merge_at_timedelta(npy_timedelta *arr, const run *stack, const npy_intp at,
                buffer_timedelta *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_timedelta *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k].
     * if try to comment this out for debugging purpose, remember
     * in the merging process the first element is skipped
     */
    k = gallop_right_timedelta(arr + s1, l1, arr[s2]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + s1 + k;
    l1 -= k;
    p2 = arr + s2;
    /* arr[s2-1] belongs to arr[s2+l2] */
    l2 = gallop_left_timedelta(arr + s2, l2, arr[s2 - 1]);

    if (l2 < l1) {
        ret = resize_buffer_timedelta(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_timedelta(p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_timedelta(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_timedelta(p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
try_collapse_timedelta(npy_timedelta *arr, run *stack, npy_intp *stack_ptr,
                    buffer_timedelta *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_timedelta(arr, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_timedelta(arr, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_timedelta(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}

static int
force_collapse_timedelta(npy_timedelta *arr, run *stack, npy_intp *stack_ptr,
                      buffer_timedelta *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_timedelta(arr, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_timedelta(arr, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_timedelta(arr, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_timedelta(void *start, npy_intp num, void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_timedelta buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = count_run_timedelta(start, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_timedelta(start, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_timedelta(start, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    free(buffer.pw);

    return ret;
}


/* argsort */


static npy_intp
acount_run_timedelta(npy_timedelta *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun)
{
    npy_intp sz;
    npy_timedelta vc;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!TIMEDELTA_LT(arr[*(pl + 1)], arr[*pl])) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !TIMEDELTA_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && TIMEDELTA_LT(arr[*(pi + 1)], arr[*pi]); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            vc = arr[*pi];
            pj = pi;

            while (pl < pj && TIMEDELTA_LT(vc, arr[*(pj - 1)])) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_right_timedelta(const npy_timedelta *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_timedelta key)
{
    npy_intp last_ofs, ofs, m;

    if (TIMEDELTA_LT(key, arr[tosort[0]])) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (TIMEDELTA_LT(key, arr[tosort[ofs]])) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]] <= key < arr[tosort[ofs]] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (TIMEDELTA_LT(key, arr[tosort[m]])) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]] <= key < arr[tosort[ofs]] */
    return ofs;
}



static npy_intp
agallop_left_timedelta(const npy_timedelta *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_timedelta key)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (TIMEDELTA_LT(arr[tosort[size - 1]], key)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (TIMEDELTA_LT(arr[tosort[size - ofs - 1]], key)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]] < key <= arr[tosort[size-last_ofs-1]] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (TIMEDELTA_LT(arr[tosort[m]], key)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]] < key <= arr[tosort[r]] */
    return r;
}


static void
amerge_left_timedelta(npy_timedelta *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2,
                   npy_intp *p3)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (TIMEDELTA_LT(arr[*p2], arr[*p3])) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_timedelta(npy_timedelta *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2,
                    npy_intp *p3)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (TIMEDELTA_LT(arr[*p3], arr[*p1])) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}


static int
amerge_at_timedelta(npy_timedelta *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at,
                 buffer_intp *buffer)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_timedelta(arr, tosort + s1, l1, arr[tosort[s2]]);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_timedelta(arr, tosort + s2, l2, arr[tosort[s2 - 1]]);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_timedelta(arr, p1, l1, p2, l2, buffer->pw);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_timedelta(arr, p1, l1, p2, l2, buffer->pw);
    }

    return 0;
}


static int
atry_collapse_timedelta(npy_timedelta *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr,
                     buffer_intp *buffer)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_timedelta(arr, tosort, stack, top - 3, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_timedelta(arr, tosort, stack, top - 2, buffer);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_timedelta(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
aforce_collapse_timedelta(npy_timedelta *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr,
                       buffer_intp *buffer)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_timedelta(arr, tosort, stack, top - 3, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_timedelta(arr, tosort, stack, top - 2, buffer);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_timedelta(arr, tosort, stack, top - 2, buffer);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_timedelta(void *v, npy_intp *tosort, npy_intp num,
                void *NPY_UNUSED(varr))
{
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    buffer_intp buffer;
    run stack[TIMSORT_STACK_SIZE];
    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run(num);

    for (l = 0; l < num;) {
        n = acount_run_timedelta(v, tosort, l, num, minrun);
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_timedelta(v, tosort, stack, &stack_ptr, &buffer);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_timedelta(v, tosort, stack, &stack_ptr, &buffer);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;
cleanup:

    if (buffer.pw != NULL) {
        free(buffer.pw);
    }

    return ret;
}





/* For string sorts and generic sort, element comparisons are very expensive,
 * and the time cost of insertion sort (involves N**2 comparison) clearly hurts.
 * Implementing binary insertion sort and probably gallop mode during merging process
 * can hopefully boost the performance. Here as a temporary workaround we use shorter
 * run length to reduce the cost of insertion sort.
 */

static npy_intp compute_min_run_short(npy_intp num)
{
    npy_intp r = 0;

    while (16 < num) {
        r |= num & 1;
        num >>= 1;
    }

    return num + r;
}

/*
 *****************************************************************************
 **                             STRING SORTS                                **
 *****************************************************************************
 */


#line 930


typedef struct {
    npy_char * pw;
    npy_intp size;
    size_t len;
} buffer_string;


static NPY_INLINE int
resize_buffer_string(buffer_string *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(sizeof(npy_char) * new_size * buffer->len);
    } else {
        buffer->pw = realloc(buffer->pw,  sizeof(npy_char) * new_size * buffer->len);
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_string(npy_char *arr, npy_intp l, npy_intp num, npy_intp minrun,
                 npy_char *vp, size_t len)
{
    npy_intp sz;
    npy_char *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l * len;

    /* (not strictly) ascending sequence */
    if (!STRING_LT(pl + len, pl, len)) {
        for (pi = pl + len; pi < arr + (num - 1) * len
                && !STRING_LT(pi + len, pi, len); pi += len) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + len; pi < arr + (num - 1) * len
                && STRING_LT(pi + len, pi, len); pi += len) {
        }

        for (pj = pl, pr = pi; pj < pr; pj += len, pr -= len) {
            STRING_SWAP(pj, pr, len);
        }
    }

    pi += len;
    sz = (pi - pl) / len;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz * len;

        /* insertion sort */
        for (; pi < pr; pi += len) {
            STRING_COPY(vp, pi, len);
            pj = pi;

            while (pl < pj && STRING_LT(vp, pj - len, len)) {
                STRING_COPY(pj, pj - len, len);
                pj -= len;
            }

            STRING_COPY(pj, vp, len);
        }
    }

    return sz;
}


static npy_intp
gallop_right_string(const npy_char *arr, const npy_intp size,
                    const npy_char *key, size_t len)
{
    npy_intp last_ofs, ofs, m;

    if (STRING_LT(key, arr, len)) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (STRING_LT(key, arr + ofs * len, len)) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs*len] <= key < arr[ofs*len] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (STRING_LT(key, arr + m * len, len)) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[(ofs-1)*len] <= key < arr[ofs*len] */
    return ofs;
}



static npy_intp
gallop_left_string(const npy_char *arr, const npy_intp size, const npy_char *key,
                   size_t len)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (STRING_LT(arr + (size - 1) * len, key, len)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (STRING_LT(arr + (size - ofs - 1) * len, key, len)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[(size-ofs-1)*len] < key <= arr[(size-last_ofs-1)*len] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (STRING_LT(arr + m * len, key, len)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[(r-1)*len] < key <= arr[r*len] */
    return r;
}


static void
merge_left_string(npy_char *p1, npy_intp l1, npy_char *p2, npy_intp l2,
                  npy_char *p3, size_t len)
{
    npy_char *end = p2 + l2 * len;
    memcpy(p3, p1, sizeof(npy_char) * l1 * len);
    /* first element must be in p2 otherwise skipped in the caller */
    STRING_COPY(p1, p2, len);
    p1 += len;
    p2 += len;

    while (p1 < p2 && p2 < end) {
        if (STRING_LT(p2, p3, len)) {
            STRING_COPY(p1, p2, len);
            p1 += len;
            p2 += len;
        } else {
            STRING_COPY(p1, p3, len);
            p1 += len;
            p3 += len;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_char) * (p2 - p1));
    }
}


static void
merge_right_string(npy_char *p1, npy_intp l1, npy_char *p2, npy_intp l2,
                   npy_char *p3, size_t len)
{
    npy_intp ofs;
    npy_char *start = p1 - len;
    memcpy(p3, p2, sizeof(npy_char) * l2 * len);
    p1 += (l1 - 1) * len;
    p2 += (l2 - 1) * len;
    p3 += (l2 - 1) * len;
    /* first element must be in p1 otherwise skipped in the caller */
    STRING_COPY(p2, p1, len);
    p2 -= len;
    p1 -= len;

    while (p1 < p2 && start < p1) {
        if (STRING_LT(p3, p1, len)) {
            STRING_COPY(p2, p1, len);
            p2 -= len;
            p1 -= len;
        } else {
            STRING_COPY(p2, p3, len);
            p2 -= len;
            p3 -= len;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + len, p3 - ofs + len, sizeof(npy_char) * ofs);
    }
}


static int
merge_at_string(npy_char *arr, const run *stack, const npy_intp at,
                buffer_string *buffer, size_t len)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_char *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k] */
    STRING_COPY(buffer->pw, arr + s2 * len, len);
    k = gallop_right_string(arr + s1 * len, l1, buffer->pw, len);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + (s1 + k) * len;
    l1 -= k;
    p2 = arr + s2 * len;
    /* arr[s2-1] belongs to arr[s2+l2] */
    STRING_COPY(buffer->pw, arr + (s2 - 1) * len, len);
    l2 = gallop_left_string(arr + s2 * len, l2, buffer->pw, len);

    if (l2 < l1) {
        ret = resize_buffer_string(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_string(p1, l1, p2, l2, buffer->pw, len);
    } else {
        ret = resize_buffer_string(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_string(p1, l1, p2, l2, buffer->pw, len);
    }

    return 0;
}


static int
try_collapse_string(npy_char *arr, run *stack, npy_intp *stack_ptr,
                    buffer_string *buffer, size_t len)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_string(arr, stack, top - 3, buffer, len);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_string(arr, stack, top - 2, buffer, len);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_string(arr, stack, top - 2, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
force_collapse_string(npy_char *arr, run *stack, npy_intp *stack_ptr,
                      buffer_string *buffer, size_t len)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_string(arr, stack, top - 3, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_string(arr, stack, top - 2, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_string(arr, stack, top - 2, buffer, len);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_string(void *start, npy_intp num, void *varr)
{
    PyArrayObject *arr = varr;
    size_t elsize = PyArray_ITEMSIZE(arr);
    size_t len = elsize / sizeof(npy_char);
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    run stack[TIMSORT_STACK_SIZE];
    buffer_string buffer;

    /* Items that have zero size don't make sense to sort */
    if (len == 0) {
        return 0;
    }

    buffer.pw = NULL;
    buffer.size = 0;
    buffer.len = len;
    stack_ptr = 0;
    minrun = compute_min_run_short(num);
    /* used for insertion sort and gallop key */
    ret = resize_buffer_string(&buffer, 1);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    for (l = 0; l < num;) {
        n = count_run_string(start, l, num, minrun, buffer.pw, len);
        /* both s and l are scaled by len */
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_string(start, stack, &stack_ptr, &buffer, len);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_string(start, stack, &stack_ptr, &buffer, len);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;

cleanup:
    if (buffer.pw != NULL) {
        free(buffer.pw);
    }
    return ret;
}


/* argsort */


static npy_intp
acount_run_string(npy_char *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun, size_t len)
{
    npy_intp sz;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!STRING_LT(arr + (*(pl + 1)) * len, arr + (*pl) * len, len)) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !STRING_LT(arr + (*(pi + 1)) * len, arr + (*pi) * len, len); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && STRING_LT(arr + (*(pi + 1)) * len, arr + (*pi) * len, len); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            pj = pi;

            while (pl < pj && STRING_LT(arr + vi * len, arr + (*(pj - 1)) * len, len)) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_left_string(const npy_char *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_char *key, size_t len)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (STRING_LT(arr + tosort[size - 1] * len, key, len)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (STRING_LT(arr + tosort[size - ofs - 1] * len, key, len)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]*len] < key <= arr[tosort[size-last_ofs-1]*len] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (STRING_LT(arr + tosort[m] * len, key, len)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]*len] < key <= arr[tosort[r]*len] */
    return r;
}


static npy_intp
agallop_right_string(const npy_char *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_char *key, size_t len)
{
    npy_intp last_ofs, ofs, m;

    if (STRING_LT(key, arr + tosort[0] * len, len)) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (STRING_LT(key, arr + tosort[ofs] * len, len)) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]*len] <= key < arr[tosort[ofs]*len] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (STRING_LT(key, arr + tosort[m] * len, len)) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]*len] <= key < arr[tosort[ofs]*len] */
    return ofs;
}



static void
amerge_left_string(npy_char *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2, npy_intp *p3, size_t len)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (STRING_LT(arr + (*p2) * len, arr + (*p3) * len, len)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_string(npy_char *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2, npy_intp *p3, size_t len)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (STRING_LT(arr + (*p3) * len, arr + (*p1) * len, len)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}



static int
amerge_at_string(npy_char *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at, buffer_intp *buffer, size_t len)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_string(arr, tosort + s1, l1, arr + tosort[s2] * len, len);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_string(arr, tosort + s2, l2, arr + tosort[s2 - 1] * len,
                             len);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_string(arr, p1, l1, p2, l2, buffer->pw, len);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_string(arr, p1, l1, p2, l2, buffer->pw, len);
    }

    return 0;
}


static int
atry_collapse_string(npy_char *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr, buffer_intp *buffer, size_t len)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_string(arr, tosort, stack, top - 3, buffer, len);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_string(arr, tosort, stack, top - 2, buffer, len);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_string(arr, tosort, stack, top - 2, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}



static int
aforce_collapse_string(npy_char *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr, buffer_intp *buffer, size_t len)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_string(arr, tosort, stack, top - 3, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_string(arr, tosort, stack, top - 2, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_string(arr, tosort, stack, top - 2, buffer, len);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_string(void *start, npy_intp *tosort, npy_intp num, void *varr)
{
    PyArrayObject *arr = varr;
    size_t elsize = PyArray_ITEMSIZE(arr);
    size_t len = elsize / sizeof(npy_char);
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    run stack[TIMSORT_STACK_SIZE];
    buffer_intp buffer;

    /* Items that have zero size don't make sense to sort */
    if (len == 0) {
        return 0;
    }

    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run_short(num);

    for (l = 0; l < num;) {
        n = acount_run_string(start, tosort, l, num, minrun, len);
        /* both s and l are scaled by len */
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_string(start, tosort, stack, &stack_ptr, &buffer, len);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_string(start, tosort, stack, &stack_ptr, &buffer, len);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;

cleanup:
    if (buffer.pw != NULL) {
        free(buffer.pw);
    }
    return ret;
}



#line 930


typedef struct {
    npy_ucs4 * pw;
    npy_intp size;
    size_t len;
} buffer_unicode;


static NPY_INLINE int
resize_buffer_unicode(buffer_unicode *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(sizeof(npy_ucs4) * new_size * buffer->len);
    } else {
        buffer->pw = realloc(buffer->pw,  sizeof(npy_ucs4) * new_size * buffer->len);
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
count_run_unicode(npy_ucs4 *arr, npy_intp l, npy_intp num, npy_intp minrun,
                 npy_ucs4 *vp, size_t len)
{
    npy_intp sz;
    npy_ucs4 *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l * len;

    /* (not strictly) ascending sequence */
    if (!UNICODE_LT(pl + len, pl, len)) {
        for (pi = pl + len; pi < arr + (num - 1) * len
                && !UNICODE_LT(pi + len, pi, len); pi += len) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + len; pi < arr + (num - 1) * len
                && UNICODE_LT(pi + len, pi, len); pi += len) {
        }

        for (pj = pl, pr = pi; pj < pr; pj += len, pr -= len) {
            UNICODE_SWAP(pj, pr, len);
        }
    }

    pi += len;
    sz = (pi - pl) / len;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz * len;

        /* insertion sort */
        for (; pi < pr; pi += len) {
            UNICODE_COPY(vp, pi, len);
            pj = pi;

            while (pl < pj && UNICODE_LT(vp, pj - len, len)) {
                UNICODE_COPY(pj, pj - len, len);
                pj -= len;
            }

            UNICODE_COPY(pj, vp, len);
        }
    }

    return sz;
}


static npy_intp
gallop_right_unicode(const npy_ucs4 *arr, const npy_intp size,
                    const npy_ucs4 *key, size_t len)
{
    npy_intp last_ofs, ofs, m;

    if (UNICODE_LT(key, arr, len)) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (UNICODE_LT(key, arr + ofs * len, len)) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs*len] <= key < arr[ofs*len] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (UNICODE_LT(key, arr + m * len, len)) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[(ofs-1)*len] <= key < arr[ofs*len] */
    return ofs;
}



static npy_intp
gallop_left_unicode(const npy_ucs4 *arr, const npy_intp size, const npy_ucs4 *key,
                   size_t len)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (UNICODE_LT(arr + (size - 1) * len, key, len)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (UNICODE_LT(arr + (size - ofs - 1) * len, key, len)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[(size-ofs-1)*len] < key <= arr[(size-last_ofs-1)*len] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (UNICODE_LT(arr + m * len, key, len)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[(r-1)*len] < key <= arr[r*len] */
    return r;
}


static void
merge_left_unicode(npy_ucs4 *p1, npy_intp l1, npy_ucs4 *p2, npy_intp l2,
                  npy_ucs4 *p3, size_t len)
{
    npy_ucs4 *end = p2 + l2 * len;
    memcpy(p3, p1, sizeof(npy_ucs4) * l1 * len);
    /* first element must be in p2 otherwise skipped in the caller */
    UNICODE_COPY(p1, p2, len);
    p1 += len;
    p2 += len;

    while (p1 < p2 && p2 < end) {
        if (UNICODE_LT(p2, p3, len)) {
            UNICODE_COPY(p1, p2, len);
            p1 += len;
            p2 += len;
        } else {
            UNICODE_COPY(p1, p3, len);
            p1 += len;
            p3 += len;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_ucs4) * (p2 - p1));
    }
}


static void
merge_right_unicode(npy_ucs4 *p1, npy_intp l1, npy_ucs4 *p2, npy_intp l2,
                   npy_ucs4 *p3, size_t len)
{
    npy_intp ofs;
    npy_ucs4 *start = p1 - len;
    memcpy(p3, p2, sizeof(npy_ucs4) * l2 * len);
    p1 += (l1 - 1) * len;
    p2 += (l2 - 1) * len;
    p3 += (l2 - 1) * len;
    /* first element must be in p1 otherwise skipped in the caller */
    UNICODE_COPY(p2, p1, len);
    p2 -= len;
    p1 -= len;

    while (p1 < p2 && start < p1) {
        if (UNICODE_LT(p3, p1, len)) {
            UNICODE_COPY(p2, p1, len);
            p2 -= len;
            p1 -= len;
        } else {
            UNICODE_COPY(p2, p3, len);
            p2 -= len;
            p3 -= len;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + len, p3 - ofs + len, sizeof(npy_ucs4) * ofs);
    }
}


static int
merge_at_unicode(npy_ucs4 *arr, const run *stack, const npy_intp at,
                buffer_unicode *buffer, size_t len)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_ucs4 *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k] */
    UNICODE_COPY(buffer->pw, arr + s2 * len, len);
    k = gallop_right_unicode(arr + s1 * len, l1, buffer->pw, len);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + (s1 + k) * len;
    l1 -= k;
    p2 = arr + s2 * len;
    /* arr[s2-1] belongs to arr[s2+l2] */
    UNICODE_COPY(buffer->pw, arr + (s2 - 1) * len, len);
    l2 = gallop_left_unicode(arr + s2 * len, l2, buffer->pw, len);

    if (l2 < l1) {
        ret = resize_buffer_unicode(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_right_unicode(p1, l1, p2, l2, buffer->pw, len);
    } else {
        ret = resize_buffer_unicode(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        merge_left_unicode(p1, l1, p2, l2, buffer->pw, len);
    }

    return 0;
}


static int
try_collapse_unicode(npy_ucs4 *arr, run *stack, npy_intp *stack_ptr,
                    buffer_unicode *buffer, size_t len)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = merge_at_unicode(arr, stack, top - 3, buffer, len);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = merge_at_unicode(arr, stack, top - 2, buffer, len);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = merge_at_unicode(arr, stack, top - 2, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
force_collapse_unicode(npy_ucs4 *arr, run *stack, npy_intp *stack_ptr,
                      buffer_unicode *buffer, size_t len)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = merge_at_unicode(arr, stack, top - 3, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = merge_at_unicode(arr, stack, top - 2, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = merge_at_unicode(arr, stack, top - 2, buffer, len);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
timsort_unicode(void *start, npy_intp num, void *varr)
{
    PyArrayObject *arr = varr;
    size_t elsize = PyArray_ITEMSIZE(arr);
    size_t len = elsize / sizeof(npy_ucs4);
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    run stack[TIMSORT_STACK_SIZE];
    buffer_unicode buffer;

    /* Items that have zero size don't make sense to sort */
    if (len == 0) {
        return 0;
    }

    buffer.pw = NULL;
    buffer.size = 0;
    buffer.len = len;
    stack_ptr = 0;
    minrun = compute_min_run_short(num);
    /* used for insertion sort and gallop key */
    ret = resize_buffer_unicode(&buffer, 1);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    for (l = 0; l < num;) {
        n = count_run_unicode(start, l, num, minrun, buffer.pw, len);
        /* both s and l are scaled by len */
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = try_collapse_unicode(start, stack, &stack_ptr, &buffer, len);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = force_collapse_unicode(start, stack, &stack_ptr, &buffer, len);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;

cleanup:
    if (buffer.pw != NULL) {
        free(buffer.pw);
    }
    return ret;
}


/* argsort */


static npy_intp
acount_run_unicode(npy_ucs4 *arr, npy_intp *tosort, npy_intp l, npy_intp num,
                  npy_intp minrun, size_t len)
{
    npy_intp sz;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (!UNICODE_LT(arr + (*(pl + 1)) * len, arr + (*pl) * len, len)) {
        for (pi = pl + 1; pi < tosort + num - 1
                && !UNICODE_LT(arr + (*(pi + 1)) * len, arr + (*pi) * len, len); ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && UNICODE_LT(arr + (*(pi + 1)) * len, arr + (*pi) * len, len); ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            pj = pi;

            while (pl < pj && UNICODE_LT(arr + vi * len, arr + (*(pj - 1)) * len, len)) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
agallop_left_unicode(const npy_ucs4 *arr, const npy_intp *tosort,
                    const npy_intp size, const npy_ucs4 *key, size_t len)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (UNICODE_LT(arr + tosort[size - 1] * len, key, len)) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (UNICODE_LT(arr + tosort[size - ofs - 1] * len, key, len)) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]*len] < key <= arr[tosort[size-last_ofs-1]*len] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (UNICODE_LT(arr + tosort[m] * len, key, len)) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]*len] < key <= arr[tosort[r]*len] */
    return r;
}


static npy_intp
agallop_right_unicode(const npy_ucs4 *arr, const npy_intp *tosort,
                     const npy_intp size, const npy_ucs4 *key, size_t len)
{
    npy_intp last_ofs, ofs, m;

    if (UNICODE_LT(key, arr + tosort[0] * len, len)) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (UNICODE_LT(key, arr + tosort[ofs] * len, len)) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]*len] <= key < arr[tosort[ofs]*len] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (UNICODE_LT(key, arr + tosort[m] * len, len)) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]*len] <= key < arr[tosort[ofs]*len] */
    return ofs;
}



static void
amerge_left_unicode(npy_ucs4 *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                   npy_intp l2, npy_intp *p3, size_t len)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (UNICODE_LT(arr + (*p2) * len, arr + (*p3) * len, len)) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
amerge_right_unicode(npy_ucs4 *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                    npy_intp l2, npy_intp *p3, size_t len)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (UNICODE_LT(arr + (*p3) * len, arr + (*p1) * len, len)) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}



static int
amerge_at_unicode(npy_ucs4 *arr, npy_intp *tosort, const run *stack,
                 const npy_intp at, buffer_intp *buffer, size_t len)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = agallop_right_unicode(arr, tosort + s1, l1, arr + tosort[s2] * len, len);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = agallop_left_unicode(arr, tosort + s2, l2, arr + tosort[s2 - 1] * len,
                             len);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_right_unicode(arr, p1, l1, p2, l2, buffer->pw, len);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        amerge_left_unicode(arr, p1, l1, p2, l2, buffer->pw, len);
    }

    return 0;
}


static int
atry_collapse_unicode(npy_ucs4 *arr, npy_intp *tosort, run *stack,
                     npy_intp *stack_ptr, buffer_intp *buffer, size_t len)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = amerge_at_unicode(arr, tosort, stack, top - 3, buffer, len);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = amerge_at_unicode(arr, tosort, stack, top - 2, buffer, len);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = amerge_at_unicode(arr, tosort, stack, top - 2, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}



static int
aforce_collapse_unicode(npy_ucs4 *arr, npy_intp *tosort, run *stack,
                       npy_intp *stack_ptr, buffer_intp *buffer, size_t len)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = amerge_at_unicode(arr, tosort, stack, top - 3, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = amerge_at_unicode(arr, tosort, stack, top - 2, buffer, len);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = amerge_at_unicode(arr, tosort, stack, top - 2, buffer, len);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
atimsort_unicode(void *start, npy_intp *tosort, npy_intp num, void *varr)
{
    PyArrayObject *arr = varr;
    size_t elsize = PyArray_ITEMSIZE(arr);
    size_t len = elsize / sizeof(npy_ucs4);
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    run stack[TIMSORT_STACK_SIZE];
    buffer_intp buffer;

    /* Items that have zero size don't make sense to sort */
    if (len == 0) {
        return 0;
    }

    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run_short(num);

    for (l = 0; l < num;) {
        n = acount_run_unicode(start, tosort, l, num, minrun, len);
        /* both s and l are scaled by len */
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = atry_collapse_unicode(start, tosort, stack, &stack_ptr, &buffer, len);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = aforce_collapse_unicode(start, tosort, stack, &stack_ptr, &buffer, len);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;

cleanup:
    if (buffer.pw != NULL) {
        free(buffer.pw);
    }
    return ret;
}






/*
 *****************************************************************************
 **                             GENERIC SORT                                **
 *****************************************************************************
 */


typedef struct {
    char *pw;
    npy_intp size;
    size_t len;
} buffer_char;


static NPY_INLINE int
resize_buffer_char(buffer_char *buffer, npy_intp new_size)
{
    if (new_size <= buffer->size) {
        return 0;
    }

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        buffer->pw = malloc(sizeof(char) * new_size * buffer->len);
    } else {
        buffer->pw = realloc(buffer->pw,  sizeof(char) * new_size * buffer->len);
    }

    buffer->size = new_size;

    if (NPY_UNLIKELY(buffer->pw == NULL)) {
        return -NPY_ENOMEM;
    } else {
        return 0;
    }
}


static npy_intp
npy_count_run(char *arr, npy_intp l, npy_intp num, npy_intp minrun,
              char *vp, size_t len, PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    npy_intp sz;
    char *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = arr + l * len;

    /* (not strictly) ascending sequence */
    if (cmp(pl, pl + len, py_arr) <= 0) {
        for (pi = pl + len; pi < arr + (num - 1) * len
                && cmp(pi, pi + len, py_arr) <= 0; pi += len) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + len; pi < arr + (num - 1) * len
                && cmp(pi + len, pi, py_arr) < 0; pi += len) {
        }

        for (pj = pl, pr = pi; pj < pr; pj += len, pr -= len) {
            GENERIC_SWAP(pj, pr, len);
        }
    }

    pi += len;
    sz = (pi - pl) / len;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz * len;

        /* insertion sort */
        for (; pi < pr; pi += len) {
            GENERIC_COPY(vp, pi, len);
            pj = pi;

            while (pl < pj && cmp(vp, pj - len, py_arr) < 0) {
                GENERIC_COPY(pj, pj - len, len);
                pj -= len;
            }

            GENERIC_COPY(pj, vp, len);
        }
    }

    return sz;
}


static npy_intp
npy_gallop_right(const char *arr, const npy_intp size, const char *key,
                 size_t len, PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    npy_intp last_ofs, ofs, m;

    if (cmp(key, arr, py_arr) < 0) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (cmp(key, arr + ofs * len, py_arr) < 0) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[last_ofs*len] <= key < arr[ofs*len] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (cmp(key, arr + m * len, py_arr) < 0) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[(ofs-1)*len] <= key < arr[ofs*len] */
    return ofs;
}



static npy_intp
npy_gallop_left(const char *arr, const npy_intp size, const char *key,
                size_t len, PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (cmp(arr + (size - 1) * len, key, py_arr) < 0) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (cmp(arr + (size - ofs - 1) * len, key, py_arr) < 0) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[(size-ofs-1)*len] < key <= arr[(size-last_ofs-1)*len] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (cmp(arr + m * len, key, py_arr) < 0) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[(r-1)*len] < key <= arr[r*len] */
    return r;
}


static void
npy_merge_left(char *p1, npy_intp l1, char *p2, npy_intp l2,
               char *p3, size_t len,
               PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    char *end = p2 + l2 * len;
    memcpy(p3, p1, sizeof(char) * l1 * len);
    /* first element must be in p2 otherwise skipped in the caller */
    GENERIC_COPY(p1, p2, len);
    p1 += len;
    p2 += len;

    while (p1 < p2 && p2 < end) {
        if (cmp(p2, p3, py_arr) < 0) {
            GENERIC_COPY(p1, p2, len);
            p1 += len;
            p2 += len;
        } else {
            GENERIC_COPY(p1, p3, len);
            p1 += len;
            p3 += len;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(char) * (p2 - p1));
    }
}


static void
npy_merge_right(char *p1, npy_intp l1, char *p2, npy_intp l2,
                char *p3, size_t len,
                PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    npy_intp ofs;
    char *start = p1 - len;
    memcpy(p3, p2, sizeof(char) * l2 * len);
    p1 += (l1 - 1) * len;
    p2 += (l2 - 1) * len;
    p3 += (l2 - 1) * len;
    /* first element must be in p1 otherwise skipped in the caller */
    GENERIC_COPY(p2, p1, len);
    p2 -= len;
    p1 -= len;

    while (p1 < p2 && start < p1) {
        if (cmp(p3, p1, py_arr) < 0) {
            GENERIC_COPY(p2, p1, len);
            p2 -= len;
            p1 -= len;
        } else {
            GENERIC_COPY(p2, p3, len);
            p2 -= len;
            p3 -= len;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + len, p3 - ofs + len, sizeof(char) * ofs);
    }
}



static int
npy_merge_at(char *arr, const run *stack, const npy_intp at,
             buffer_char *buffer, size_t len,
             PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    char *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* arr[s2] belongs to arr[s1+k] */
    GENERIC_COPY(buffer->pw, arr + s2 * len, len);
    k = npy_gallop_right(arr + s1 * len, l1, buffer->pw, len, cmp, py_arr);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = arr + (s1 + k) * len;
    l1 -= k;
    p2 = arr + s2 * len;
    /* arr[s2-1] belongs to arr[s2+l2] */
    GENERIC_COPY(buffer->pw, arr + (s2 - 1) * len, len);
    l2 = npy_gallop_left(arr + s2 * len, l2, buffer->pw, len, cmp, py_arr);

    if (l2 < l1) {
        ret = resize_buffer_char(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        npy_merge_right(p1, l1, p2, l2, buffer->pw, len, cmp, py_arr);
    } else {
        ret = resize_buffer_char(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        npy_merge_left(p1, l1, p2, l2, buffer->pw, len, cmp, py_arr);
    }

    return 0;
}


static int
npy_try_collapse(char *arr, run *stack, npy_intp *stack_ptr,
                 buffer_char *buffer, size_t len,
                 PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = npy_merge_at(arr, stack, top - 3, buffer, len, cmp, py_arr);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = npy_merge_at(arr, stack, top - 2, buffer, len, cmp, py_arr);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = npy_merge_at(arr, stack, top - 2, buffer, len, cmp, py_arr);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
npy_force_collapse(char *arr, run *stack, npy_intp *stack_ptr,
                   buffer_char *buffer, size_t len,
                   PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = npy_merge_at(arr, stack, top - 3, buffer, len, cmp, py_arr);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = npy_merge_at(arr, stack, top - 2, buffer, len, cmp, py_arr);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = npy_merge_at(arr, stack, top - 2, buffer, len, cmp, py_arr);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
npy_timsort(void *start, npy_intp num, void *varr)
{
    PyArrayObject *arr = varr;
    size_t len = PyArray_ITEMSIZE(arr);
    PyArray_CompareFunc *cmp = PyArray_DESCR(arr)->f->compare;
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    run stack[TIMSORT_STACK_SIZE];
    buffer_char buffer;

    /* Items that have zero size don't make sense to sort */
    if (len == 0) {
        return 0;
    }

    buffer.pw = NULL;
    buffer.size = 0;
    buffer.len = len;
    stack_ptr = 0;
    minrun = compute_min_run_short(num);

    /* used for insertion sort and gallop key */
    ret = resize_buffer_char(&buffer, len);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    for (l = 0; l < num;) {
        n = npy_count_run(start, l, num, minrun, buffer.pw, len, cmp, arr);

        /* both s and l are scaled by len */
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = npy_try_collapse(start, stack, &stack_ptr, &buffer, len, cmp, arr);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = npy_force_collapse(start, stack, &stack_ptr, &buffer, len, cmp, arr);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;

cleanup:
    if (buffer.pw != NULL) {
        free(buffer.pw);
    }
    return ret;
}


/* argsort */

static npy_intp
npy_acount_run(char *arr, npy_intp *tosort, npy_intp l, npy_intp num,
               npy_intp minrun, size_t len,
               PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    npy_intp sz;
    npy_intp vi;
    npy_intp *pl, *pi, *pj, *pr;

    if (NPY_UNLIKELY(num - l == 1)) {
        return 1;
    }

    pl = tosort + l;

    /* (not strictly) ascending sequence */
    if (cmp(arr + (*pl) * len, arr + (*(pl + 1)) * len, py_arr) <= 0) {
        for (pi = pl + 1; pi < tosort + num - 1
                && cmp(arr + (*pi) * len, arr + (*(pi + 1)) * len, py_arr) <= 0; ++pi) {
        }
    } else {  /* (strictly) descending sequence */
        for (pi = pl + 1; pi < tosort + num - 1
                && cmp(arr + (*(pi + 1)) * len, arr + (*pi) * len, py_arr) < 0; ++pi) {
        }

        for (pj = pl, pr = pi; pj < pr; ++pj, --pr) {
            INTP_SWAP(*pj, *pr);
        }
    }

    ++pi;
    sz = pi - pl;

    if (sz < minrun) {
        if (l + minrun < num) {
            sz = minrun;
        } else {
            sz = num - l;
        }

        pr = pl + sz;

        /* insertion sort */
        for (; pi < pr; ++pi) {
            vi = *pi;
            pj = pi;

            while (pl < pj && cmp(arr + vi * len, arr + (*(pj - 1)) * len, py_arr) < 0) {
                *pj = *(pj - 1);
                --pj;
            }

            *pj = vi;
        }
    }

    return sz;
}


static npy_intp
npy_agallop_left(const char *arr, const npy_intp *tosort,
                 const npy_intp size, const char *key, size_t len,
                 PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    npy_intp last_ofs, ofs, l, m, r;

    if (cmp(arr + tosort[size - 1] * len, key, py_arr) < 0) {
        return size;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size;
            break;
        }

        if (cmp(arr + tosort[size - ofs - 1] * len, key, py_arr) < 0) {
            break;
        } else {
            last_ofs = ofs;
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[size-ofs-1]*len] < key <= arr[tosort[size-last_ofs-1]*len] */
    l = size - ofs - 1;
    r = size - last_ofs - 1;

    while (l + 1 < r) {
        m = l + ((r - l) >> 1);

        if (cmp(arr + tosort[m] * len, key, py_arr) < 0) {
            l = m;
        } else {
            r = m;
        }
    }

    /* now that arr[tosort[r-1]*len] < key <= arr[tosort[r]*len] */
    return r;
}


static npy_intp
npy_agallop_right(const char *arr, const npy_intp *tosort,
                  const npy_intp size, const char *key, size_t len,
                  PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    npy_intp last_ofs, ofs, m;

    if (cmp(key, arr + tosort[0] * len, py_arr) < 0) {
        return 0;
    }

    last_ofs = 0;
    ofs = 1;

    for (;;) {
        if (size <= ofs || ofs < 0) {
            ofs = size; /* arr[ofs] is never accessed */
            break;
        }

        if (cmp(key, arr + tosort[ofs] * len, py_arr) < 0) {
            break;
        } else {
            last_ofs = ofs;
            /* ofs = 1, 3, 7, 15... */
            ofs = (ofs << 1) + 1;
        }
    }

    /* now that arr[tosort[last_ofs]*len] <= key < arr[tosort[ofs]*len] */
    while (last_ofs + 1 < ofs) {
        m = last_ofs + ((ofs - last_ofs) >> 1);

        if (cmp(key, arr + tosort[m] * len, py_arr) < 0) {
            ofs = m;
        } else {
            last_ofs = m;
        }
    }

    /* now that arr[tosort[ofs-1]*len] <= key < arr[tosort[ofs]*len] */
    return ofs;
}


static void
npy_amerge_left(char *arr, npy_intp *p1, npy_intp l1, npy_intp *p2,
                npy_intp l2, npy_intp *p3, size_t len,
                PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    npy_intp *end = p2 + l2;
    memcpy(p3, p1, sizeof(npy_intp) * l1);
    /* first element must be in p2 otherwise skipped in the caller */
    *p1++ = *p2++;

    while (p1 < p2 && p2 < end) {
        if (cmp(arr + (*p2) * len, arr + (*p3) * len, py_arr) < 0) {
            *p1++ = *p2++;
        } else {
            *p1++ = *p3++;
        }
    }

    if (p1 != p2) {
        memcpy(p1, p3, sizeof(npy_intp) * (p2 - p1));
    }
}


static void
npy_amerge_right(char *arr, npy_intp* p1, npy_intp l1, npy_intp *p2,
                 npy_intp l2, npy_intp *p3, size_t len,
                 PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    npy_intp ofs;
    npy_intp *start = p1 - 1;
    memcpy(p3, p2, sizeof(npy_intp) * l2);
    p1 += l1 - 1;
    p2 += l2 - 1;
    p3 += l2 - 1;
    /* first element must be in p1 otherwise skipped in the caller */
    *p2-- = *p1--;

    while (p1 < p2 && start < p1) {
        if (cmp(arr + (*p3) * len, arr + (*p1) * len, py_arr) < 0) {
            *p2-- = *p1--;
        } else {
            *p2-- = *p3--;
        }
    }

    if (p1 != p2) {
        ofs = p2 - start;
        memcpy(start + 1, p3 - ofs + 1, sizeof(npy_intp) * ofs);
    }
}



static int
npy_amerge_at(char *arr, npy_intp *tosort, const run *stack,
              const npy_intp at, buffer_intp *buffer, size_t len,
              PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    int ret;
    npy_intp s1, l1, s2, l2, k;
    npy_intp *p1, *p2;
    s1 = stack[at].s;
    l1 = stack[at].l;
    s2 = stack[at + 1].s;
    l2 = stack[at + 1].l;
    /* tosort[s2] belongs to tosort[s1+k] */
    k = npy_agallop_right(arr, tosort + s1, l1, arr + tosort[s2] * len, len, cmp,
                          py_arr);

    if (l1 == k) {
        /* already sorted */
        return 0;
    }

    p1 = tosort + s1 + k;
    l1 -= k;
    p2 = tosort + s2;
    /* tosort[s2-1] belongs to tosort[s2+l2] */
    l2 = npy_agallop_left(arr, tosort + s2, l2, arr + tosort[s2 - 1] * len,
                          len, cmp, py_arr);

    if (l2 < l1) {
        ret = resize_buffer_intp(buffer, l2);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        npy_amerge_right(arr, p1, l1, p2, l2, buffer->pw, len, cmp, py_arr);
    } else {
        ret = resize_buffer_intp(buffer, l1);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }

        npy_amerge_left(arr, p1, l1, p2, l2, buffer->pw, len, cmp, py_arr);
    }

    return 0;
}


static int
npy_atry_collapse(char *arr, npy_intp *tosort, run *stack,
                  npy_intp *stack_ptr, buffer_intp *buffer, size_t len,
                  PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    int ret;
    npy_intp A, B, C, top;
    top = *stack_ptr;

    while (1 < top) {
        B = stack[top - 2].l;
        C = stack[top - 1].l;

        if ((2 < top && stack[top - 3].l <= B + C) ||
                (3 < top && stack[top - 4].l <= stack[top - 3].l + B)) {
            A = stack[top - 3].l;

            if (A <= C) {
                ret = npy_amerge_at(arr, tosort, stack, top - 3, buffer, len, cmp, py_arr);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 3].l += B;
                stack[top - 2] = stack[top - 1];
                --top;
            } else {
                ret = npy_amerge_at(arr, tosort, stack, top - 2, buffer, len, cmp, py_arr);

                if (NPY_UNLIKELY(ret < 0)) { return ret; }

                stack[top - 2].l += C;
                --top;
            }
        } else if (1 < top && B <= C) {
            ret = npy_amerge_at(arr, tosort, stack, top - 2, buffer, len, cmp, py_arr);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += C;
            --top;
        } else {
            break;
        }
    }

    *stack_ptr = top;
    return 0;
}


static int
npy_aforce_collapse(char *arr, npy_intp *tosort, run *stack,
                    npy_intp *stack_ptr, buffer_intp *buffer, size_t len,
                    PyArray_CompareFunc *cmp, PyArrayObject *py_arr)
{
    int ret;
    npy_intp top = *stack_ptr;

    while (2 < top) {
        if (stack[top - 3].l <= stack[top - 1].l) {
            ret = npy_amerge_at(arr, tosort, stack, top - 3, buffer, len, cmp, py_arr);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 3].l += stack[top - 2].l;
            stack[top - 2] = stack[top - 1];
            --top;
        } else {
            ret = npy_amerge_at(arr, tosort, stack, top - 2, buffer, len, cmp, py_arr);

            if (NPY_UNLIKELY(ret < 0)) { return ret; }

            stack[top - 2].l += stack[top - 1].l;
            --top;
        }
    }

    if (1 < top) {
        ret = npy_amerge_at(arr, tosort, stack, top - 2, buffer, len, cmp, py_arr);

        if (NPY_UNLIKELY(ret < 0)) { return ret; }
    }

    return 0;
}


NPY_NO_EXPORT int
npy_atimsort(void *start, npy_intp *tosort, npy_intp num, void *varr)
{
    PyArrayObject *arr = varr;
    size_t len = PyArray_ITEMSIZE(arr);
    PyArray_CompareFunc *cmp = PyArray_DESCR(arr)->f->compare;
    int ret;
    npy_intp l, n, stack_ptr, minrun;
    run stack[TIMSORT_STACK_SIZE];
    buffer_intp buffer;

    /* Items that have zero size don't make sense to sort */
    if (len == 0) {
        return 0;
    }

    buffer.pw = NULL;
    buffer.size = 0;
    stack_ptr = 0;
    minrun = compute_min_run_short(num);

    for (l = 0; l < num;) {
        n = npy_acount_run(start, tosort, l, num, minrun, len, cmp, arr);
        /* both s and l are scaled by len */
        stack[stack_ptr].s = l;
        stack[stack_ptr].l = n;
        ++stack_ptr;
        ret = npy_atry_collapse(start, tosort, stack, &stack_ptr, &buffer, len, cmp,
                                arr);

        if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

        l += n;
    }

    ret = npy_aforce_collapse(start, tosort, stack, &stack_ptr, &buffer, len,
                              cmp, arr);

    if (NPY_UNLIKELY(ret < 0)) { goto cleanup; }

    ret = 0;

cleanup:
    if (buffer.pw != NULL) {
        free(buffer.pw);
    }
    return ret;
}

