/*

* This file contains low-level loops for copying and byte-swapping

* strided data.

*

* Copyright (c) 2010 by Mark Wiebe ([email protected])

* The University of British Columbia

*

* See LICENSE.txt for the license.

*/

#define PY_SSIZE_T_CLEAN

#include <Python.h>

#define NPY_NO_DEPRECATED_API NPY_API_VERSION

#define _MULTIARRAYMODULE

#define _UMATHMODULE

#include <numpy/arrayobject.h>

#include <numpy/npy_cpu.h>

#include <numpy/halffloat.h>

#include <float.h>

#include "lowlevel_strided_loops.h"

#include "array_assign.h"

#include "array_method.h"

#include "usertypes.h"

#include "umathmodule.h"

#include "gil_utils.h"

/*

* x86 platform works with unaligned access but the compiler is allowed to

* assume all data is aligned to its size by the C standard. This means it can

* vectorize instructions peeling only by the size of the type, if the data is

* not aligned to this size one ends up with data not correctly aligned for SSE

* instructions (16 byte).

* So this flag can only be enabled if autovectorization is disabled.

*/

#define NPY_USE_UNALIGNED_ACCESS 0

#define _NPY_NOP1(x) (x)

#define _NPY_NOP2(x) (x)

#define _NPY_NOP4(x) (x)

#define _NPY_NOP8(x) (x)

#define _NPY_SWAP2(x) npy_bswap2(x)

#define _NPY_SWAP4(x) npy_bswap4(x)

#define _NPY_SWAP_PAIR4(x) (((((npy_uint32)x)&0xffu) << 8) | \

((((npy_uint32)x)&0xff00u) >> 8) | \

((((npy_uint32)x)&0xff0000u) << 8) | \

((((npy_uint32)x)&0xff000000u) >> 8))

#define _NPY_SWAP8(x) npy_bswap8(x)

#define _NPY_SWAP_PAIR8(x) (((((npy_uint64)x)&0xffULL) << 24) | \

((((npy_uint64)x)&0xff00ULL) << 8) | \

((((npy_uint64)x)&0xff0000ULL) >> 8) | \

((((npy_uint64)x)&0xff000000ULL) >> 24) | \

((((npy_uint64)x)&0xff00000000ULL) << 24) | \

((((npy_uint64)x)&0xff0000000000ULL) << 8) | \

((((npy_uint64)x)&0xff000000000000ULL) >> 8) | \

((((npy_uint64)x)&0xff00000000000000ULL) >> 24))

#define _NPY_SWAP_INPLACE2(x) npy_bswap2_unaligned(x)

#define _NPY_SWAP_INPLACE4(x) npy_bswap4_unaligned(x)

#define _NPY_SWAP_INPLACE8(x) npy_bswap8_unaligned(x)

#define _NPY_SWAP_INPLACE16(x) { \

char a = (x)[0]; (x)[0] = (x)[15]; (x)[15] = a; \

a = (x)[1]; (x)[1] = (x)[14]; (x)[14] = a; \

a = (x)[2]; (x)[2] = (x)[13]; (x)[13] = a; \

a = (x)[3]; (x)[3] = (x)[12]; (x)[12] = a; \

a = (x)[4]; (x)[4] = (x)[11]; (x)[11] = a; \

a = (x)[5]; (x)[5] = (x)[10]; (x)[10] = a; \

a = (x)[6]; (x)[6] = (x)[9]; (x)[9] = a; \

a = (x)[7]; (x)[7] = (x)[8]; (x)[8] = a; \

}

/************* STRIDED COPYING/SWAPPING SPECIALIZED FUNCTIONS *************/

/**begin repeat

* #elsize = 1, 2, 4, 8, 16#

* #elsize_half = 0, 1, 2, 4, 8#

* #type = npy_uint8, npy_uint16, npy_uint32, npy_uint64, npy_uint64#

*/

/**begin repeat1

* #oper = strided_to_strided, strided_to_contig,

* contig_to_strided, contig_to_contig#

* #src_contig = 0, 0, 1 ,1#

* #dst_contig = 0, 1, 0 ,1#

*/

/**begin repeat2

* #swap = _NPY_NOP, _NPY_NOP, _NPY_SWAP_INPLACE, _NPY_SWAP,

* _NPY_SWAP_INPLACE, _NPY_SWAP_PAIR#

* #prefix = , _aligned, _swap, _aligned_swap, _swap_pair, _aligned_swap_pair#

* #is_aligned = 0, 1, 0, 1, 0, 1#

* #minelsize = 1, 1, 2, 2, 4, 4#

* #is_swap = 0, 0, 1, 1, 2, 2#

*/

#if (@elsize@ >= @minelsize@) && \

(@elsize@ > 1 || @is_aligned@) && \

(!NPY_USE_UNALIGNED_ACCESS || @is_aligned@)

#if @is_swap@ || @src_contig@ == 0 || @dst_contig@ == 0

/*

* unrolling gains about 20-50% if the copy can be done in one mov instruction

* if not it can decrease performance

* tested to improve performance on intel xeon 5x/7x, core2duo, amd phenom x4

*/

static int

#if @is_aligned@ && @is_swap@ == 0 && @elsize@ <= NPY_SIZEOF_INTP

NPY_GCC_UNROLL_LOOPS

#endif

@prefix@_@oper@_size@elsize@(

PyArrayMethod_Context *context, char *const *args,

const npy_intp *dimensions, const npy_intp *strides,

NpyAuxData *NPY_UNUSED(auxdata))

{

npy_intp N = dimensions[0];

char *src = args[0], *dst = args[1];

#if !@src_contig@

npy_intp src_stride = strides[0];

#endif

#if !@dst_contig@

npy_intp dst_stride = strides[1];

#endif

#if @is_aligned@

/* sanity check */

assert(N == 0 || npy_is_aligned(dst, NPY_ALIGNOF_UINT(@type@)));

assert(N == 0 || npy_is_aligned(src, NPY_ALIGNOF_UINT(@type@)));

#endif

/*printf("fn @prefix@_@oper@_size@elsize@\n");*/

while (N > 0) {

#if @is_aligned@

/* aligned copy and swap */

# if @elsize@ != 16

(*((@type@ *)dst)) = @swap@@elsize@(*((@type@ *)src));

# else

# if @is_swap@ == 0

(*((npy_uint64 *)dst)) = (*((npy_uint64 *)src));

(*((npy_uint64 *)dst + 1)) = (*((npy_uint64 *)src + 1));

# elif @is_swap@ == 1

(*((npy_uint64 *)dst)) = _NPY_SWAP8(*((npy_uint64 *)src + 1));

(*((npy_uint64 *)dst + 1)) = _NPY_SWAP8(*((npy_uint64 *)src));

# elif @is_swap@ == 2

(*((npy_uint64 *)dst)) = _NPY_SWAP8(*((npy_uint64 *)src));

(*((npy_uint64 *)dst + 1)) = _NPY_SWAP8(*((npy_uint64 *)src + 1));

# endif

# endif

#else

/* unaligned copy and swap */

memmove(dst, src, @elsize@);

# if @is_swap@ == 1

@swap@@elsize@(dst);

# elif @is_swap@ == 2

@swap@@elsize_half@(dst);

@swap@@elsize_half@(dst + @elsize_half@);

# endif

#endif

#if @dst_contig@

dst += @elsize@;

#else

dst += dst_stride;

#endif

#if @src_contig@

src += @elsize@;

#else

src += src_stride;

#endif

--N;

}

return 0;

}

#endif

/*

* specialized copy and swap for source stride 0,

* interestingly unrolling here is like above is only marginally profitable for

* small types and detrimental for >= 8byte moves on x86

* but it profits from vectorization enabled with -O3

*/

#if (@src_contig@ == 0) && @is_aligned@

static NPY_GCC_OPT_3 int

@prefix@_@oper@_size@elsize@_srcstride0(

PyArrayMethod_Context *context, char *const *args,

const npy_intp *dimensions, const npy_intp *strides,

NpyAuxData *NPY_UNUSED(auxdata))

{

npy_intp N = dimensions[0];

char *src = args[0], *dst = args[1];

#if !@dst_contig@

npy_intp dst_stride = strides[1];

#endif

#if @elsize@ != 16

# if !(@elsize@ == 1 && @dst_contig@)

@type@ temp;

# endif

#else

npy_uint64 temp0, temp1;

#endif

if (N == 0) {

return 0;

}

#if @is_aligned@ && @elsize@ != 16

/* sanity check */

assert(N == 0 || npy_is_aligned(dst, NPY_ALIGNOF_UINT(@type@)));

assert(N == 0 || npy_is_aligned(src, NPY_ALIGNOF_UINT(@type@)));

#endif

#if @elsize@ == 1 && @dst_contig@

memset(dst, *src, N);

#else

# if @elsize@ != 16

temp = @swap@@elsize@(*((@type@ *)src));

# else

# if @is_swap@ == 0

temp0 = (*((npy_uint64 *)src));

temp1 = (*((npy_uint64 *)src + 1));

# elif @is_swap@ == 1

temp0 = _NPY_SWAP8(*((npy_uint64 *)src + 1));

temp1 = _NPY_SWAP8(*((npy_uint64 *)src));

# elif @is_swap@ == 2

temp0 = _NPY_SWAP8(*((npy_uint64 *)src));

temp1 = _NPY_SWAP8(*((npy_uint64 *)src + 1));

# endif

# endif

while (N > 0) {

# if @elsize@ != 16

*((@type@ *)dst) = temp;

# else

*((npy_uint64 *)dst) = temp0;

*((npy_uint64 *)dst + 1) = temp1;

# endif

# if @dst_contig@

dst += @elsize@;

# else

dst += dst_stride;

# endif

--N;

}

#endif/* @elsize == 1 && @dst_contig@ -- else */

return 0;

}

#endif/* (@src_contig@ == 0) && @is_aligned@ */

#endif/* @elsize@ >= @minelsize@ */

/**end repeat2**/

/**end repeat1**/

/**end repeat**/

static int

_strided_to_strided(

PyArrayMethod_Context *context, char *const *args,

const npy_intp *dimensions, const npy_intp *strides,

NpyAuxData *NPY_UNUSED(data))

{

npy_intp N = dimensions[0];

char *src = args[0], *dst = args[1];

npy_intp src_stride = strides[0], dst_stride = strides[1];

npy_intp src_itemsize = context->descriptors[0]->elsize;

while (N > 0) {

memmove(dst, src, src_itemsize);

dst += dst_stride;

src += src_stride;

--N;

}

return 0;

}

/*

* NOTE: This function is currently unused. It would currently be used for

* builtin dtypes that have an elsize other than 2, 4, 8, or 16 bytes.

* Since unicode and complex swap differently, no such dtype exists.

*/

static int

_swap_strided_to_strided(

PyArrayMethod_Context *context, char *const *args,

const npy_intp *dimensions, const npy_intp *strides,

NpyAuxData *NPY_UNUSED(data))

{

npy_intp N = dimensions[0];

char *src = args[0], *dst = args[1];

npy_intp src_stride = strides[0], dst_stride = strides[1];

npy_intp src_itemsize = context->descriptors[0]->elsize;

char *a, *b, c;

while (N > 0) {

memmove(dst, src, src_itemsize);

/* general in-place swap */

a = dst;

b = dst + src_itemsize - 1;

while (a < b) {

c = *a;

*a = *b;

*b = c;

++a; --b;

}

dst += dst_stride;

src += src_stride;

--N;

}

return 0;

}

static int

_swap_pair_strided_to_strided(

PyArrayMethod_Context *context, char *const *args,

const npy_intp *dimensions, const npy_intp *strides,

NpyAuxData *NPY_UNUSED(data))

{

npy_intp N = dimensions[0];

char *src = args[0], *dst = args[1];

npy_intp src_stride = strides[0], dst_stride = strides[1];

npy_intp src_itemsize = context->descriptors[0]->elsize;

char *a, *b, c;

npy_intp itemsize_half = src_itemsize / 2;

while (N > 0) {

memmove(dst, src, src_itemsize);

/* general in-place swap */

a = dst;

b = dst + itemsize_half - 1;

while (a < b) {

c = *a;

*a = *b;

*b = c;

++a; --b;

}

/* general in-place swap */

a = dst + itemsize_half;

b = dst + 2*itemsize_half - 1;

while (a < b) {

c = *a;

*a = *b;

*b = c;

++a; --b;

}

dst += dst_stride;

src += src_stride;

--N;

}

return 0;

}

static int

_contig_to_contig(

PyArrayMethod_Context *context, char *const *args,

const npy_intp *dimensions, const npy_intp *NPY_UNUSED(strides),

NpyAuxData *NPY_UNUSED(data))

{

npy_intp N = dimensions[0];

char *src = args[0], *dst = args[1];

npy_intp src_itemsize = context->descriptors[0]->elsize;

memmove(dst, src, src_itemsize*N);

return 0;

}

NPY_NO_EXPORT PyArrayMethod_StridedLoop *

PyArray_GetStridedCopyFn(int aligned, npy_intp src_stride,

npy_intp dst_stride, npy_intp itemsize)

{

/*

* Skip the "unaligned" versions on CPUs which support unaligned

* memory accesses.

*/

#if !NPY_USE_UNALIGNED_ACCESS

if (aligned) {

#endif/*!NPY_USE_UNALIGNED_ACCESS*/

/* contiguous dst */

if (itemsize != 0 && dst_stride == itemsize) {

/* constant src */

if (src_stride == 0) {

switch (itemsize) {

/**begin repeat

* #elsize = 1, 2, 4, 8, 16#

*/

case @elsize@:

return

&_aligned_strided_to_contig_size@elsize@_srcstride0;

/**end repeat**/

}

}

/* contiguous src */

else if (src_stride == itemsize) {

return &_contig_to_contig;

}

/* general src */

else {

switch (itemsize) {

/**begin repeat

* #elsize = 1, 2, 4, 8, 16#

*/

case @elsize@:

return &_aligned_strided_to_contig_size@elsize@;

/**end repeat**/

}

}

return &_strided_to_strided;

}

/* general dst */

else {

/* constant src */

if (src_stride == 0) {

switch (itemsize) {

/**begin repeat

* #elsize = 1, 2, 4, 8, 16#

*/

case @elsize@:

return

&_aligned_strided_to_strided_size@elsize@_srcstride0;

/**end repeat**/

}

}

/* contiguous src */

else if (src_stride == itemsize) {

switch (itemsize) {

/**begin repeat

* #elsize = 1, 2, 4, 8, 16#

*/

case @elsize@:

return &_aligned_contig_to_strided_size@elsize@;

/**end repeat**/

}

return &_strided_to_strided;

}

else {

switch (itemsize) {

/**begin repeat

* #elsize = 1, 2, 4, 8, 16#

*/

case @elsize@:

return &_aligned_strided_to_strided_size@elsize@;

/**end repeat**/

}

}

}

#if !NPY_USE_UNALIGNED_ACCESS

}

else {

if (itemsize != 0) {

if (dst_stride == itemsize) {

/* contiguous dst */

if (src_stride == itemsize) {

/* contiguous src, dst */

return &_contig_to_contig;

}

else {

/* general src */

switch (itemsize) {

case 1:

return &_aligned_strided_to_contig_size1;

/**begin repeat

* #elsize = 2, 4, 8, 16#

*/

case @elsize@:

return &_strided_to_contig_size@elsize@;

/**end repeat**/

}

}

return &_strided_to_strided;

}

else if (src_stride == itemsize) {

/* contiguous src, general dst */

switch (itemsize) {

case 1:

return &_aligned_contig_to_strided_size1;

/**begin repeat

* #elsize = 2, 4, 8, 16#

*/

case @elsize@:

return &_contig_to_strided_size@elsize@;

/**end repeat**/

}

return &_strided_to_strided;

}

}

else {

/* general src, dst */

switch (itemsize) {

case 1:

return &_aligned_strided_to_strided_size1;

/**begin repeat

* #elsize = 2, 4, 8, 16#

*/

case @elsize@:

return &_strided_to_strided_size@elsize@;

/**end repeat**/

}

}

}

#endif/*!NPY_USE_UNALIGNED_ACCESS*/

return &_strided_to_strided;

}

/*

* PyArray_GetStridedCopySwapFn and PyArray_GetStridedCopySwapPairFn are

* nearly identical, so can do a repeat for them.

*/

/**begin repeat

* #function = PyArray_GetStridedCopySwapFn, PyArray_GetStridedCopySwapPairFn#

* #tag = , _pair#

* #not_pair = 1, 0#

*/

NPY_NO_EXPORT PyArrayMethod_StridedLoop *

@function@(int aligned, npy_intp src_stride,

npy_intp dst_stride, npy_intp itemsize)

{

/*

* Skip the "unaligned" versions on CPUs which support unaligned

* memory accesses.

*/

#if !NPY_USE_UNALIGNED_ACCESS

if (aligned) {

#endif/*!NPY_USE_UNALIGNED_ACCESS*/

/* contiguous dst */

if (itemsize != 0 && dst_stride == itemsize) {

/* constant src */

if (src_stride == 0) {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return

&_aligned_swap@tag@_strided_to_contig_size@elsize@_srcstride0;

#endif

/**end repeat1**/

}

}

/* contiguous src */

else if (src_stride == itemsize) {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return &_aligned_swap@tag@_contig_to_contig_size@elsize@;

#endif

/**end repeat1**/

}

}

/* general src */

else {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return &_aligned_swap@tag@_strided_to_contig_size@elsize@;

#endif

/**end repeat1**/

}

}

}

/* general dst */

else {

/* constant src */

if (src_stride == 0) {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return

&_aligned_swap@tag@_strided_to_strided_size@elsize@_srcstride0;

#endif

/**end repeat1**/

}

}

/* contiguous src */

else if (src_stride == itemsize) {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return &_aligned_swap@tag@_contig_to_strided_size@elsize@;

#endif

/**end repeat1**/

}

return &_swap@tag@_strided_to_strided;

}

else {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return &_aligned_swap@tag@_strided_to_strided_size@elsize@;

#endif

/**end repeat1**/

}

}

}

#if !NPY_USE_UNALIGNED_ACCESS

}

else {

/* contiguous dst */

if (itemsize != 0 && dst_stride == itemsize) {

/* contiguous src */

if (src_stride == itemsize) {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return &_swap@tag@_contig_to_contig_size@elsize@;

#endif

/**end repeat1**/

}

}

/* general src */

else {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return &_swap@tag@_strided_to_contig_size@elsize@;

#endif

/**end repeat1**/

}

}

return &_swap@tag@_strided_to_strided;

}

/* general dst */

else {

/* contiguous src */

if (itemsize != 0 && src_stride == itemsize) {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return &_swap@tag@_contig_to_strided_size@elsize@;

#endif

/**end repeat1**/

}

return &_swap@tag@_strided_to_strided;

}

/* general src */

else {

switch (itemsize) {

/**begin repeat1

* #elsize = 2, 4, 8, 16#

*/

#if @not_pair@ || @elsize@ > 2

case @elsize@:

return &_swap@tag@_strided_to_strided_size@elsize@;

#endif

/**end repeat1**/

}

}

}

}

#endif/*!NPY_USE_UNALIGNED_ACCESS*/

return &_swap@tag@_strided_to_strided;

}

/**end repeat**/

/************* STRIDED CASTING SPECIALIZED FUNCTIONS *************/

#if defined(NPY_HAVE_NEON_FP16)

#define EMULATED_FP16 0

#define NATIVE_FP16 1

typedef _Float16 _npy_half;

#else

#define EMULATED_FP16 1

#define NATIVE_FP16 0

typedef npy_half _npy_half;

#endif

#if EMULATED_FP16

/* half-to-double, copied from CPP to allow inlining */

static NPY_GCC_OPT_3

uint64_t ToDoubleBits(uint16_t h)

{

uint16_t h_exp = (h&0x7c00u);

uint64_t d_sgn = ((uint64_t)h&0x8000u) << 48;

switch (h_exp) {

case 0x0000u: { // 0 or subnormal

uint16_t h_sig = (h&0x03ffu);

// Signed zero

if (h_sig == 0) {

return d_sgn;

}

// Subnormal

h_sig <<= 1;

while ((h_sig&0x0400u) == 0) {

h_sig <<= 1;

h_exp++;

}

uint64_t d_exp = ((uint64_t)(1023 - 15 - h_exp)) << 52;

uint64_t d_sig = ((uint64_t)(h_sig&0x03ffu)) << 42;

return d_sgn + d_exp + d_sig;

}

case 0x7c00u: // inf or NaN

// All-ones exponent and a copy of the significand

return d_sgn + 0x7ff0000000000000ULL + (((uint64_t)(h&0x03ffu)) << 42);

default: // normalized

// Just need to adjust the exponent and shift

return d_sgn + (((uint64_t)(h&0x7fffu) + 0xfc000u) << 42);

}

}

NPY_FINLINE

npy_uint64 _npy_halfbits_to_doublebits(npy_uint16 h){

/*

* Use npymath versions for all the special cases, only inline the

* x86_64 non-intrinsic case. Someday we will rewrite this in CPP and

* can then explore inlining more

*/

#if defined(NPY_HAVE_AVX512FP16)

return npy_halfbits_to_doublebits(h);

#elif defined(NPY_HAVE_VSX3) && defined(NPY_HAVE_VSX3_HALF_DOUBLE)

return npy_halfbits_to_doublebits(h);

#elif defined(__ARM_FP16_FORMAT_IEEE)

return npy_halfbits_to_doublebits(h);

#else

return ToDoubleBits(h);

#endif

}

#endif

/**begin repeat

*

* #NAME1 = BOOL,

* UBYTE, USHORT, UINT, ULONG, ULONGLONG,

* BYTE, SHORT, INT, LONG, LONGLONG,

* HALF, FLOAT, DOUBLE, LONGDOUBLE,

* CFLOAT, CDOUBLE, CLONGDOUBLE#

* #name1 = bool,

* ubyte, ushort, uint, ulong, ulonglong,

* byte, short, int, long, longlong,

* half, float, double, longdouble,

* cfloat, cdouble, clongdouble#

* #type1 = npy_bool,

* npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong,

* npy_byte, npy_short, npy_int, npy_long, npy_longlong,

* _npy_half, npy_float, npy_double, npy_longdouble,

* npy_cfloat, npy_cdouble, npy_clongdouble#

* #rtype1 = npy_bool,

* npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong,

* npy_byte, npy_short, npy_int, npy_long, npy_longlong,

* _npy_half, npy_float, npy_double, npy_longdouble,

* npy_float, npy_double, npy_longdouble#

* #is_bool1 = 1, 0*17#

* #is_emu_half1 = 0*11, EMULATED_FP16, 0*6#

* #is_native_half1 = 0*11, NATIVE_FP16, 0*6#

* #is_float1 = 0*12, 1, 0, 0, 1, 0, 0#

* #is_double1 = 0*13, 1, 0, 0, 1, 0#

* #is_complex1 = 0*15, 1*3#

* #is_unsigned1 = 1*6, 0*12#

*/

/**begin repeat1

*

* #NAME2 = BOOL,

* UBYTE, USHORT, UINT, ULONG, ULONGLONG,

* BYTE, SHORT, INT, LONG, LONGLONG,

* HALF, FLOAT, DOUBLE, LONGDOUBLE,

* CFLOAT, CDOUBLE, CLONGDOUBLE#

* #name2 = bool,

* ubyte, ushort, uint, ulong, ulonglong,

* byte, short, int, long, longlong,

* half, float, double, longdouble,

* cfloat, cdouble, clongdouble#

* #type2 = npy_bool,

* npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong,

* npy_byte, npy_short, npy_int, npy_long, npy_longlong,

* _npy_half, npy_float, npy_double, npy_longdouble,

* npy_cfloat, npy_cdouble, npy_clongdouble#

* #rtype2 = npy_bool,

* npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong,

* npy_byte, npy_short, npy_int, npy_long, npy_longlong,

* _npy_half, npy_float, npy_double, npy_longdouble,

* npy_float, npy_double, npy_longdouble#

* #type2max = 0,

* UCHAR_MAX, USHRT_MAX, UINT_MAX, ULONG_MAX, ULLONG_MAX,

* SCHAR_MAX, SHRT_MAX, INT_MAX, LONG_MAX, LLONG_MAX,

* 65500.0f, FLT_MAX, DBL_MAX, LDBL_MAX,

* FLT_MAX, DBL_MAX, LDBL_MAX#

* #type2min = 0,

* 0, 0, 0, 0, 0,

* SCHAR_MIN, SHRT_MIN, INT_MIN, LONG_MIN, LLONG_MIN,

* -65500.0f, -FLT_MAX, -DBL_MAX, -LDBL_MAX,

* -FLT_MAX, -DBL_MAX, -LDBL_MAX#

* #is_bool2 = 1, 0*17#

* #is_emu_half2 = 0*11, EMULATED_FP16, 0*6#

* #is_native_half2 = 0*11, NATIVE_FP16, 0*6#

* #is_float2 = 0*12, 1, 0, 0, 1, 0, 0#

* #is_double2 = 0*13, 1, 0, 0, 1, 0#

* #is_complex2 = 0*15, 1*3#

*/

/**begin repeat2

* #prefix = _aligned,,_aligned_contig,_contig#

* #aligned = 1,0,1,0#

* #contig = 0,0,1,1#

*/

#if !(NPY_USE_UNALIGNED_ACCESS && !@aligned@)

/*

* For emulated half types, don't use actual double/float types in conversion

* except for *_check_same_value_*(), follow _ROUND_TRIP and _TO_RTYPE1.

*/

#if @is_emu_half1@ || @is_emu_half2@

# if @is_float1@

# define _TYPE1 npy_uint32

# elif @is_double1@

# define _TYPE1 npy_uint64

# else

# define _TYPE1 @rtype1@

# endif

# if @is_float2@

# define _TYPE2 npy_uint32

# elif @is_double2@

# define _TYPE2 npy_uint64

# else

# define _TYPE2 @rtype2@

# endif

#else

#define _TYPE1 @rtype1@

#define _TYPE2 @rtype2@

#endif

/* Determine an appropriate casting conversion function */

#if @is_emu_half1@

# define _TO_RTYPE1(x) npy_half_to_float(x)

# if @is_float2@

# define _CONVERT_FN(x) npy_halfbits_to_floatbits(x)

# define _ROUND_TRIP(x) npy_floatbits_to_halfbits(_CONVERT_FN(x))

# elif @is_double2@

# define _CONVERT_FN(x) _npy_halfbits_to_doublebits(x)

# define _ROUND_TRIP(x) npy_doublebits_to_halfbits(_CONVERT_FN(x))

# elif @is_emu_half2@

# define _CONVERT_FN(x) (x)

# define _ROUND_TRIP(x) (x)

# elif @is_bool2@

# define _CONVERT_FN(x) ((npy_bool)!npy_half_iszero(x))

# define _ROUND_TRIP(x) npy_float_to_half((float)(!npy_half_iszero(x)))

# else

# define _CONVERT_FN(x) ((_TYPE2)npy_half_to_float(x))

# define _ROUND_TRIP(x) npy_float_to_half((float)_CONVERT_FN(x))

# endif

#elif @is_emu_half2@

# define _TO_RTYPE1(x) (@rtype1@)(x)

# if @is_float1@

# define _CONVERT_FN(x) npy_floatbits_to_halfbits(x)

# define _ROUND_TRIP(x) npy_half_to_float(npy_float_to_half(x))

# elif @is_double1@

# define _CONVERT_FN(x) npy_doublebits_to_halfbits(x)

# define _ROUND_TRIP(x) npy_half_to_double(npy_double_to_half(x))

# elif @is_emu_half1@

# define _CONVERT_FN(x) (x)

# define _ROUND_TRIP(x) (x)

# elif @is_bool1@

# define _CONVERT_FN(x) npy_float_to_half((float)(x!=0))

# define _ROUND_TRIP(x) (x)

# else

# define _CONVERT_FN(x) npy_float_to_half((float)x)

# define _ROUND_TRIP(x) ((@rtype1@)npy_half_to_float(_CONVERT_FN(x)))

# endif

#else

# if @is_bool2@ || @is_bool1@

# define _CONVERT_FN(x) ((npy_bool)(x != 0))

# else

# define _CONVERT_FN(x) ((_TYPE2)x)

# endif

# define _TO_RTYPE1(x) (@rtype1@)(x)

# define _ROUND_TRIP(x) _TO_RTYPE1(_CONVERT_FN(x))

#endif

// Enable auto-vectorization for floating point casts with clang

#if @is_native_half1@ || @is_float1@ || @is_double1@

#if @is_native_half2@ || @is_float2@ || @is_double2@

#if defined(__clang__) && !defined(__EMSCRIPTEN__) && !defined(__wasm__)

#if __clang_major__ >= 12

_Pragma("clang fp exceptions(ignore)")

#endif

#endif

#endif

#endif

// Work around GCC bug for double->half casts. For SVE and

// OPT_LEVEL > 1, it implements this as double->single->half

// which is incorrect as it introduces double rounding with

// narrowing casts.

#if (@is_double1@ && @is_native_half2@) && \

defined(NPY_HAVE_SVE) && defined(__GNUC__)

#define GCC_CAST_OPT_LEVEL __attribute__((optimize("O1")))

#else

#define GCC_CAST_OPT_LEVEL NPY_GCC_OPT_3

#endif

#define _RETURN_SAME_VALUE_FAILURE \

npy_gil_error(PyExc_ValueError, "could not cast 'same_value' @name1@ to @name2@"); \

return -1

#if !@is_bool2@

/*

* Check various modes of failure to accurately cast src_value to dst

*/

static GCC_CAST_OPT_LEVEL int

@prefix@_check_same_value_@name1@_to_@name2@(@rtype1@ *src_valueP) {

@rtype1@ src_value = *src_valueP;

/* 1. NaN/Infs always work for float to float and otherwise never */

#if (@is_float1@ || @is_emu_half1@ || @is_double1@ || @is_native_half1@)

if (!npy_isfinite(_TO_RTYPE1(src_value))) {

# if (@is_float2@ || @is_emu_half2@ || @is_double2@ || @is_native_half2@)

return 0; /* float to float can preserve NaN/Inf */

# else

_RETURN_SAME_VALUE_FAILURE; /* cannot preserve NaN/Inf */

# endif

}

#endif

/*

* 2. Check that the src does not overflow the dst.

* This is complicated by a warning that, for instance, int8 cannot

* overflow int64max

*/

# ifdef __GNUC__

# pragma GCC diagnostic push

# ifdef __clang__

# pragma GCC diagnostic ignored "-Wtautological-constant-out-of-range-compare"

# pragma GCC diagnostic ignored "-Wimplicit-const-int-float-conversion"

# endif

# pragma GCC diagnostic ignored "-Wtautological-compare"

# endif

# if !@is_bool1@

if (_TO_RTYPE1(src_value) > @type2max@) {

_RETURN_SAME_VALUE_FAILURE;

}

# if !@is_unsigned1@

if (_TO_RTYPE1(src_value) < @type2min@) {

_RETURN_SAME_VALUE_FAILURE;

}

# endif

# endif /* !is_bool1 */

/* 3. Check that the value can round trip exactly */

if (src_value != _ROUND_TRIP(src_value)) {

_RETURN_SAME_VALUE_FAILURE;