mirror of
https://github.com/KartKrewDev/RingRacers.git
synced 2025-10-30 08:01:28 +00:00
Update libdivide
This commit is contained in:
Eidolon
parent
aadddc291c
commit
ffc0ce0a72
1 changed files with 212 additions and 106 deletions
318
src/libdivide.h
318
src/libdivide.h
|
|
@ -11,37 +11,56 @@
|
|||
#ifndef LIBDIVIDE_H
|
||||
#define LIBDIVIDE_H
|
||||
|
||||
#define LIBDIVIDE_VERSION "5.0"
|
||||
// *** Version numbers are auto generated - do not edit ***
|
||||
#define LIBDIVIDE_VERSION "5.2.0"
|
||||
#define LIBDIVIDE_VERSION_MAJOR 5
|
||||
#define LIBDIVIDE_VERSION_MINOR 0
|
||||
#define LIBDIVIDE_VERSION_MINOR 2
|
||||
#define LIBDIVIDE_VERSION_PATCH 0
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#if !defined(__AVR__)
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) && (defined(__cplusplus) && (__cplusplus >= 202002L)) || \
|
||||
(defined(_MSVC_LANG) && (_MSVC_LANG >= 202002L))
|
||||
#include <limits.h>
|
||||
#include <type_traits>
|
||||
#define LIBDIVIDE_VC_CXX20
|
||||
#endif
|
||||
|
||||
#if defined(LIBDIVIDE_SSE2)
|
||||
#include <emmintrin.h>
|
||||
#endif
|
||||
|
||||
#if defined(LIBDIVIDE_AVX2) || defined(LIBDIVIDE_AVX512)
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
|
||||
#if defined(LIBDIVIDE_NEON)
|
||||
#include <arm_neon.h>
|
||||
#endif
|
||||
|
||||
// Clang-cl prior to Visual Studio 2022 doesn't include __umulh/__mulh intrinsics
|
||||
#if defined(_MSC_VER) && (!defined(__clang__) || _MSC_VER > 1930) && \
|
||||
(defined(_M_X64) || defined(_M_ARM64) || defined(_M_HYBRID_X86_ARM64) || defined(_M_ARM64EC))
|
||||
#define LIBDIVIDE_MULH_INTRINSICS
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#if defined(LIBDIVIDE_MULH_INTRINSICS) || !defined(__clang__)
|
||||
#include <intrin.h>
|
||||
#endif
|
||||
#ifndef __clang__
|
||||
#pragma warning(push)
|
||||
// disable warning C4146: unary minus operator applied
|
||||
// to unsigned type, result still unsigned
|
||||
// 4146: unary minus operator applied to unsigned type, result still unsigned
|
||||
#pragma warning(disable : 4146)
|
||||
// disable warning C4204: nonstandard extension used : non-constant aggregate
|
||||
// initializer
|
||||
//
|
||||
// It's valid C99
|
||||
|
||||
// 4204: nonstandard extension used : non-constant aggregate initializer
|
||||
#pragma warning(disable : 4204)
|
||||
#endif
|
||||
#define LIBDIVIDE_VC
|
||||
#endif
|
||||
|
||||
|
|
@ -83,8 +102,12 @@
|
|||
#endif
|
||||
#endif
|
||||
#ifndef LIBDIVIDE_INLINE
|
||||
#ifdef _MSC_VER
|
||||
#define LIBDIVIDE_INLINE __forceinline
|
||||
#else
|
||||
#define LIBDIVIDE_INLINE inline
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(__AVR__)
|
||||
#define LIBDIVIDE_ERROR(msg)
|
||||
|
|
@ -110,9 +133,81 @@
|
|||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
// For constexpr zero initialization, c++11 might handle things ok,
|
||||
// but just limit to at least c++14 to ensure we don't break anyone's code:
|
||||
|
||||
// Use https://en.cppreference.com/w/cpp/feature_test#cpp_constexpr
|
||||
#if defined(__cpp_constexpr) && (__cpp_constexpr >= 201304L)
|
||||
#define LIBDIVIDE_CONSTEXPR constexpr LIBDIVIDE_INLINE
|
||||
|
||||
// Supposedly, MSVC might not implement feature test macros right:
|
||||
// https://stackoverflow.com/questions/49316752/feature-test-macros-not-working-properly-in-visual-c
|
||||
// so check that _MSVC_LANG corresponds to at least c++14, and _MSC_VER corresponds to at least VS
|
||||
// 2017 15.0 (for extended constexpr support:
|
||||
// https://learn.microsoft.com/en-us/cpp/overview/visual-cpp-language-conformance?view=msvc-170)
|
||||
#elif (defined(_MSC_VER) && _MSC_VER >= 1910) && (defined(_MSVC_LANG) && _MSVC_LANG >= 201402L)
|
||||
#define LIBDIVIDE_CONSTEXPR constexpr LIBDIVIDE_INLINE
|
||||
|
||||
#else
|
||||
#define LIBDIVIDE_CONSTEXPR LIBDIVIDE_INLINE
|
||||
#endif
|
||||
|
||||
namespace libdivide {
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
#if defined(LIBDIVIDE_VC_CXX20)
|
||||
static LIBDIVIDE_CONSTEXPR int __builtin_clz(unsigned x) {
|
||||
if (std::is_constant_evaluated()) {
|
||||
for (int i = 0; i < sizeof(x) * CHAR_BIT; ++i) {
|
||||
if (x >> (sizeof(x) * CHAR_BIT - 1 - i)) return i;
|
||||
}
|
||||
return sizeof(x) * CHAR_BIT;
|
||||
}
|
||||
#else
|
||||
static LIBDIVIDE_INLINE int __builtin_clz(unsigned x) {
|
||||
#endif
|
||||
#if defined(_M_ARM) || defined(_M_ARM64) || defined(_M_HYBRID_X86_ARM64) || defined(_M_ARM64EC)
|
||||
return (int)_CountLeadingZeros(x);
|
||||
#elif defined(__AVX2__) || defined(__LZCNT__)
|
||||
return (int)_lzcnt_u32(x);
|
||||
#else
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, x);
|
||||
return (int)(r ^ 31);
|
||||
#endif
|
||||
}
|
||||
|
||||
#if defined(LIBDIVIDE_VC_CXX20)
|
||||
static LIBDIVIDE_CONSTEXPR int __builtin_clzll(unsigned long long x) {
|
||||
if (std::is_constant_evaluated()) {
|
||||
for (int i = 0; i < sizeof(x) * CHAR_BIT; ++i) {
|
||||
if (x >> (sizeof(x) * CHAR_BIT - 1 - i)) return i;
|
||||
}
|
||||
return sizeof(x) * CHAR_BIT;
|
||||
}
|
||||
#else
|
||||
static LIBDIVIDE_INLINE int __builtin_clzll(unsigned long long x) {
|
||||
#endif
|
||||
#if defined(_M_ARM) || defined(_M_ARM64) || defined(_M_HYBRID_X86_ARM64) || defined(_M_ARM64EC)
|
||||
return (int)_CountLeadingZeros64(x);
|
||||
#elif defined(_WIN64)
|
||||
#if defined(__AVX2__) || defined(__LZCNT__)
|
||||
return (int)_lzcnt_u64(x);
|
||||
#else
|
||||
unsigned long r;
|
||||
_BitScanReverse64(&r, x);
|
||||
return (int)(r ^ 63);
|
||||
#endif
|
||||
#else
|
||||
int l = __builtin_clz((unsigned)x) + 32;
|
||||
int h = __builtin_clz((unsigned)(x >> 32));
|
||||
return !!((unsigned)(x >> 32)) ? h : l;
|
||||
#endif
|
||||
}
|
||||
#endif // defined(_MSC_VER) && !defined(__clang__)
|
||||
|
||||
// pack divider structs to prevent compilers from padding.
|
||||
// This reduces memory usage by up to 43% when using a large
|
||||
// array of libdivide dividers and improves performance
|
||||
|
|
@ -235,18 +330,28 @@ static LIBDIVIDE_INLINE struct libdivide_u32_branchfree_t libdivide_u32_branchfr
|
|||
static LIBDIVIDE_INLINE struct libdivide_s64_branchfree_t libdivide_s64_branchfree_gen(int64_t d);
|
||||
static LIBDIVIDE_INLINE struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d);
|
||||
|
||||
static LIBDIVIDE_INLINE int16_t libdivide_s16_do_raw(int16_t numer, int16_t magic, uint8_t more);
|
||||
static LIBDIVIDE_INLINE int16_t libdivide_s16_do_raw(
|
||||
int16_t numer, int16_t magic, uint8_t more);
|
||||
static LIBDIVIDE_INLINE int16_t libdivide_s16_do(
|
||||
int16_t numer, const struct libdivide_s16_t *denom);
|
||||
static LIBDIVIDE_INLINE uint16_t libdivide_u16_do_raw(uint16_t numer, uint16_t magic, uint8_t more);
|
||||
static LIBDIVIDE_INLINE uint16_t libdivide_u16_do_raw(
|
||||
uint16_t numer, uint16_t magic, uint8_t more);
|
||||
static LIBDIVIDE_INLINE uint16_t libdivide_u16_do(
|
||||
uint16_t numer, const struct libdivide_u16_t *denom);
|
||||
static LIBDIVIDE_INLINE int32_t libdivide_s32_do_raw(
|
||||
int32_t numer, int32_t magic, uint8_t more);
|
||||
static LIBDIVIDE_INLINE int32_t libdivide_s32_do(
|
||||
int32_t numer, const struct libdivide_s32_t *denom);
|
||||
static LIBDIVIDE_INLINE uint32_t libdivide_u32_do_raw(
|
||||
uint32_t numer, uint32_t magic, uint8_t more);
|
||||
static LIBDIVIDE_INLINE uint32_t libdivide_u32_do(
|
||||
uint32_t numer, const struct libdivide_u32_t *denom);
|
||||
static LIBDIVIDE_INLINE int64_t libdivide_s64_do_raw(
|
||||
int64_t numer, int64_t magic, uint8_t more);
|
||||
static LIBDIVIDE_INLINE int64_t libdivide_s64_do(
|
||||
int64_t numer, const struct libdivide_s64_t *denom);
|
||||
static LIBDIVIDE_INLINE uint64_t libdivide_u64_do_raw(
|
||||
uint64_t numer, uint64_t magic, uint8_t more);
|
||||
static LIBDIVIDE_INLINE uint64_t libdivide_u64_do(
|
||||
uint64_t numer, const struct libdivide_u64_t *denom);
|
||||
|
||||
|
|
@ -312,7 +417,7 @@ static LIBDIVIDE_INLINE int32_t libdivide_mullhi_s32(int32_t x, int32_t y) {
|
|||
}
|
||||
|
||||
static LIBDIVIDE_INLINE uint64_t libdivide_mullhi_u64(uint64_t x, uint64_t y) {
|
||||
#if defined(LIBDIVIDE_VC) && defined(LIBDIVIDE_X86_64)
|
||||
#if defined(LIBDIVIDE_MULH_INTRINSICS)
|
||||
return __umulh(x, y);
|
||||
#elif defined(HAS_INT128_T)
|
||||
__uint128_t xl = x, yl = y;
|
||||
|
|
@ -338,7 +443,7 @@ static LIBDIVIDE_INLINE uint64_t libdivide_mullhi_u64(uint64_t x, uint64_t y) {
|
|||
}
|
||||
|
||||
static LIBDIVIDE_INLINE int64_t libdivide_mullhi_s64(int64_t x, int64_t y) {
|
||||
#if defined(LIBDIVIDE_VC) && defined(LIBDIVIDE_X86_64)
|
||||
#if defined(LIBDIVIDE_MULH_INTRINSICS)
|
||||
return __mulh(x, y);
|
||||
#elif defined(HAS_INT128_T)
|
||||
__int128_t xl = x, yl = y;
|
||||
|
|
@ -364,15 +469,9 @@ static LIBDIVIDE_INLINE int16_t libdivide_count_leading_zeros16(uint16_t val) {
|
|||
// Fast way to count leading zeros
|
||||
// On the AVR 8-bit architecture __builtin_clz() works on a int16_t.
|
||||
return __builtin_clz(val);
|
||||
#elif defined(__GNUC__) || __has_builtin(__builtin_clz)
|
||||
#elif defined(__GNUC__) || __has_builtin(__builtin_clz) || defined(_MSC_VER)
|
||||
// Fast way to count leading zeros
|
||||
return __builtin_clz(val) - 16;
|
||||
#elif defined(LIBDIVIDE_VC)
|
||||
unsigned long result;
|
||||
if (_BitScanReverse(&result, (unsigned long)val)) {
|
||||
return (int16_t)(15 - result);
|
||||
}
|
||||
return 0;
|
||||
return (int16_t)(__builtin_clz(val) - 16);
|
||||
#else
|
||||
if (val == 0) return 16;
|
||||
int16_t result = 4;
|
||||
|
|
@ -393,15 +492,9 @@ static LIBDIVIDE_INLINE int32_t libdivide_count_leading_zeros32(uint32_t val) {
|
|||
#if defined(__AVR__)
|
||||
// Fast way to count leading zeros
|
||||
return __builtin_clzl(val);
|
||||
#elif defined(__GNUC__) || __has_builtin(__builtin_clz)
|
||||
#elif defined(__GNUC__) || __has_builtin(__builtin_clz) || defined(_MSC_VER)
|
||||
// Fast way to count leading zeros
|
||||
return __builtin_clz(val);
|
||||
#elif defined(LIBDIVIDE_VC)
|
||||
unsigned long result;
|
||||
if (_BitScanReverse(&result, val)) {
|
||||
return 31 - result;
|
||||
}
|
||||
return 0;
|
||||
#else
|
||||
if (val == 0) return 32;
|
||||
int32_t result = 8;
|
||||
|
|
@ -419,15 +512,9 @@ static LIBDIVIDE_INLINE int32_t libdivide_count_leading_zeros32(uint32_t val) {
|
|||
}
|
||||
|
||||
static LIBDIVIDE_INLINE int32_t libdivide_count_leading_zeros64(uint64_t val) {
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_clzll)
|
||||
#if defined(__GNUC__) || __has_builtin(__builtin_clzll) || defined(_MSC_VER)
|
||||
// Fast way to count leading zeros
|
||||
return __builtin_clzll(val);
|
||||
#elif defined(LIBDIVIDE_VC) && defined(_WIN64)
|
||||
unsigned long result;
|
||||
if (_BitScanReverse64(&result, val)) {
|
||||
return 63 - result;
|
||||
}
|
||||
return 0;
|
||||
#else
|
||||
uint32_t hi = val >> 32;
|
||||
uint32_t lo = val & 0xFFFFFFFF;
|
||||
|
|
@ -474,7 +561,7 @@ static LIBDIVIDE_INLINE uint64_t libdivide_128_div_64_to_64(
|
|||
// it's not LIBDIVIDE_INLINEd.
|
||||
#if defined(LIBDIVIDE_X86_64) && defined(LIBDIVIDE_GCC_STYLE_ASM)
|
||||
uint64_t result;
|
||||
__asm__("divq %[v]" : "=a"(result), "=d"(*r) : [v] "r"(den), "a"(numlo), "d"(numhi));
|
||||
__asm__("div %[v]" : "=a"(result), "=d"(*r) : [v] "r"(den), "a"(numlo), "d"(numhi));
|
||||
return result;
|
||||
#else
|
||||
// We work in base 2**32.
|
||||
|
|
@ -510,7 +597,7 @@ static LIBDIVIDE_INLINE uint64_t libdivide_128_div_64_to_64(
|
|||
|
||||
// Check for overflow and divide by 0.
|
||||
if (numhi >= den) {
|
||||
if (r != NULL) *r = ~0ull;
|
||||
if (r) *r = ~0ull;
|
||||
return ~0ull;
|
||||
}
|
||||
|
||||
|
|
@ -524,7 +611,7 @@ static LIBDIVIDE_INLINE uint64_t libdivide_128_div_64_to_64(
|
|||
shift = libdivide_count_leading_zeros64(den);
|
||||
den <<= shift;
|
||||
numhi <<= shift;
|
||||
numhi |= (numlo >> (-shift & 63)) & (-(int64_t)shift >> 63);
|
||||
numhi |= (numlo >> (-shift & 63)) & (uint64_t)(-(int64_t)shift >> 63);
|
||||
numlo <<= shift;
|
||||
|
||||
// Extract the low digits of the numerator and both digits of the denominator.
|
||||
|
|
@ -556,7 +643,7 @@ static LIBDIVIDE_INLINE uint64_t libdivide_128_div_64_to_64(
|
|||
q0 = (uint32_t)qhat;
|
||||
|
||||
// Return remainder if requested.
|
||||
if (r != NULL) *r = (rem * b + num0 - q0 * den) >> shift;
|
||||
if (r) *r = (rem * b + num0 - q0 * den) >> shift;
|
||||
return ((uint64_t)q1 << 32) | q0;
|
||||
#endif
|
||||
}
|
||||
|
|
@ -733,11 +820,11 @@ static LIBDIVIDE_INLINE struct libdivide_u16_t libdivide_internal_u16_gen(
|
|||
return result;
|
||||
}
|
||||
|
||||
struct libdivide_u16_t libdivide_u16_gen(uint16_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_u16_t libdivide_u16_gen(uint16_t d) {
|
||||
return libdivide_internal_u16_gen(d, 0);
|
||||
}
|
||||
|
||||
struct libdivide_u16_branchfree_t libdivide_u16_branchfree_gen(uint16_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_u16_branchfree_t libdivide_u16_branchfree_gen(uint16_t d) {
|
||||
if (d == 1) {
|
||||
LIBDIVIDE_ERROR("branchfree divider must be != 1");
|
||||
}
|
||||
|
|
@ -750,11 +837,11 @@ struct libdivide_u16_branchfree_t libdivide_u16_branchfree_gen(uint16_t d) {
|
|||
// The original libdivide_u16_do takes a const pointer. However, this cannot be used
|
||||
// with a compile time constant libdivide_u16_t: it will generate a warning about
|
||||
// taking the address of a temporary. Hence this overload.
|
||||
uint16_t libdivide_u16_do_raw(uint16_t numer, uint16_t magic, uint8_t more) {
|
||||
static LIBDIVIDE_INLINE uint16_t libdivide_u16_do_raw(uint16_t numer, uint16_t magic, uint8_t more) {
|
||||
if (!magic) {
|
||||
return numer >> more;
|
||||
} else {
|
||||
uint16_t q = libdivide_mullhi_u16(magic, numer);
|
||||
uint16_t q = libdivide_mullhi_u16(numer, magic);
|
||||
if (more & LIBDIVIDE_ADD_MARKER) {
|
||||
uint16_t t = ((numer - q) >> 1) + q;
|
||||
return t >> (more & LIBDIVIDE_16_SHIFT_MASK);
|
||||
|
|
@ -766,18 +853,18 @@ uint16_t libdivide_u16_do_raw(uint16_t numer, uint16_t magic, uint8_t more) {
|
|||
}
|
||||
}
|
||||
|
||||
uint16_t libdivide_u16_do(uint16_t numer, const struct libdivide_u16_t *denom) {
|
||||
static LIBDIVIDE_INLINE uint16_t libdivide_u16_do(uint16_t numer, const struct libdivide_u16_t *denom) {
|
||||
return libdivide_u16_do_raw(numer, denom->magic, denom->more);
|
||||
}
|
||||
|
||||
uint16_t libdivide_u16_branchfree_do(
|
||||
static LIBDIVIDE_INLINE uint16_t libdivide_u16_branchfree_do(
|
||||
uint16_t numer, const struct libdivide_u16_branchfree_t *denom) {
|
||||
uint16_t q = libdivide_mullhi_u16(denom->magic, numer);
|
||||
uint16_t q = libdivide_mullhi_u16(numer, denom->magic);
|
||||
uint16_t t = ((numer - q) >> 1) + q;
|
||||
return t >> denom->more;
|
||||
}
|
||||
|
||||
uint16_t libdivide_u16_recover(const struct libdivide_u16_t *denom) {
|
||||
static LIBDIVIDE_INLINE uint16_t libdivide_u16_recover(const struct libdivide_u16_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_16_SHIFT_MASK;
|
||||
|
||||
|
|
@ -815,7 +902,7 @@ uint16_t libdivide_u16_recover(const struct libdivide_u16_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
uint16_t libdivide_u16_branchfree_recover(const struct libdivide_u16_branchfree_t *denom) {
|
||||
static LIBDIVIDE_INLINE uint16_t libdivide_u16_branchfree_recover(const struct libdivide_u16_branchfree_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_16_SHIFT_MASK;
|
||||
|
||||
|
|
@ -897,11 +984,11 @@ static LIBDIVIDE_INLINE struct libdivide_u32_t libdivide_internal_u32_gen(
|
|||
return result;
|
||||
}
|
||||
|
||||
struct libdivide_u32_t libdivide_u32_gen(uint32_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_u32_t libdivide_u32_gen(uint32_t d) {
|
||||
return libdivide_internal_u32_gen(d, 0);
|
||||
}
|
||||
|
||||
struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d) {
|
||||
if (d == 1) {
|
||||
LIBDIVIDE_ERROR("branchfree divider must be != 1");
|
||||
}
|
||||
|
|
@ -911,12 +998,11 @@ struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d) {
|
|||
return ret;
|
||||
}
|
||||
|
||||
uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
if (!denom->magic) {
|
||||
static LIBDIVIDE_INLINE uint32_t libdivide_u32_do_raw(uint32_t numer, uint32_t magic, uint8_t more) {
|
||||
if (!magic) {
|
||||
return numer >> more;
|
||||
} else {
|
||||
uint32_t q = libdivide_mullhi_u32(denom->magic, numer);
|
||||
uint32_t q = libdivide_mullhi_u32(numer, magic);
|
||||
if (more & LIBDIVIDE_ADD_MARKER) {
|
||||
uint32_t t = ((numer - q) >> 1) + q;
|
||||
return t >> (more & LIBDIVIDE_32_SHIFT_MASK);
|
||||
|
|
@ -928,14 +1014,18 @@ uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
uint32_t libdivide_u32_branchfree_do(
|
||||
static LIBDIVIDE_INLINE uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) {
|
||||
return libdivide_u32_do_raw(numer, denom->magic, denom->more);
|
||||
}
|
||||
|
||||
static LIBDIVIDE_INLINE uint32_t libdivide_u32_branchfree_do(
|
||||
uint32_t numer, const struct libdivide_u32_branchfree_t *denom) {
|
||||
uint32_t q = libdivide_mullhi_u32(denom->magic, numer);
|
||||
uint32_t q = libdivide_mullhi_u32(numer, denom->magic);
|
||||
uint32_t t = ((numer - q) >> 1) + q;
|
||||
return t >> denom->more;
|
||||
}
|
||||
|
||||
uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) {
|
||||
static LIBDIVIDE_INLINE uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK;
|
||||
|
||||
|
|
@ -973,7 +1063,7 @@ uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom) {
|
||||
static LIBDIVIDE_INLINE uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK;
|
||||
|
||||
|
|
@ -1002,7 +1092,7 @@ uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_
|
|||
}
|
||||
}
|
||||
|
||||
/////////// UINT64
|
||||
////////// UINT64
|
||||
|
||||
static LIBDIVIDE_INLINE struct libdivide_u64_t libdivide_internal_u64_gen(
|
||||
uint64_t d, int branchfree) {
|
||||
|
|
@ -1057,11 +1147,11 @@ static LIBDIVIDE_INLINE struct libdivide_u64_t libdivide_internal_u64_gen(
|
|||
return result;
|
||||
}
|
||||
|
||||
struct libdivide_u64_t libdivide_u64_gen(uint64_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_u64_t libdivide_u64_gen(uint64_t d) {
|
||||
return libdivide_internal_u64_gen(d, 0);
|
||||
}
|
||||
|
||||
struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d) {
|
||||
if (d == 1) {
|
||||
LIBDIVIDE_ERROR("branchfree divider must be != 1");
|
||||
}
|
||||
|
|
@ -1071,12 +1161,11 @@ struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d) {
|
|||
return ret;
|
||||
}
|
||||
|
||||
uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
if (!denom->magic) {
|
||||
static LIBDIVIDE_INLINE uint64_t libdivide_u64_do_raw(uint64_t numer, uint64_t magic, uint8_t more) {
|
||||
if (!magic) {
|
||||
return numer >> more;
|
||||
} else {
|
||||
uint64_t q = libdivide_mullhi_u64(denom->magic, numer);
|
||||
uint64_t q = libdivide_mullhi_u64(numer, magic);
|
||||
if (more & LIBDIVIDE_ADD_MARKER) {
|
||||
uint64_t t = ((numer - q) >> 1) + q;
|
||||
return t >> (more & LIBDIVIDE_64_SHIFT_MASK);
|
||||
|
|
@ -1088,14 +1177,18 @@ uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
uint64_t libdivide_u64_branchfree_do(
|
||||
static LIBDIVIDE_INLINE uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) {
|
||||
return libdivide_u64_do_raw(numer, denom->magic, denom->more);
|
||||
}
|
||||
|
||||
static LIBDIVIDE_INLINE uint64_t libdivide_u64_branchfree_do(
|
||||
uint64_t numer, const struct libdivide_u64_branchfree_t *denom) {
|
||||
uint64_t q = libdivide_mullhi_u64(denom->magic, numer);
|
||||
uint64_t q = libdivide_mullhi_u64(numer, denom->magic);
|
||||
uint64_t t = ((numer - q) >> 1) + q;
|
||||
return t >> denom->more;
|
||||
}
|
||||
|
||||
uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom) {
|
||||
static LIBDIVIDE_INLINE uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK;
|
||||
|
||||
|
|
@ -1139,7 +1232,7 @@ uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom) {
|
||||
static LIBDIVIDE_INLINE uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK;
|
||||
|
||||
|
|
@ -1174,7 +1267,7 @@ uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_
|
|||
}
|
||||
}
|
||||
|
||||
/////////// SINT16
|
||||
////////// SINT16
|
||||
|
||||
static LIBDIVIDE_INLINE struct libdivide_s16_t libdivide_internal_s16_gen(
|
||||
int16_t d, int branchfree) {
|
||||
|
|
@ -1242,11 +1335,11 @@ static LIBDIVIDE_INLINE struct libdivide_s16_t libdivide_internal_s16_gen(
|
|||
return result;
|
||||
}
|
||||
|
||||
struct libdivide_s16_t libdivide_s16_gen(int16_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_s16_t libdivide_s16_gen(int16_t d) {
|
||||
return libdivide_internal_s16_gen(d, 0);
|
||||
}
|
||||
|
||||
struct libdivide_s16_branchfree_t libdivide_s16_branchfree_gen(int16_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_s16_branchfree_t libdivide_s16_branchfree_gen(int16_t d) {
|
||||
struct libdivide_s16_t tmp = libdivide_internal_s16_gen(d, 1);
|
||||
struct libdivide_s16_branchfree_t result = {tmp.magic, tmp.more};
|
||||
return result;
|
||||
|
|
@ -1255,7 +1348,7 @@ struct libdivide_s16_branchfree_t libdivide_s16_branchfree_gen(int16_t d) {
|
|||
// The original libdivide_s16_do takes a const pointer. However, this cannot be used
|
||||
// with a compile time constant libdivide_s16_t: it will generate a warning about
|
||||
// taking the address of a temporary. Hence this overload.
|
||||
int16_t libdivide_s16_do_raw(int16_t numer, int16_t magic, uint8_t more) {
|
||||
static LIBDIVIDE_INLINE int16_t libdivide_s16_do_raw(int16_t numer, int16_t magic, uint8_t more) {
|
||||
uint8_t shift = more & LIBDIVIDE_16_SHIFT_MASK;
|
||||
|
||||
if (!magic) {
|
||||
|
|
@ -1267,7 +1360,7 @@ int16_t libdivide_s16_do_raw(int16_t numer, int16_t magic, uint8_t more) {
|
|||
q = (q ^ sign) - sign;
|
||||
return q;
|
||||
} else {
|
||||
uint16_t uq = (uint16_t)libdivide_mullhi_s16(magic, numer);
|
||||
uint16_t uq = (uint16_t)libdivide_mullhi_s16(numer, magic);
|
||||
if (more & LIBDIVIDE_ADD_MARKER) {
|
||||
// must be arithmetic shift and then sign extend
|
||||
int16_t sign = (int8_t)more >> 7;
|
||||
|
|
@ -1282,17 +1375,17 @@ int16_t libdivide_s16_do_raw(int16_t numer, int16_t magic, uint8_t more) {
|
|||
}
|
||||
}
|
||||
|
||||
int16_t libdivide_s16_do(int16_t numer, const struct libdivide_s16_t *denom) {
|
||||
static LIBDIVIDE_INLINE int16_t libdivide_s16_do(int16_t numer, const struct libdivide_s16_t *denom) {
|
||||
return libdivide_s16_do_raw(numer, denom->magic, denom->more);
|
||||
}
|
||||
|
||||
int16_t libdivide_s16_branchfree_do(int16_t numer, const struct libdivide_s16_branchfree_t *denom) {
|
||||
static LIBDIVIDE_INLINE int16_t libdivide_s16_branchfree_do(int16_t numer, const struct libdivide_s16_branchfree_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_16_SHIFT_MASK;
|
||||
// must be arithmetic shift and then sign extend
|
||||
int16_t sign = (int8_t)more >> 7;
|
||||
int16_t magic = denom->magic;
|
||||
int16_t q = libdivide_mullhi_s16(magic, numer);
|
||||
int16_t q = libdivide_mullhi_s16(numer, magic);
|
||||
q += numer;
|
||||
|
||||
// If q is non-negative, we have nothing to do
|
||||
|
|
@ -1310,7 +1403,7 @@ int16_t libdivide_s16_branchfree_do(int16_t numer, const struct libdivide_s16_br
|
|||
return q;
|
||||
}
|
||||
|
||||
int16_t libdivide_s16_recover(const struct libdivide_s16_t *denom) {
|
||||
static LIBDIVIDE_INLINE int16_t libdivide_s16_recover(const struct libdivide_s16_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_16_SHIFT_MASK;
|
||||
if (!denom->magic) {
|
||||
|
|
@ -1345,11 +1438,12 @@ int16_t libdivide_s16_recover(const struct libdivide_s16_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
int16_t libdivide_s16_branchfree_recover(const struct libdivide_s16_branchfree_t *denom) {
|
||||
return libdivide_s16_recover((const struct libdivide_s16_t *)denom);
|
||||
static LIBDIVIDE_INLINE int16_t libdivide_s16_branchfree_recover(const struct libdivide_s16_branchfree_t *denom) {
|
||||
const struct libdivide_s16_t den = {denom->magic, denom->more};
|
||||
return libdivide_s16_recover(&den);
|
||||
}
|
||||
|
||||
/////////// SINT32
|
||||
////////// SINT32
|
||||
|
||||
static LIBDIVIDE_INLINE struct libdivide_s32_t libdivide_internal_s32_gen(
|
||||
int32_t d, int branchfree) {
|
||||
|
|
@ -1417,21 +1511,20 @@ static LIBDIVIDE_INLINE struct libdivide_s32_t libdivide_internal_s32_gen(
|
|||
return result;
|
||||
}
|
||||
|
||||
struct libdivide_s32_t libdivide_s32_gen(int32_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_s32_t libdivide_s32_gen(int32_t d) {
|
||||
return libdivide_internal_s32_gen(d, 0);
|
||||
}
|
||||
|
||||
struct libdivide_s32_branchfree_t libdivide_s32_branchfree_gen(int32_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_s32_branchfree_t libdivide_s32_branchfree_gen(int32_t d) {
|
||||
struct libdivide_s32_t tmp = libdivide_internal_s32_gen(d, 1);
|
||||
struct libdivide_s32_branchfree_t result = {tmp.magic, tmp.more};
|
||||
return result;
|
||||
}
|
||||
|
||||
int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
static LIBDIVIDE_INLINE int32_t libdivide_s32_do_raw(int32_t numer, int32_t magic, uint8_t more) {
|
||||
uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK;
|
||||
|
||||
if (!denom->magic) {
|
||||
if (!magic) {
|
||||
uint32_t sign = (int8_t)more >> 7;
|
||||
uint32_t mask = ((uint32_t)1 << shift) - 1;
|
||||
uint32_t uq = numer + ((numer >> 31) & mask);
|
||||
|
|
@ -1440,7 +1533,7 @@ int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom) {
|
|||
q = (q ^ sign) - sign;
|
||||
return q;
|
||||
} else {
|
||||
uint32_t uq = (uint32_t)libdivide_mullhi_s32(denom->magic, numer);
|
||||
uint32_t uq = (uint32_t)libdivide_mullhi_s32(numer, magic);
|
||||
if (more & LIBDIVIDE_ADD_MARKER) {
|
||||
// must be arithmetic shift and then sign extend
|
||||
int32_t sign = (int8_t)more >> 7;
|
||||
|
|
@ -1455,13 +1548,17 @@ int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom) {
|
||||
static LIBDIVIDE_INLINE int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom) {
|
||||
return libdivide_s32_do_raw(numer, denom->magic, denom->more);
|
||||
}
|
||||
|
||||
static LIBDIVIDE_INLINE int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK;
|
||||
// must be arithmetic shift and then sign extend
|
||||
int32_t sign = (int8_t)more >> 7;
|
||||
int32_t magic = denom->magic;
|
||||
int32_t q = libdivide_mullhi_s32(magic, numer);
|
||||
int32_t q = libdivide_mullhi_s32(numer, magic);
|
||||
q += numer;
|
||||
|
||||
// If q is non-negative, we have nothing to do
|
||||
|
|
@ -1479,7 +1576,7 @@ int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_br
|
|||
return q;
|
||||
}
|
||||
|
||||
int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) {
|
||||
static LIBDIVIDE_INLINE int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK;
|
||||
if (!denom->magic) {
|
||||
|
|
@ -1514,11 +1611,12 @@ int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom) {
|
||||
return libdivide_s32_recover((const struct libdivide_s32_t *)denom);
|
||||
static LIBDIVIDE_INLINE int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom) {
|
||||
const struct libdivide_s32_t den = {denom->magic, denom->more};
|
||||
return libdivide_s32_recover(&den);
|
||||
}
|
||||
|
||||
///////////// SINT64
|
||||
////////// SINT64
|
||||
|
||||
static LIBDIVIDE_INLINE struct libdivide_s64_t libdivide_internal_s64_gen(
|
||||
int64_t d, int branchfree) {
|
||||
|
|
@ -1586,21 +1684,20 @@ static LIBDIVIDE_INLINE struct libdivide_s64_t libdivide_internal_s64_gen(
|
|||
return result;
|
||||
}
|
||||
|
||||
struct libdivide_s64_t libdivide_s64_gen(int64_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_s64_t libdivide_s64_gen(int64_t d) {
|
||||
return libdivide_internal_s64_gen(d, 0);
|
||||
}
|
||||
|
||||
struct libdivide_s64_branchfree_t libdivide_s64_branchfree_gen(int64_t d) {
|
||||
static LIBDIVIDE_INLINE struct libdivide_s64_branchfree_t libdivide_s64_branchfree_gen(int64_t d) {
|
||||
struct libdivide_s64_t tmp = libdivide_internal_s64_gen(d, 1);
|
||||
struct libdivide_s64_branchfree_t ret = {tmp.magic, tmp.more};
|
||||
return ret;
|
||||
}
|
||||
|
||||
int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
static LIBDIVIDE_INLINE int64_t libdivide_s64_do_raw(int64_t numer, int64_t magic, uint8_t more) {
|
||||
uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK;
|
||||
|
||||
if (!denom->magic) { // shift path
|
||||
if (!magic) { // shift path
|
||||
uint64_t mask = ((uint64_t)1 << shift) - 1;
|
||||
uint64_t uq = numer + ((numer >> 63) & mask);
|
||||
int64_t q = (int64_t)uq;
|
||||
|
|
@ -1610,7 +1707,7 @@ int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) {
|
|||
q = (q ^ sign) - sign;
|
||||
return q;
|
||||
} else {
|
||||
uint64_t uq = (uint64_t)libdivide_mullhi_s64(denom->magic, numer);
|
||||
uint64_t uq = (uint64_t)libdivide_mullhi_s64(numer, magic);
|
||||
if (more & LIBDIVIDE_ADD_MARKER) {
|
||||
// must be arithmetic shift and then sign extend
|
||||
int64_t sign = (int8_t)more >> 7;
|
||||
|
|
@ -1625,13 +1722,17 @@ int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom) {
|
||||
static LIBDIVIDE_INLINE int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) {
|
||||
return libdivide_s64_do_raw(numer, denom->magic, denom->more);
|
||||
}
|
||||
|
||||
static LIBDIVIDE_INLINE int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK;
|
||||
// must be arithmetic shift and then sign extend
|
||||
int64_t sign = (int8_t)more >> 7;
|
||||
int64_t magic = denom->magic;
|
||||
int64_t q = libdivide_mullhi_s64(magic, numer);
|
||||
int64_t q = libdivide_mullhi_s64(numer, magic);
|
||||
q += numer;
|
||||
|
||||
// If q is non-negative, we have nothing to do.
|
||||
|
|
@ -1649,7 +1750,7 @@ int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_br
|
|||
return q;
|
||||
}
|
||||
|
||||
int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) {
|
||||
static LIBDIVIDE_INLINE int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) {
|
||||
uint8_t more = denom->more;
|
||||
uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK;
|
||||
if (denom->magic == 0) { // shift path
|
||||
|
|
@ -1675,8 +1776,9 @@ int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) {
|
|||
}
|
||||
}
|
||||
|
||||
int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom) {
|
||||
return libdivide_s64_recover((const struct libdivide_s64_t *)denom);
|
||||
static LIBDIVIDE_INLINE int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom) {
|
||||
const struct libdivide_s64_t den = {denom->magic, denom->more};
|
||||
return libdivide_s64_recover(&den);
|
||||
}
|
||||
|
||||
// Simplest possible vector type division: treat the vector type as an array
|
||||
|
|
@ -2959,7 +3061,7 @@ __m128i libdivide_s64_branchfree_do_vec128(
|
|||
|
||||
#endif
|
||||
|
||||
/////////// C++ stuff
|
||||
////////// C++ stuff
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
|
|
@ -3056,6 +3158,7 @@ struct NeonVecFor {
|
|||
#define DISPATCHER_GEN(T, ALGO) \
|
||||
libdivide_##ALGO##_t denom; \
|
||||
LIBDIVIDE_INLINE dispatcher() {} \
|
||||
explicit LIBDIVIDE_CONSTEXPR dispatcher(decltype(nullptr)) : denom{} {} \
|
||||
LIBDIVIDE_INLINE dispatcher(T d) : denom(libdivide_##ALGO##_gen(d)) {} \
|
||||
LIBDIVIDE_INLINE T divide(T n) const { return libdivide_##ALGO##_do(n, &denom); } \
|
||||
LIBDIVIDE_INLINE T recover() const { return libdivide_##ALGO##_recover(&denom); } \
|
||||
|
|
@ -3147,6 +3250,9 @@ class divider {
|
|||
// later doesn't slow us down.
|
||||
divider() {}
|
||||
|
||||
// constexpr zero-initialization to allow for use w/ static constinit
|
||||
explicit LIBDIVIDE_CONSTEXPR divider(decltype(nullptr)) : div(nullptr) {}
|
||||
|
||||
// Constructor that takes the divisor as a parameter
|
||||
LIBDIVIDE_INLINE divider(T d) : div(d) {}
|
||||
|
||||
|
|
@ -3158,7 +3264,7 @@ class divider {
|
|||
T recover() const { return div.recover(); }
|
||||
|
||||
bool operator==(const divider<T, ALGO> &other) const {
|
||||
return div.denom.magic == other.denom.magic && div.denom.more == other.denom.more;
|
||||
return div.denom.magic == other.div.denom.magic && div.denom.more == other.div.denom.more;
|
||||
}
|
||||
|
||||
bool operator!=(const divider<T, ALGO> &other) const { return !(*this == other); }
|
||||
|
|
@ -3262,7 +3368,7 @@ using branchfree_divider = divider<T, BRANCHFREE>;
|
|||
|
||||
#endif // __cplusplus
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue