From e8eda43a6e9f2530d3a85284351d0b70bce05619 Mon Sep 17 00:00:00 2001 From: Mamoru TASAKA Date: Wed, 22 Dec 2021 10:12:16 +0900 Subject: [PATCH] define uint64_t type grib_en/decode and use them Currently grib_encode_size_t / grib_decode_size_t are defined and especially on windows where size_t is 64 bit, encode_double_array and decode_double_array use them. So especially for 32 bit linux system, define uint64_t type grib_encode / grib_decode and use them instead of size_t type ones. This method is more portable. --- src/grib_accessor_class_bufr_data_array.c | 24 +++--- src/grib_api_prototypes.h | 2 + src/grib_bits.c | 18 +++-- src/grib_bits_any_endian.c | 93 +++++++++++++++++++++++ src/grib_bits_fast_big_endian.c | 70 +++++++++++++++++ 5 files changed, 189 insertions(+), 18 deletions(-) diff --git a/src/grib_accessor_class_bufr_data_array.c b/src/grib_accessor_class_bufr_data_array.c index 1058bb88e..462d62b72 100644 --- a/src/grib_accessor_class_bufr_data_array.c +++ b/src/grib_accessor_class_bufr_data_array.c @@ -628,7 +628,7 @@ static grib_darray* decode_double_array(grib_context* c, unsigned char* data, lo { grib_darray* ret = NULL; int j; - size_t lval; + uint64_t lval; int localReference, localWidth, modifiedWidth, modifiedReference; double modifiedFactor, dval; int bufr_multi_element_constant_arrays = c->bufr_multi_element_constant_arrays; @@ -649,7 +649,7 @@ static grib_darray* decode_double_array(grib_context* c, unsigned char* data, lo *err = 0; return ret; } - lval = grib_decode_size_t(data, pos, modifiedWidth); + lval = grib_decode_uint64_t(data, pos, modifiedWidth); localReference = (long)lval + modifiedReference; localWidth = grib_decode_unsigned_long(data, pos, 6); grib_context_log(c, GRIB_LOG_DEBUG, "BUFR data decoding: \tlocalWidth=%d", localWidth); @@ -666,7 +666,7 @@ static grib_darray* decode_double_array(grib_context* c, unsigned char* data, lo return ret; } for (j = 0; j < self->numberOfSubsets; j++) { - lval = grib_decode_size_t(data, pos, localWidth); + lval = grib_decode_uint64_t(data, pos, localWidth); if (canBeMissing && grib_is_all_bits_one(lval, localWidth)) { dval = GRIB_MISSING_DOUBLE; } @@ -779,7 +779,7 @@ static int encode_double_array(grib_context* c, grib_buffer* buff, long* pos, bu { int err = 0; int j, i; - size_t lval; + uint64_t lval; long localReference = 0, localWidth = 0, modifiedWidth, modifiedReference; long reference, allone; double localRange, modifiedFactor, inverseFactor; @@ -841,7 +841,7 @@ static int encode_double_array(grib_context* c, grib_buffer* buff, long* pos, bu } else { lval = round(*v * inverseFactor) - modifiedReference; - grib_encode_size_tb(buff->data, lval, pos, modifiedWidth); + grib_encode_uint64_tb(buff->data, lval, pos, modifiedWidth); } } grib_buffer_set_ulength_bits(c, buff, buff->ulength_bits + 6); @@ -870,7 +870,7 @@ static int encode_double_array(grib_context* c, grib_buffer* buff, long* pos, bu } else { lval = round(*v * inverseFactor) - modifiedReference; - grib_encode_size_tb(buff->data, lval, pos, modifiedWidth); + grib_encode_uint64_tb(buff->data, lval, pos, modifiedWidth); } grib_buffer_set_ulength_bits(c, buff, buff->ulength_bits + 6); grib_encode_unsigned_longb(buff->data, localWidth, pos, 6); @@ -967,7 +967,7 @@ static int encode_double_array(grib_context* c, grib_buffer* buff, long* pos, bu } else { lval = localReference - modifiedReference; - grib_encode_size_tb(buff->data, lval, pos, modifiedWidth); + grib_encode_uint64_tb(buff->data, lval, pos, modifiedWidth); } } grib_buffer_set_ulength_bits(c, buff, buff->ulength_bits + 6); @@ -981,7 +981,7 @@ static int encode_double_array(grib_context* c, grib_buffer* buff, long* pos, bu } else { lval = round(values[j] * inverseFactor) - reference; - grib_encode_size_tb(buff->data, lval, pos, localWidth); + grib_encode_uint64_tb(buff->data, lval, pos, localWidth); } } } @@ -994,7 +994,7 @@ static int encode_double_array(grib_context* c, grib_buffer* buff, long* pos, bu static int encode_double_value(grib_context* c, grib_buffer* buff, long* pos, bufr_descriptor* bd, grib_accessor_bufr_data_array* self, double value) { - size_t lval; + uint64_t lval; double maxAllowed, minAllowed; int err = 0; int modifiedWidth, modifiedReference; @@ -1031,7 +1031,7 @@ static int encode_double_value(grib_context* c, grib_buffer* buff, long* pos, bu lval = round(value / modifiedFactor) - modifiedReference; if (c->debug) grib_context_log(c, GRIB_LOG_DEBUG, "encode_double_value %s: value=%.15f lval=%lu\n", bd->shortName, value, lval); - grib_encode_size_tb(buff->data, lval, pos, modifiedWidth); + grib_encode_uint64_tb(buff->data, lval, pos, modifiedWidth); } return err; @@ -1078,7 +1078,7 @@ static double decode_double_value(grib_context* c, unsigned char* data, long* po bufr_descriptor* bd, int canBeMissing, grib_accessor_bufr_data_array* self, int* err) { - size_t lval; + uint64_t lval; int modifiedWidth, modifiedReference; double modifiedFactor; double dval = 0; @@ -1095,7 +1095,7 @@ static double decode_double_value(grib_context* c, unsigned char* data, long* po return GRIB_MISSING_DOUBLE; } - lval = grib_decode_size_t(data, pos, modifiedWidth); + lval = grib_decode_uint64_t(data, pos, modifiedWidth); if (canBeMissing && grib_is_all_bits_one(lval, modifiedWidth)) { dval = GRIB_MISSING_DOUBLE; } diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h index 3fe136518..43555b2f3 100644 --- a/src/grib_api_prototypes.h +++ b/src/grib_api_prototypes.h @@ -1536,8 +1536,10 @@ char* grib_decode_string(const unsigned char* bitStream, long* bitOffset, size_t unsigned long grib_decode_unsigned_long(const unsigned char* p, long* bitp, long nbits); int grib_encode_unsigned_long(unsigned char* p, unsigned long val, long* bitp, long nbits); size_t grib_decode_size_t(const unsigned char* p, long* bitp, long nbits); +uint64_t grib_decode_uint64_t(const unsigned char* p, long* bitp, long nbits); int grib_encode_unsigned_longb(unsigned char* p, unsigned long val, long* bitp, long nb); int grib_encode_size_tb(unsigned char* p, size_t val, long* bitp, long nb); +int grib_encode_uint64_tb(unsigned char* p, uint64_t val, long* bitp, long nb); /* grib_bits_any_endian_simple.c */ diff --git a/src/grib_bits.c b/src/grib_bits.c index b99f8e033..3131a0d47 100644 --- a/src/grib_bits.c +++ b/src/grib_bits.c @@ -18,23 +18,29 @@ #include "omp.h" #endif -#define mask1(i) (1UL << i) +#define mask1(i) ((uint64_t)1 << i) #define test(n, i) !!((n)&mask1(i)) -long GRIB_MASK = -1; /* Mask of sword bits */ +uint64_t GRIB_MASK = -1; /* Mask of sword bits */ #define VALUE(p, q, b) \ - (((b) == max_nbits ? GRIB_MASK : ~(GRIB_MASK << (b))) & ((p) >> (max_nbits - ((q) + (b))))) + (((b) == max_nbits ? (unsigned long)GRIB_MASK : ~((unsigned long)GRIB_MASK << (b))) & ((p) >> (max_nbits - ((q) + (b))))) #define MASKVALUE(q, b) \ - ((b) == max_nbits ? GRIB_MASK : (~(GRIB_MASK << (b)) << (max_nbits - ((q) + (b))))) + ((b) == max_nbits ? (unsigned long)GRIB_MASK : (~((unsigned long)GRIB_MASK << (b)) << (max_nbits - ((q) + (b))))) #define VALUE_SIZE_T(p, q, b) \ - (((b) == max_nbits_size_t ? GRIB_MASK : ~(GRIB_MASK << (b))) & ((p) >> (max_nbits_size_t - ((q) + (b))))) + (((b) == max_nbits_size_t ? (size_t)GRIB_MASK : ~((size_t)GRIB_MASK << (b))) & ((p) >> (max_nbits_size_t - ((q) + (b))))) #define MASKVALUE_SIZE_T(q, b) \ - ((b) == max_nbits_size_t ? GRIB_MASK : (~(GRIB_MASK << (b)) << (max_nbits_size_t - ((q) + (b))))) + ((b) == max_nbits_size_t ? (size_t)GRIB_MASK : (~((size_t)GRIB_MASK << (b)) << (max_nbits_size_t - ((q) + (b))))) + +#define VALUE_UINT64_T(p, q, b) \ + (((b) == 64 ? GRIB_MASK : ~(GRIB_MASK << (b))) & ((p) >> (64 - ((q) + (b))))) + +#define MASKVALUE_UINT64_T(q, b) \ + ((b) == 64 ? GRIB_MASK : (~(GRIB_MASK << (b)) << (64 - ((q) + (b))))) static const unsigned long dmasks[] = { 0xFF, diff --git a/src/grib_bits_any_endian.c b/src/grib_bits_any_endian.c index 411d902bc..eac665699 100644 --- a/src/grib_bits_any_endian.c +++ b/src/grib_bits_any_endian.c @@ -16,6 +16,7 @@ #ifdef ECCODES_ON_WINDOWS #include #endif +#include #if GRIB_PTHREADS static pthread_once_t once = PTHREAD_ONCE_INIT; @@ -365,6 +366,71 @@ size_t grib_decode_size_t(const unsigned char* p, long* bitp, long nbits) return ret; } +#define BIT_MASK_UINT64_T(x) \ + (((x) == 64) ? (uint64_t)-1 : ((uint64_t)1 << (x)) - 1) + +uint64_t grib_decode_uint64_t(const unsigned char* p, long* bitp, long nbits) +{ + uint64_t ret = 0; + uint64_t ret_carry_up = 0; + long oc = *bitp / 8; + uint64_t mask = 0; + long pi = 0; + int usefulBitsInByte = 0; + long bitsToRead = 0; + + if (nbits == 0) + return 0; + + if (nbits > 64) { + int bits = nbits; + int mod = bits % 64; + + if (mod != 0) { + int e = grib_decode_uint64_t(p, bitp, mod); + Assert(e == 0); + bits -= mod; + } + + while (bits > 64) { + int e = grib_decode_uint64_t(p, bitp, 64); + Assert(e == 0); + bits -= 64; + } + + return grib_decode_uint64_t(p, bitp, bits); + } + + mask = BIT_MASK_UINT64_T(nbits); + /* pi: position of bitp in p[]. >>3 == /8 */ + pi = oc; + /* number of useful bits in current byte */ + usefulBitsInByte = 8 - (*bitp & 7); + /* read at least enough bits (byte by byte) from input */ + bitsToRead = nbits; + while (bitsToRead > 0) { + ret_carry_up <<= 8; + ret_carry_up |= (ret >> (64-8)); + ret <<= 8; + /* ret += p[pi]; */ + DebugAssert((ret & p[pi]) == 0); + ret = ret | p[pi]; + pi++; + bitsToRead -= usefulBitsInByte; + usefulBitsInByte = 8; + } + *bitp += nbits; + + /* bitsToRead might now be negative (too many bits read) */ + /* remove those which are too much */ + ret >>= -1 * bitsToRead; + if (bitsToRead < 0) ret |= (ret_carry_up << (64 - (-1 * bitsToRead))); + /* remove leading bits (from previous value) */ + ret &= mask; + + return ret; +} + int grib_encode_unsigned_longb(unsigned char* p, unsigned long val, long* bitp, long nb) { long i = 0; @@ -421,6 +487,33 @@ int grib_encode_size_tb(unsigned char* p, size_t val, long* bitp, long nb) return GRIB_SUCCESS; } +int grib_encode_uint64_tb(unsigned char* p, uint64_t val, long* bitp, long nb) +{ + long i = 0; + + if (nb > 64) { + fprintf(stderr, "Number of bits (%ld) exceeds maximum number of bits (%d)\n", nb, 64); + Assert(0); + } +#ifdef DEBUG + { + uint64_t maxV = grib_power(nb, 2); + if (val > maxV) { + fprintf(stderr, "grib_encode_uint64_tb: Value=%"PRIu64", but number of bits=%"PRIu64"!\n", val, nb); + Assert(0); + } + } +#endif + for (i = nb - 1; i >= 0; i--) { + if (test(val, i)) + grib_set_bit_on(p, bitp); + else + grib_set_bit_off(p, bitp); + } + return GRIB_SUCCESS; +} + + #if OMP_PACKING #include "grib_bits_any_endian_omp.c" #elif VECTOR diff --git a/src/grib_bits_fast_big_endian.c b/src/grib_bits_fast_big_endian.c index b47dcfcc7..b7667e362 100644 --- a/src/grib_bits_fast_big_endian.c +++ b/src/grib_bits_fast_big_endian.c @@ -176,6 +176,37 @@ size_t grib_decode_size_t(const unsigned char* p, long* bitp, long nbits) return val; } +uint64_t grib_decode_uint64_t(const unsigned char* p, long* bitp, long nbits) +{ + long countOfLeftmostBits = 0, leftmostBits = 0; + long startBit = *bitp; + long remainingBits = nbits; + long* pp = (long*)p; + uint64_t val = 0; + + if (startBit >= 64) { + pp += startBit / 64; + startBit %= 64; + } + + countOfLeftmostBits = startBit + remainingBits; + if (countOfLeftmostBits > 64) { + countOfLeftmostBits = 64 - startBit; + remainingBits -= countOfLeftmostBits; + leftmostBits = (VALUE_UINT64_T(*pp, startBit, countOfLeftmostBits)) << remainingBits; + startBit = 0; + pp++; + } + else + leftmostBits = 0; + + val = leftmostBits + (VALUE_UINT64_T(*pp, startBit, remainingBits)); + + *bitp += nbits; + + return val; +} + int grib_encode_unsigned_long(unsigned char* p, unsigned long val, long* bitp, long nbits) { long* destination = (long*)p; @@ -249,11 +280,50 @@ int grib_encode_size_t(unsigned char* p, size_t val, long* bitp, long nbits) return GRIB_SUCCESS; } +int grib_encode_uint64_t(unsigned char* p, uint64_t val, long* bitp, long nbits) +{ + long* destination = (long*)p; + long countOfLeftmostBits = 0, nextWord = 0, startBit = 0, remainingBits = 0, rightmostBits = 0; + + startBit = *bitp; + remainingBits = nbits; + + if (startBit >= 64) { + nextWord = startBit / 64; + startBit %= 64; + } + else + nextWord = 0; + + countOfLeftmostBits = startBit + remainingBits; + if (countOfLeftmostBits > 64) { + countOfLeftmostBits = 64 - startBit; + startBit = 64 - remainingBits; + remainingBits -= countOfLeftmostBits; + destination[nextWord] = + ((destination[nextWord] >> countOfLeftmostBits) << countOfLeftmostBits) + (VALUE_UINT64_T(val, startBit, countOfLeftmostBits)); + startBit = 0; + nextWord++; + } + + rightmostBits = VALUE_UINT64_T(val, 64 - remainingBits, remainingBits); + destination[nextWord] = + (destination[nextWord] & ~MASKVALUE_UINT64_T(startBit, remainingBits)) + (rightmostBits << 64 - (remainingBits + startBit)); + + *bitp += nbits; + return GRIB_SUCCESS; +} + int grib_encode_size_tb(unsigned char* p, size_t val, long* bitp, long nbits) { return grib_encode_size_t(p, val, bitp, nbits); } +int grib_encode_uint64_tb(unsigned char* p, uint64_t val, long* bitp, long nbits) +{ + return grib_encode_uint64_t(p, val, bitp, nbits); +} + #if VECTOR #include "grib_bits_fast_big_endian_vector.c" /* Experimental */