Codebase list c-blosc / e20be0c
New upstream version 1.21.3+ds Håvard F. Aasen 1 year, 4 months ago
8 changed file(s) with 92 addition(s) and 79 deletion(s). Raw diff Collapse all Expand all
+3
-4
ANNOUNCE.rst less more
00 ===============================================================
1 Announcing C-Blosc 1.21.2
1 Announcing C-Blosc 1.21.3
22 A blocking, shuffling and lossless compression library for C
33 ===============================================================
44
55 What is new?
66 ============
77
8 This is a maintenance release. Upgrade internal-complib zstd from
9 1.5.0 to 1.5.2 and many small code improvements, improved consistency
10 and typo fixes. An upgrade is recommended.
8 This is a maintenance release. Upgraded internal-complib lz4
9 to 1.9.4 and blosclz to 2.5.1.
1110
1211 For more info, please see the release notes in:
1312
+8
-0
RELEASE_NOTES.rst less more
00 ===========================
11 Release notes for C-Blosc
22 ===========================
3
4 Changes from 1.21.2 to 1.21.3
5 =============================
6
7 * Internal LZ4 codec updated to 1.9.4.
8
9 * Internal BloscLZ codec updated to 2.5.1.
10
311
412 Changes from 1.21.1 to 1.21.2
513 =============================
+2
-2
RELEASING.rst less more
7575 Tagging
7676 -------
7777
78 - Create a tag ``X.Y.Z`` from ``master``::
78 - Create a tag ``X.Y.Z`` from ``main``::
7979
8080 $ git tag -a vX.Y.Z -m "Tagging version X.Y.Z"
8181
9696 Post-release actions
9797 --------------------
9898
99 - Edit *VERSION* symbols in blosc/blosc.h in master to increment the
99 - Edit *VERSION* symbols in blosc/blosc.h in main to increment the
100100 version to the next minor one (i.e. X.Y.Z --> X.Y.(Z+1).dev).
101101
102102 - Create new headers for adding new features in ``RELEASE_NOTES.rst``
+1
-1
blosc/CMakeLists.txt less more
1212 if (LZ4_FOUND)
1313 set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_INCLUDE_DIR})
1414 else(LZ4_FOUND)
15 set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.9.3)
15 set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.9.4)
1616 set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_LOCAL_DIR})
1717 endif(LZ4_FOUND)
1818 endif(NOT DEACTIVATE_LZ4)
+4
-6
blosc/blosc.h less more
1818 /* Version numbers */
1919 #define BLOSC_VERSION_MAJOR 1 /* for major interface/format changes */
2020 #define BLOSC_VERSION_MINOR 21 /* for minor interface/format changes */
21 #define BLOSC_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
22
23 #define BLOSC_VERSION_STRING "1.21.2" /* string version. Sync with above! */
21 #define BLOSC_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
22
23 #define BLOSC_VERSION_STRING "1.21.3" /* string version. Sync with above! */
2424 #define BLOSC_VERSION_REVISION "$Rev$" /* revision version */
25 #define BLOSC_VERSION_DATE "$Date:: 2022-12-07 #$" /* date version */
26
27 #define BLOSCLZ_VERSION_STRING "2.5.1" /* the internal compressor version */
25 #define BLOSC_VERSION_DATE "$Date:: 2022-12-15 #$" /* date version */
2826
2927 /* The *_FORMAT symbols should be just 1-byte long */
3028 #define BLOSC_VERSION_FORMAT 2 /* Blosc format version, starting at 1 */
+68
-64
blosc/blosclz.c less more
4343 #define MAX_FARDISTANCE (65535 + MAX_DISTANCE - 1)
4444
4545 #ifdef BLOSC_STRICT_ALIGN
46 #define BLOSCLZ_READU16(p) ((p)[0] | (p)[1]<<8)
46 #define BLOSCLZ_READU16(p) ((p)[0] | (p)[1]<<8)
4747 #define BLOSCLZ_READU32(p) ((p)[0] | (p)[1]<<8 | (p)[2]<<16 | (p)[3]<<24)
4848 #else
49 #define BLOSCLZ_READU16(p) *((const uint16_t*)(p))
50 #define BLOSCLZ_READU32(p) *((const uint32_t*)(p))
49 #define BLOSCLZ_READU16(p) *((const uint16_t*)(p))
50 #define BLOSCLZ_READU32(p) *((const uint32_t*)(p))
5151 #endif
5252
5353 #define HASH_LOG (14U)
5555
5656 // This is used in LZ4 and seems to work pretty well here too
5757 #define HASH_FUNCTION(v, s, h) { \
58 v = (s * 2654435761U) >> (32U - h); \
58 (v) = ((s) * 2654435761U) >> (32U - (h)); \
5959 }
6060
6161
8686 #endif
8787
8888 #if defined(__SSE2__)
89 static uint8_t *get_run_16(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) {
89 uint8_t *get_run_16(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) {
9090 uint8_t x = ip[-1];
9191
9292 while (ip < (ip_bound - sizeof(__m128i))) {
142142
143143
144144 /* Return the byte that starts to differ */
145 static uint8_t *get_match(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) {
145 uint8_t *get_match(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) {
146146 #if !defined(BLOSC_STRICT_ALIGN)
147147 while (ip < (ip_bound - sizeof(int64_t))) {
148148 if (*(int64_t*)ref != *(int64_t*)ip) {
212212 #endif
213213
214214
215 static uint8_t* get_run_or_match(uint8_t* ip, const uint8_t* ip_bound, const uint8_t* ref, bool run) {
215 static uint8_t* get_run_or_match(uint8_t* ip, uint8_t* ip_bound, const uint8_t* ref, bool run) {
216216 if (BLOSCLZ_UNLIKELY(run)) {
217217 #if defined(__AVX2__)
218218 // Extensive experiments on AMD Ryzen3 say that regular get_run is faster
243243
244244
245245 #define LITERAL(ip, op, op_limit, anchor, copy) { \
246 if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) \
246 if (BLOSCLZ_UNLIKELY((op) + 2 > (op_limit))) \
247247 goto out; \
248 *op++ = *anchor++; \
249 ip = anchor; \
250 copy++; \
251 if (BLOSCLZ_UNLIKELY(copy == MAX_COPY)) { \
252 copy = 0; \
253 *op++ = MAX_COPY-1; \
248 *(op)++ = *(anchor)++; \
249 (ip) = (anchor); \
250 (copy)++; \
251 if (BLOSCLZ_UNLIKELY((copy) == MAX_COPY)) { \
252 (copy) = 0; \
253 *(op)++ = MAX_COPY-1; \
254254 } \
255255 }
256256
257257 #define LITERAL2(ip, anchor, copy) { \
258 oc++; anchor++; \
259 ip = anchor; \
260 copy++; \
261 if (BLOSCLZ_UNLIKELY(copy == MAX_COPY)) { \
262 copy = 0; \
258 oc++; (anchor)++; \
259 (ip) = (anchor); \
260 (copy)++; \
261 if (BLOSCLZ_UNLIKELY((copy) == MAX_COPY)) { \
262 (copy) = 0; \
263263 oc++; \
264264 } \
265265 }
266266
267 #define MATCH_SHORT(op, op_limit, len, distance) { \
268 if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) \
267 #define MATCH_SHORT(op, op_limit, len, distance) { \
268 if (BLOSCLZ_UNLIKELY((op) + 2 > (op_limit))) \
269 goto out; \
270 *(op)++ = (uint8_t)(((len) << 5U) + ((distance) >> 8U));\
271 *(op)++ = (uint8_t)(((distance) & 255U)); \
272 }
273
274 #define MATCH_LONG(op, op_limit, len, distance) { \
275 if (BLOSCLZ_UNLIKELY((op) + 1 > (op_limit))) \
269276 goto out; \
270 *op++ = (uint8_t)((len << 5U) + (distance >> 8U)); \
271 *op++ = (uint8_t)((distance & 255U)); \
272 }
273
274 #define MATCH_LONG(op, op_limit, len, distance) { \
275 if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \
277 *(op)++ = (uint8_t)((7U << 5U) + ((distance) >> 8U)); \
278 for ((len) -= 7; (len) >= 255; (len) -= 255) { \
279 if (BLOSCLZ_UNLIKELY((op) + 1 > (op_limit))) \
280 goto out; \
281 *(op)++ = 255; \
282 } \
283 if (BLOSCLZ_UNLIKELY((op) + 2 > (op_limit))) \
276284 goto out; \
277 *op++ = (uint8_t)((7U << 5U) + (distance >> 8U)); \
278 for (len -= 7; len >= 255; len -= 255) { \
279 if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \
280 goto out; \
281 *op++ = 255; \
282 } \
283 if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) \
284 goto out; \
285 *op++ = (uint8_t)len; \
286 *op++ = (uint8_t)((distance & 255U)); \
285 *(op)++ = (uint8_t)(len); \
286 *(op)++ = (uint8_t)(((distance) & 255U)); \
287287 }
288288
289289 #define MATCH_SHORT_FAR(op, op_limit, len, distance) { \
290 if (BLOSCLZ_UNLIKELY(op + 4 > op_limit)) \
290 if (BLOSCLZ_UNLIKELY((op) + 4 > (op_limit))) \
291291 goto out; \
292 *op++ = (uint8_t)((len << 5U) + 31); \
293 *op++ = 255; \
294 *op++ = (uint8_t)(distance >> 8U); \
295 *op++ = (uint8_t)(distance & 255U); \
292 *(op)++ = (uint8_t)(((len) << 5U) + 31); \
293 *(op)++ = 255; \
294 *(op)++ = (uint8_t)((distance) >> 8U); \
295 *(op)++ = (uint8_t)((distance) & 255U); \
296296 }
297297
298298 #define MATCH_LONG_FAR(op, op_limit, len, distance) { \
299 if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \
299 if (BLOSCLZ_UNLIKELY((op) + 1 > (op_limit))) \
300300 goto out; \
301 *op++ = (7U << 5U) + 31; \
302 for (len -= 7; len >= 255; len -= 255) { \
303 if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \
301 *(op)++ = (7U << 5U) + 31; \
302 for ((len) -= 7; (len) >= 255; (len) -= 255) { \
303 if (BLOSCLZ_UNLIKELY((op) + 1 > (op_limit))) \
304304 goto out; \
305 *op++ = 255; \
305 *(op)++ = 255; \
306306 } \
307 if (BLOSCLZ_UNLIKELY(op + 4 > op_limit)) \
307 if (BLOSCLZ_UNLIKELY((op) + 4 > (op_limit))) \
308308 goto out; \
309 *op++ = (uint8_t)len; \
310 *op++ = 255; \
311 *op++ = (uint8_t)(distance >> 8U); \
312 *op++ = (uint8_t)(distance & 255U); \
309 *(op)++ = (uint8_t)(len); \
310 *(op)++ = 255; \
311 *(op)++ = (uint8_t)((distance) >> 8U); \
312 *(op)++ = (uint8_t)((distance) & 255U); \
313313 }
314314
315315
376376 ip = get_run_or_match(ip, ip_bound, ref, !distance);
377377
378378 ip -= ipshift;
379 unsigned len = (int)(ip - anchor);
379 int len = (int)(ip - anchor);
380380 if (len < minlen) {
381381 LITERAL2(ip, anchor, copy)
382382 continue;
412412 oc++;
413413 }
414414
415 double ic;
416 ic = (double)(ip - ibase);
415 double ic = (double)(ip - ibase);
417416 return ic / (double)oc;
418417 }
419418
431430 // discard probes with small compression ratios (too expensive)
432431 double cratio_[10] = {0, 2, 1.5, 1.2, 1.2, 1.2, 1.2, 1.15, 1.1, 1.0};
433432 if (cratio < cratio_[clevel]) {
434 goto out;
433 goto out;
435434 }
436435
437436 /* When we go back in a match (shift), we obtain quite different compression properties.
452451 ipshift = 3;
453452 minlen = 3;
454453 }
454 else {
455 minlen = 4;
456 }
455457
456458 uint8_t hashlog_[10] = {0, HASH_LOG - 2, HASH_LOG - 1, HASH_LOG, HASH_LOG,
457459 HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG};
458460 uint8_t hashlog = hashlog_[clevel];
459461
460462 uint8_t* ip = ibase;
461 const uint8_t* ip_bound = ibase + length - 1;
462 const uint8_t* ip_limit = ibase + length - 12;
463 uint8_t* ip_bound = ibase + length - 1;
464 uint8_t* ip_limit = ibase + length - 12;
463465 uint8_t* op = (uint8_t*)output;
464466 const uint8_t* op_limit = op + maxout;
467 uint32_t seq;
468 uint8_t copy;
469 uint32_t hval;
465470
466471 /* input and output buffer cannot be less than 16 and 66 bytes or we can get into trouble */
467472 if (length < 16 || maxout < 66) {
473478 memset(htab, 0, (1U << hashlog) * sizeof(uint32_t));
474479
475480 /* we start with literal copy */
476 uint8_t copy = 4;
481 copy = 4;
477482 *op++ = MAX_COPY - 1;
478483 *op++ = *ip++;
479484 *op++ = *ip++;
487492 uint8_t* anchor = ip; /* comparison starting-point */
488493
489494 /* find potential match */
490 uint32_t seq = BLOSCLZ_READU32(ip);
491 uint32_t hval;
495 seq = BLOSCLZ_READU32(ip);
492496 HASH_FUNCTION(hval, seq, hashlog)
493497 ref = ibase + htab[hval];
494498
685689 while (1) {
686690 if (ctrl >= 32) {
687691 // match
688 int32_t len = (ctrl >> 5U) - 1 ;
689 int32_t ofs = (ctrl & 31U) << 8U;
692 int32_t len = (int32_t)(ctrl >> 5U) - 1 ;
693 int32_t ofs = (int32_t)(ctrl & 31U) << 8U;
690694 uint8_t code;
691695 const uint8_t* ref = op - ofs;
692696
752756 }
753757 else {
754758 #endif
755 op = copy_match(op, ref, (unsigned) len);
759 op = copy_match(op, ref, (unsigned) len);
756760 #if defined(__AVX2__)
757761 }
758762 #endif
+5
-1
blosc/blosclz.h less more
00 /*********************************************************************
11 Blosc - Blocked Shuffling and Compression Library
22
3 Author: Francesc Alted <francesc@blosc.org>
3 Copyright (C) 2021 The Blosc Developers <blosc@blosc.org>
4 https://blosc.org
5 License: BSD 3-Clause (see LICENSE.txt)
46
57 See LICENSE.txt for details about copyright and rights to use.
68 **********************************************************************/
1820 #if defined (__cplusplus)
1921 extern "C" {
2022 #endif
23
24 #define BLOSCLZ_VERSION_STRING "2.5.1"
2125
2226
2327 /**
+1
-1
tests/Makefile less more
88 EXECUTABLES := $(patsubst %.c, %.exe, $(SOURCES))
99
1010 # Support for internal LZ4 and LZ4HC
11 LZ4_DIR = ../internal-complibs/lz4-1.9.3
11 LZ4_DIR = ../internal-complibs/lz4-1.9.4
1212 CFLAGS += -DHAVE_LZ4 -I$(LZ4_DIR)
1313 BLOSC_LIB += $(wildcard $(LZ4_DIR)/*.c)
1414