Commit f6a41664d5e511d38b392e455e1020cf4a4b4b0e - c-blosc

New upstream version 1.20.1+ds1 Håvard Flaget Aasen 3 years ago

45 changed file(s) with 1573 addition(s) and 1041 deletion(s). Raw diff Collapse all Expand all

+161

-0

.github/workflows/cmake.yml less more

	0	name: CI CMake
	1	on: [push, pull_request]
	2	jobs:
	3	ci-cmake:
	4	name: ${{ matrix.name }}
	5	runs-on: ${{ matrix.os }}
	6	strategy:
	7	fail-fast: false
	8	matrix:
	9	name: [
	10	Ubuntu GCC,
	11	Ubuntu GCC OSB,
	12	Ubuntu GCC External LZ4,
	13	Ubuntu GCC External SNAPPY,
	14	Ubuntu GCC External ZLIB,
	15	Ubuntu GCC External ZSTD,
	16	Ubuntu Clang,
	17	Ubuntu Clang No SSE2,
	18	Ubuntu Clang No AVX2,
	19	Ubuntu Clang No AVX2 No SSE2,
	20	Ubuntu Clang No LZ4,
	21	Ubuntu Clang No ZLIB,
	22	Ubuntu Clang No ZSTD,
	23	Windows MSVC Win32,
	24	Windows MSVC Win64,
	25	Windows GCC,
	26	macOS Clang,
	27	macOS GCC
	28	]
	29	include:
	30	- name: Ubuntu GCC
	31	os: ubuntu-latest
	32	compiler: gcc
	33
	34	# Out of source build
	35	- name: Ubuntu GCC OSB
	36	os: ubuntu-latest
	37	compiler: gcc
	38	build-dir: ../build
	39	build-src-dir: ../c-blosc
	40
	41	- name: Ubuntu GCC External LZ4
	42	os: ubuntu-latest
	43	compiler: gcc
	44	packages: liblz4-1 liblz4-dev
	45	cmake-args: -DPREFER_EXTERNAL_LZ4=ON
	46
	47	- name: Ubuntu GCC External SNAPPY
	48	os: ubuntu-latest
	49	compiler: gcc
	50	packages: libsnappy-dev
	51	cmake-args: -DDEACTIVATE_SNAPPY=OFF
	52
	53	- name: Ubuntu GCC External ZLIB
	54	os: ubuntu-latest
	55	compiler: gcc
	56	packages: zlib1g-dev
	57	cmake-args: -DPREFER_EXTERNAL_ZLIB=ON
	58
	59	- name: Ubuntu GCC External ZSTD
	60	os: ubuntu-latest
	61	compiler: gcc
	62	packages: zstd libzstd-dev
	63	cmake-args: -DPREFER_EXTERNAL_ZSTD=ON
	64
	65	- name: Ubuntu Clang
	66	os: ubuntu-latest
	67	compiler: clang
	68
	69	- name: Ubuntu Clang No SSE2
	70	os: ubuntu-latest
	71	compiler: clang
	72	cmake-args: -DDEACTIVATE_SSE2=ON
	73
	74	- name: Ubuntu Clang No AVX2
	75	os: ubuntu-latest
	76	compiler: clang
	77	cmake-args: -DDEACTIVATE_AVX2=ON
	78
	79	- name: Ubuntu Clang No AVX2 No SSE2
	80	os: ubuntu-latest
	81	compiler: clang
	82	cmake-args: -DDEACTIVATE_AVX2=ON -DDEACTIVATE_SSE2=ON
	83
	84	- name: Ubuntu Clang No LZ4
	85	os: ubuntu-latest
	86	compiler: clang
	87	cmake-args: -DDEACTIVATE_LZ4=ON
	88
	89	- name: Ubuntu Clang No ZLIB
	90	os: ubuntu-latest
	91	compiler: clang
	92	cmake-args: -DDEACTIVATE_ZLIB=ON
	93
	94	- name: Ubuntu Clang No ZSTD
	95	os: ubuntu-latest
	96	compiler: clang
	97	cmake-args: -DDEACTIVATE_ZSTD=ON
	98
	99	- name: Windows MSVC Win32
	100	os: windows-latest
	101	compiler: cl
	102	cmake-args: -A Win32
	103
	104	- name: Windows MSVC Win64
	105	os: windows-latest
	106	compiler: cl
	107	cmake-args: -A x64
	108
	109	- name: Windows GCC
	110	os: windows-latest
	111	compiler: gcc
	112	cmake-args: -G Ninja
	113
	114	- name: macOS Clang
	115	os: macOS-latest
	116	compiler: clang
	117
	118	- name: macOS GCC
	119	os: macOS-latest
	120	compiler: gcc
	121
	122	steps:
	123	- uses: actions/checkout@v1
	124
	125	- name: Install packages (Ubuntu)
	126	if: runner.os == 'Linux' && matrix.packages
	127	run: \|
	128	sudo apt-get update
	129	sudo apt-get install -y ${{ matrix.packages }}
	130
	131	- name: Install packages (Windows)
	132	if: runner.os == 'Windows'
	133	run: \|
	134	choco install ninja ${{ matrix.packages }}
	135
	136	- name: Install packages (macOS)
	137	if: runner.os == 'macOS'
	138	run: \|
	139	brew install ninja ${{ matrix.packages }}
	140
	141	- name: Generate project files
	142	run: \|
	143	mkdir ${{ matrix.build-dir \|\| '.not-used' }}
	144	cd ${{ matrix.build-dir \|\| '.' }}
	145	cmake ${{ matrix.build-src-dir \|\| '.' }} ${{ matrix.cmake-args }} -DCMAKE_BUILD_TYPE=${{ matrix.build-config \|\| 'Release' }} -DBUILD_SHARED_LIBS=OFF -DBUILD_FUZZERS=ON
	146	env:
	147	CC: ${{ matrix.compiler }}
	148	CFLAGS: ${{ matrix.cflags }}
	149	LDFLAGS: ${{ matrix.ldflags }}
	150	CI: true
	151
	152	- name: Compile source code
	153	run: \|
	154	cd ${{ matrix.build-dir \|\| '.' }}
	155	cmake --build . --config ${{ matrix.build-config \|\| 'Release' }}
	156
	157	- name: Run test cases
	158	run: \|
	159	cd ${{ matrix.build-dir \|\| '.' }}
	160	ctest -C Release --output-on-failure --max-width 120

+23

-0

.github/workflows/fuzz.yml less more

	0	name: CIFuzz
	1	on: [pull_request]
	2	jobs:
	3	Fuzzing:
	4	runs-on: ubuntu-latest
	5	steps:
	6	- name: Build Fuzzers
	7	uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
	8	with:
	9	oss-fuzz-project-name: 'c-blosc'
	10	dry-run: false
	11	- name: Run Fuzzers
	12	uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
	13	with:
	14	oss-fuzz-project-name: 'c-blosc'
	15	fuzz-seconds: 600
	16	dry-run: false
	17	- name: Upload Crash
	18	uses: actions/upload-artifact@v1
	19	if: failure()
	20	with:
	21	name: artifacts
	22	path: ./out/artifacts

-80

~~.travis.yml~~ less more

0		env:
1		global:
2		- CONAN_USERNAME: "francescalted"
3		- CONAN_LOGIN_USERNAME: "francescalted"
4		- CONAN_CHANNEL: "stable"
5		- CONAN_UPLOAD: "https://api.bintray.com/conan/blosc/Conan"
6		- CONAN_TOTAL_PAGES: 2
7
8		linux: &linux
9		os: linux
10		sudo: required
11		language: python
12		python: "3.6"
13		services:
14		- docker
15
16		osx: &osx
17		os: osx
18		language: generic
19
20		matrix:
21		include:
22		- <<: *osx
23		osx_image: xcode8.3
24		env: CONAN_APPLE_CLANG_VERSIONS=8.1 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1
25		- <<: *osx
26		osx_image: xcode8.3
27		env: CONAN_APPLE_CLANG_VERSIONS=8.1 CONAN_CURRENT_PAGE=2
28
29		- <<: *osx
30		osx_image: xcode9
31		env: CONAN_APPLE_CLANG_VERSIONS=9.0 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1
32		- <<: *osx
33		osx_image: xcode9
34		env: CONAN_APPLE_CLANG_VERSIONS=9.0 CONAN_CURRENT_PAGE=2
35
36		- <<: *linux
37		env: CONAN_GCC_VERSIONS=4.9 CONAN_DOCKER_IMAGE=lasote/conangcc49 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1
38		- <<: *linux
39		env: CONAN_GCC_VERSIONS=4.9 CONAN_DOCKER_IMAGE=lasote/conangcc49 CONAN_CURRENT_PAGE=2
40
41		- <<: *linux
42		env: CONAN_GCC_VERSIONS=5 CONAN_DOCKER_IMAGE=lasote/conangcc5 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1
43		- <<: *linux
44		env: CONAN_GCC_VERSIONS=5 CONAN_DOCKER_IMAGE=lasote/conangcc5 CONAN_CURRENT_PAGE=2
45
46		- <<: *linux
47		env: CONAN_GCC_VERSIONS=6 CONAN_DOCKER_IMAGE=lasote/conangcc6 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1
48		- <<: *linux
49		env: CONAN_GCC_VERSIONS=6 CONAN_DOCKER_IMAGE=lasote/conangcc6 CONAN_CURRENT_PAGE=2
50
51		- <<: *linux
52		env: CONAN_GCC_VERSIONS=7 CONAN_DOCKER_IMAGE=lasote/conangcc7 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1
53		- <<: *linux
54		env: CONAN_GCC_VERSIONS=7 CONAN_DOCKER_IMAGE=lasote/conangcc7 CONAN_CURRENT_PAGE=2
55
56		- <<: *linux
57		env: CONAN_CLANG_VERSIONS=3.9 CONAN_DOCKER_IMAGE=lasote/conanclang39 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1
58		- <<: *linux
59		env: CONAN_CLANG_VERSIONS=3.9 CONAN_DOCKER_IMAGE=lasote/conanclang39 CONAN_CURRENT_PAGE=2
60
61		- <<: *linux
62		env: CONAN_CLANG_VERSIONS=4.0 CONAN_DOCKER_IMAGE=lasote/conanclang40 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1
63		- <<: *linux
64		env: CONAN_CLANG_VERSIONS=4.0 CONAN_DOCKER_IMAGE=lasote/conanclang40 CONAN_CURRENT_PAGE=2
65
66		before_script:
67		- if [ "$TRAVIS_OS_NAME" == "osx" ]; then
68		brew update;
69		brew upgrade pyenv;
70		pyenv install 3.6.6;
71		eval "$(pyenv init -)";
72		pyenv global 3.6.6;
73		python3 --version;
74		pip3 install --upgrade pip;
75		fi
76		- pip3 install conan_package_tools==0.19.3
77
78		script:
79		- python3 build.py

-3

ANNOUNCE.rst less more

0	0	===============================================================
1		Announcing C-Blosc 1.17.1
	1	Announcing C-Blosc 1.20.1
2	2	A blocking, shuffling and lossless compression library for C
3	3	===============================================================
4	4
5	5	What is new?
6	6	============
7	7
8		A maintenance release where LZ4 and Zstd internal codecs have been updated
9		to latest versions.
	8	This is a maintenance release. Vendored zlib 1.2.8 is now compatible
	9	with Python 3.8 in recent Mac OSX. For details, see:
	10	https://github.com/Blosc/python-blosc/issues/229
10	11
11	12	For more info, please see the release notes in:
12	13

+30

-24

CMakeLists.txt less more

8	8	# build the shared library version of the Blosc library
9	9	# BUILD_TESTS: default ON
10	10	# build test programs and generates the "test" target
	11	# BUILD_FUZZERS: default ON
	12	# build fuzz test programs and generates the "test" target
11	13	# BUILD_BENCHMARKS: default ON
12	14	# build the benchmark program
13	15	# DEACTIVATE_SSE2: default OFF

16	18	# do not attempt to build with AVX2 instructions
17	19	# DEACTIVATE_LZ4: default OFF
18	20	# do not include support for the LZ4 library
19		# DEACTIVATE_SNAPPY: default OFF
	21	# DEACTIVATE_SNAPPY: default ON
20	22	# do not include support for the Snappy library
21	23	# DEACTIVATE_ZLIB: default OFF
22	24	# do not include support for the Zlib library

26	28	# do not check for symbols in shared or static libraries
27	29	# PREFER_EXTERNAL_LZ4: default OFF
28	30	# when found, use the installed LZ4 libs instead of included
29		# sources
30		# PREFER_EXTERNAL_SNAPPY: default OFF
31		# when found, use the installed Snappy libs instead of included
32	31	# sources
33	32	# PREFER_EXTERNAL_ZLIB: default OFF
34	33	# when found, use the installed zlib libs instead of included

74	73	if (NOT CMAKE_VERSION VERSION_LESS 3.3)
75	74	cmake_policy(SET CMP0063 NEW)
76	75	endif()
77		project(blosc)
	76	project(blosc C)
78	77
79	78	# parse the full version numbers from blosc.h
80	79	file(READ ${CMAKE_CURRENT_SOURCE_DIR}/blosc/blosc.h _blosc_h_contents)

95	94	option(BUILD_SHARED
96	95	"Build a shared library version of the blosc library." ON)
97	96	option(BUILD_TESTS
98		"Build test programs form the blosc compression library" ON)
	97	"Build test programs from the blosc compression library" ON)
	98	option(BUILD_FUZZERS
	99	"Build fuzzer programs from the blosc compression library" ${BUILD_STATIC})
99	100	option(BUILD_BENCHMARKS
100		"Build benchmark programs form the blosc compression library" ON)
	101	"Build benchmark programs from the blosc compression library" ON)
101	102	option(DEACTIVATE_SSE2
102		"Do not attempt to build with SSE2 instructions" OFF)
	103	"Do not attempt to build with SSE2 instructions" OFF)
103	104	option(DEACTIVATE_AVX2
104		"Do not attempt to build with AVX2 instructions" OFF)
	105	"Do not attempt to build with AVX2 instructions" OFF)
105	106	option(DEACTIVATE_LZ4
106	107	"Do not include support for the LZ4 library." OFF)
107	108	option(DEACTIVATE_SNAPPY
108		"Do not include support for the Snappy library." OFF)
	109	"Do not include support for the Snappy library." ON)
109	110	option(DEACTIVATE_ZLIB
110	111	"Do not include support for the Zlib library." OFF)
111	112	option(DEACTIVATE_ZSTD
112		"Do not include support for the Zstd library." OFF)
	113	"Do not include support for the Zstd library." OFF)
113	114	option(DEACTIVATE_SYMBOLS_CHECK
114		"Do not check for symbols in shared or static libraries." ON)
	115	"Do not check for symbols in shared or static libraries." ON)
115	116	option(PREFER_EXTERNAL_LZ4
116	117	"Find and use external LZ4 library instead of included sources." OFF)
117		option(PREFER_EXTERNAL_SNAPPY
118		"Find and use external Snappy library instead of included sources." OFF)
119	118	option(PREFER_EXTERNAL_ZLIB
120	119	"Find and use external Zlib library instead of included sources." OFF)
121	120	option(PREFER_EXTERNAL_ZSTD

136	135	endif(NOT DEACTIVATE_LZ4)
137	136
138	137	if(NOT DEACTIVATE_SNAPPY)
139		if(PREFER_EXTERNAL_SNAPPY)
140		find_package(Snappy)
141		else()
142		message(STATUS "Using Snappy internal sources.")
143		endif(PREFER_EXTERNAL_SNAPPY)
144		# HAVE_SNAPPY will be set to true because even if the library is not found,
145		# we will use the included sources for it
146		set(HAVE_SNAPPY TRUE)
	138	find_package(Snappy)
	139	if(SNAPPY_FOUND)
	140	message(STATUS "Activating support for SNAPPY.")
	141	set(HAVE_SNAPPY TRUE)
	142	else()
	143	message(STATUS "SNAPPY not found. De-activating support for it.")
	144	endif()
147	145	endif(NOT DEACTIVATE_SNAPPY)
148	146
149	147	if(NOT DEACTIVATE_ZLIB)

254	252	set(COMPILER_SUPPORT_SSE2 FALSE)
255	253	endif()
256	254
257		# disable AVX2 if specified
258		if(DEACTIVATE_AVX2)
	255	# disable AVX2 if specified or if SSE is deactivated
	256	if(DEACTIVATE_AVX2 OR DEACTIVATE_SSE2)
259	257	set(COMPILER_SUPPORT_AVX2 FALSE)
260	258	endif()
261	259

322	320	add_subdirectory(tests)
323	321	add_subdirectory(compat)
324	322	endif(BUILD_TESTS)
	323
	324	if(BUILD_FUZZERS)
	325	if(NOT BUILD_STATIC)
	326	message(FATAL_ERROR "BUILD_FUZZERS requires BUILD_STATIC to be enabled.")
	327	endif()
	328	enable_testing()
	329	add_subdirectory(tests/fuzz)
	330	endif(BUILD_FUZZERS)
325	331
326	332	if(BUILD_BENCHMARKS)
327	333	add_subdirectory(bench)

-5

~~CODE_OF_CONDUCT.md~~ less more

0		# Code of Conduct
1
2		The Blosc community has adopted a Code of Conduct that we expect project participants to adhere to.
3		Please read the [full text](https://github.com/Blosc/CodeOfConduct/README.md)
4		so that you can understand what actions will and will not be tolerated.

-4

README.md less more

2	2	\|--------\|---------\|-----\|
3	3	\| Blosc Development Team \| blosc@blosc.org \| http://www.blosc.org \|
4	4
5		\| Gitter \| Travis CI \| Appveyor \| NumFOCUS \|
6		\|--------\|-----------\|----------\|----------\|
7		\| [![Build Status](https://badges.gitter.im/Blosc/c-blosc.svg)](https://gitter.im/Blosc/c-blosc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) \| [![Build Status](https://travis-ci.org/Blosc/c-blosc.svg?branch=master)](https://travis-ci.org/Blosc/c-blosc) \| [![Build Status](https://ci.appveyor.com/api/projects/status/3mlyjc1ak0lbkmte?svg=true)](https://ci.appveyor.com/project/FrancescAlted/c-blosc/branch/master) \| [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](http://numfocus.org) \|
8
	5	\| Gitter \| GH Actions \| NumFOCUS \| Code of Conduct \|
	6	\|--------\|------------\|----------\|-----------------\|
	7	\| [![Gitter](https://badges.gitter.im/Blosc/c-blosc.svg)](https://gitter.im/Blosc/c-blosc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) \| [![CI CMake](https://github.com/Blosc/c-blosc/workflows/CI%20CMake/badge.svg)](https://github.com/Blosc/c-blosc/actions?query=workflow%3A%22CI+CMake%22) \| [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](http://numfocus.org) \| [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](code_of_conduct.md) \|
9	8
10	9	## What is it?
11	10

+97

-0

README_CHUNK_FORMAT.rst less more

	0	Blosc Chunk Format
	1	==================
	2
	3	The chunk is composed by a header and a blocks / splits section::
	4
	5	+---------+--------+---------+
	6	\| header \| blocks / splits \|
	7	+---------+--------+---------+
	8
	9	These are described below.
	10
	11	The header section
	12	------------------
	13
	14	Blosc (as of Version 1.0.0) has the following 16 byte header that stores
	15	information about the compressed buffer::
	16
	17	\|-0-\|-1-\|-2-\|-3-\|-4-\|-5-\|-6-\|-7-\|-8-\|-9-\|-A-\|-B-\|-C-\|-D-\|-E-\|-F-\|
	18	^ ^ ^ ^ \| nbytes \| blocksize \| cbytes \|
	19	\| \| \| \|
	20	\| \| \| +--typesize
	21	\| \| +------flags
	22	\| +----------versionlz
	23	+--------------version
	24
	25	Datatypes of the header entries
	26	-------------------------------
	27
	28	All entries are little endian.
	29
	30	:version:
	31	(``uint8``) Blosc format version.
	32	:versionlz:
	33	(``uint8``) Version of the internal compressor used.
	34	:flags and compressor enumeration:
	35	(``bitfield``) The flags of the buffer
	36
	37	:bit 0 (``0x01``):
	38	Whether the byte-shuffle filter has been applied or not.
	39	:bit 1 (``0x02``):
	40	Whether the internal buffer is a pure memcpy or not.
	41	:bit 2 (``0x04``):
	42	Whether the bit-shuffle filter has been applied or not.
	43	:bit 3 (``0x08``):
	44	Reserved, must be zero.
	45	:bit 4 (``0x10``):
	46	If set, the blocks will not be split in sub-blocks during compression.
	47	:bit 5 (``0x20``):
	48	Part of the enumeration for compressors.
	49	:bit 6 (``0x40``):
	50	Part of the enumeration for compressors.
	51	:bit 7 (``0x80``):
	52	Part of the enumeration for compressors.
	53
	54	The last three bits form an enumeration that allows to use alternative
	55	compressors.
	56
	57	:``0``:
	58	``blosclz``
	59	:``1``:
	60	``lz4`` or ``lz4hc``
	61	:``2``:
	62	``snappy``
	63	:``3``:
	64	``zlib``
	65	:``4``:
	66	``zstd``
	67
	68	:typesize:
	69	(``uint8``) Number of bytes for the atomic type.
	70	:nbytes:
	71	(``uint32``) Uncompressed size of the buffer (this header is not included).
	72	:blocksize:
	73	(``uint32``) Size of internal blocks.
	74	:cbytes:
	75	(``uint32``) Compressed size of the buffer (including this header).
	76
	77	The blocks / splits section
	78	---------------------------
	79
	80	After the header, there come the blocks / splits section. Blocks are equal-sized parts of the chunk, except for the last block that can be shorter or equal than the rest.
	81
	82	At the beginning of the blocks section, there come a list of `int32_t bstarts` to indicate where the different encoded blocks starts (counting from the end of this `bstarts` section)::
	83
	84	+=========+=========+========+=========+
	85	\| bstart0 \| bstart1 \| ... \| bstartN \|
	86	+=========+=========+========+=========+
	87
	88	Finally, it comes the actual list of compressed blocks / splits data streams. It turns out that a block may optionally (see bit 4 in `flags` above) be further split in so-called splits which are the actual data streams that are transmitted to codecs for compression. If a block is not split, then the split is equivalent to a whole block. Before each split in the list, there is the compressed size of it, expressed as an `int32_t`::
	89
	90	+========+========+========+========+========+========+========+
	91	\| csize0 \| split0 \| csize1 \| split1 \| ... \| csizeN \| splitN \|
	92	+========+========+========+========+========+========+========+
	93
	94
	95	Note: all the integers are stored in little endian.
	96

-65

~~README_HEADER.rst~~ less more

0		Blosc Header Format
1		===================
2
3		Blosc (as of Version 1.0.0) has the following 16 byte header that stores
4		information about the compressed buffer::
5
6		\|-0-\|-1-\|-2-\|-3-\|-4-\|-5-\|-6-\|-7-\|-8-\|-9-\|-A-\|-B-\|-C-\|-D-\|-E-\|-F-\|
7		^ ^ ^ ^ \| nbytes \| blocksize \| ctbytes \|
8		\| \| \| \|
9		\| \| \| +--typesize
10		\| \| +------flags
11		\| +----------versionlz
12		+--------------version
13
14		Datatypes of the Header Entries
15		-------------------------------
16
17		All entries are little endian.
18
19		:version:
20		(``uint8``) Blosc format version.
21		:versionlz:
22		(``uint8``) Version of the internal compressor used.
23		:flags and compressor enumeration:
24		(``bitfield``) The flags of the buffer
25
26		:bit 0 (``0x01``):
27		Whether the byte-shuffle filter has been applied or not.
28		:bit 1 (``0x02``):
29		Whether the internal buffer is a pure memcpy or not.
30		:bit 2 (``0x04``):
31		Whether the bit-shuffle filter has been applied or not.
32		:bit 3 (``0x08``):
33		Reserved, must be zero.
34		:bit 4 (``0x10``):
35		If set, the blocks will not be split in sub-blocks during compression.
36		:bit 5 (``0x20``):
37		Part of the enumeration for compressors.
38		:bit 6 (``0x40``):
39		Part of the enumeration for compressors.
40		:bit 7 (``0x80``):
41		Part of the enumeration for compressors.
42
43		The last three bits form an enumeration that allows to use alternative
44		compressors.
45
46		:``0``:
47		``blosclz``
48		:``1``:
49		``lz4`` or ``lz4hc``
50		:``2``:
51		``snappy``
52		:``3``:
53		``zlib``
54		:``4``:
55		``zstd``
56
57		:typesize:
58		(``uint8``) Number of bytes for the atomic type.
59		:nbytes:
60		(``uint32``) Uncompressed size of the buffer.
61		:blocksize:
62		(``uint32``) Size of internal blocks.
63		:ctbytes:
64		(``uint32``) Compressed size of the buffer.

+71

-0

RELEASE_NOTES.rst less more

0	0	===========================
1	1	Release notes for C-Blosc
2	2	===========================
	3
	4	Changes from 1.20.0 to 1.20.1
	5	=============================
	6
	7	* Added `<unistd.h>` in vendored zlib 1.2.8 for compatibility with Python 3.8
	8	in recent Mac OSX. For details, see:
	9	https://github.com/Blosc/python-blosc/issues/229
	10
	11	Changes from 1.19.1 to 1.20.0
	12	=============================
	13
	14	* More saftey checks have been implemented so that potential flaws
	15	discovered by new fuzzers in OSS-Fuzzer are fixed now. Thanks to
	16	Nathan Moinvaziri (@nmoinvaz).
	17
	18	* BloscLZ updated to 2.3.0. Expect better compression ratios for faster
	19	codecs. For details, see our new blog post:
	20	https://blosc.org/posts/beast-release/
	21
	22	* Fixed the `_xgetbv()` collision. Thanks to Michał Górny (@mgorny).
	23
	24	* The chunk format has been fully described so that 3rd party software
	25	may come with a different implementation, but still compatible with
	26	C-Blosc chunks.
	27
	28
	29	Changes from 1.19.0 to 1.19.1
	30	=============================
	31
	32	- pthread_create() errors are now handled and propagated back to the user.
	33	See https://github.com/Blosc/c-blosc/pull/299.
	34
	35
	36	Changes from 1.18.1 to 1.19.0
	37	=============================
	38
	39	- The length of automatic blocksizes for fast codecs (lz4, blosclz) has
	40	been incremented quite a bit (up to 256 KB) for better compression ratios.
	41	The performance in modern CPUs (with at least 256 KB in L2 cache) should
	42	be better too (for older CPUs the performance should stay roughly the same).
	43
	44	- Continuous integration has been migrated to GitHub actions and much
	45	more scenarios are tested (specially linking with external codecs).
	46	Also, a new OSS-Fuzz workflow has been added for increased detection
	47	of possible vulnerabilities. Thanks to Nathan Moinvaziri.
	48
	49	- For small buffers that cannot be compressed (typically < 128 bytes),
	50	`blosc_compress()` returns now a 0 (cannot compress) instead of a negative
	51	number (internal error). See https://github.com/Blosc/c-blosc/pull/294.
	52	Thanks to @kalvdans for providing the initial patch.
	53
	54	- blosclz codec updated to 2.1.0. Expect better compression ratios and
	55	performance in a wider variety of scenarios.
	56
	57	- `blosc_decompress_unsafe()`, `blosc_decompress_ctx_unsafe()` and
	58	`blosc_getitem_unsafe()` have been removed because they are dangerous
	59	and after latest improvements, they should not be used in production.
	60
	61	- zstd codec updated to 1.4.5.
	62
	63	- Conan packaging has been deprecated (from now on, we should try
	64	to focus on supporting wheels only).
	65
	66
	67	Changes from 1.17.1 to 1.18.1
	68	=============================
	69
	70	- Fixed the copy of the leftovers of a chunk when its size is not a
	71	multiple of the typesize. Although this is a very unusual situation,
	72	it can certainly happen (e.g.
	73	https://github.com/Blosc/python-blosc/issues/220).
3	74
4	75
5	76	Changes from 1.17.0 to 1.17.1

+10

-3

RELEASING.rst less more

44	44	$ cd ../compat
45	45	$ export LD_LIBRARY_PATH=../build/blosc
46	46	$ gcc -o filegen filegen.c -L$LD_LIBRARY_PATH -lblosc -I../blosc
47		$ ./filegen compress lz4 blosc-lz4-1.y.z.cdata
	47	$ ./filegen compress lz4 blosc-1.y.z-lz4.cdata
48	48
49	49	In order to make sure that we are not breaking forward compatibility,
50	50	link and run the `compat/filegen` utility against different versions of

57	57
58	58	Then, test the file created with the new version with::
59	59
60		$ ./filegen decompress blosc-lz4-1.y.z.cdata
	60	$ ./filegen decompress blosc-1.y.z-lz4.cdata
	61
	62	If that works and you want to keep track of this for future compatibility checks
	63	just add the new file to the suite::
	64
	65	$ git add blosc-1.y.z-lz4.cdata
	66	$ git commit -m"Add a new cdata file for compatibility checks"
61	67
62	68	Repeat this for every codec shipped with Blosc (blosclz, lz4, lz4hc, snappy,
63	69	zlib and zstd).

65	71	Tagging
66	72	-------
67	73
68		- Create a tag ``X.Y.Z`` from ``master``. Use the next message::
	74	- Create a tag ``X.Y.Z`` from ``master``::
69	75
	76	$ git switch master
70	77	$ git tag -a vX.Y.Z -m "Tagging version X.Y.Z"
71	78
72	79	- Push the previous commits and tag to the github repo::

+14

-27

bench/bench.c less more

170	170
171	171
172	172	void do_bench(char compressor, char shuffle, int nthreads, int size, int elsize,
173		int rshift, int unsafe, FILE * ofile) {
	173	int rshift, FILE * ofile) {
174	174	void src, srccpy;
175	175	void dest[NCHUNKS], dest2;
176	176	int nbytes = 0, cbytes = 0;

214	214	if (retcode) abort();
215	215	}
216	216
217		fprintf(ofile, "--> %d, %d, %d, %d, %s, %s, %s\n", nthreads, size, elsize,
218		rshift, compressor, shuffle, unsafe ? "unsafe" : "safe");
	217	fprintf(ofile, "--> %d, %d, %d, %d, %s, %s\n", nthreads, size, elsize,
	218	rshift, compressor, shuffle);
219	219	fprintf(ofile, "******************** Run info ****************************\n");
220	220	fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
221	221	fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift);
222	222	fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize);
223		fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB);
	223	fprintf(ofile, "Working set: %.1f MB\t\t", (size * nchunks) / (float)MB);
224	224	fprintf(ofile, "Number of threads: %d\n", nthreads);
225	225	fprintf(ofile, "******************** Running benchmarks *******************\n");
226	226

282	282	nbytes = size;
283	283	}
284	284	else {
285		nbytes = unsafe ? blosc_decompress_unsafe(dest[j], dest2, size)
286		: blosc_decompress(dest[j], dest2, size);
	285	nbytes = blosc_decompress(dest[j], dest2, size);
287	286	}
288	287	}
289	288	}

377	376	int extreme_suite = 0;
378	377	int debug_suite = 0;
379	378	int nthreads = 4; /* The number of threads */
380		int size = 2MB; / Buffer size */
	379	int size = 4 * MB; /* Buffer size */
381	380	int elsize = 8; /* Datatype size */
382	381	int rshift = 19; /* Significant bits */
383		int workingset = 256MB; / The maximum allocated memory */
	382	int workingset = 256 * MB; /* The maximum allocated memory */
384	383	int nthreads_, size_, elsize_, rshift_, i;
385		int unsafe = 1;
386	384	FILE * output_file = stdout;
387	385	blosc_timestamp_t last, current;
388	386	float totaltime;

393	391	strncpy(usage, "Usage: bench [blosclz \| lz4 \| lz4hc \| snappy \| zlib \| zstd] "
394	392	"[noshuffle \| shuffle \| bitshuffle] "
395	393	"[single \| suite \| hardsuite \| extremesuite \| debugsuite] "
396		"[nthreads] [bufsize(bytes)] [typesize] [sbits] [unsafe \| safe]", 255);
	394	"[nthreads] [bufsize(bytes)] [typesize] [sbits]", 255);
397	395
398	396	if (argc < 2) {
399	397	printf("%s\n", usage);

491	489	rshift = atoi(argv[7]);
492	490	}
493	491
494		if (argc >= 9) {
495		if (strcmp(argv[8], "unsafe") == 0) {
496		unsafe = 1;
497		} else if (strcmp(argv[8], "safe") == 0) {
498		unsafe = 0;
499		} else {
500		printf("%s\n", usage);
501		exit(1);
502		}
503		}
504
505		if ((argc >= 10) \|\| !(single \|\| suite \|\| hard_suite \|\| extreme_suite)) {
	492	if ((argc >= 9) \|\| !(single \|\| suite \|\| hard_suite \|\| extreme_suite)) {
506	493	printf("%s\n", usage);
507	494	exit(1);
508	495	}

514	501
515	502	if (suite) {
516	503	for (nthreads_=1; nthreads_ <= nthreads; nthreads_++) {
517		do_bench(compressor, shuffle, nthreads_, size, elsize, rshift, unsafe, output_file);
	504	do_bench(compressor, shuffle, nthreads_, size, elsize, rshift, output_file);
518	505	}
519	506	}
520	507	else if (hard_suite) {

529	516	nchunks = get_nchunks(size_+i, workingset);
530	517	niter = 1;
531	518	for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) {
532		do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, unsafe, output_file);
	519	do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file);
533	520	blosc_set_timestamp(&current);
534	521	totaltime = (float)getseconds(last, current);
535	522	printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n",

548	535	for (size_ = 32KB; size_ <= size; size_ = 2) {
549	536	nchunks = get_nchunks(size_+i, workingset);
550	537	for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) {
551		do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, unsafe, output_file);
	538	do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file);
552	539	blosc_set_timestamp(&current);
553	540	totaltime = (float)getseconds(last, current);
554	541	printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n",

567	554	for (size_ = size; size_ <= 16MB; size_ = 2) {
568	555	nchunks = get_nchunks(size_+i, workingset);
569	556	for (nthreads_ = nthreads; nthreads_ <= 6; nthreads_++) {
570		do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, unsafe, output_file);
	557	do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file);
571	558	blosc_set_timestamp(&current);
572	559	totaltime = (float)getseconds(last, current);
573	560	printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n",

580	567	}
581	568	/* Single mode */
582	569	else {
583		do_bench(compressor, shuffle, nthreads, size, elsize, rshift, unsafe, output_file);
	570	do_bench(compressor, shuffle, nthreads, size, elsize, rshift, output_file);
584	571	}
585	572
586	573	/* Print out some statistics */

-12

bench/plot-speeds.py less more

28	28	tmp = line.split('-->')[1]
29	29	parts = tmp.split(', ')
30	30	nthreads, size, elsize, sbits, codec, shuffle = parts[:6]
31		safe = 'unsafe'
32		if len(parts) > 6:
33		safe = parts[6]
34	31	nthreads, size, elsize, sbits = map(int, (nthreads, size, elsize, sbits))
35		values["size"] = size * NCHUNKS / MB_
	32	values["size"] = size / MB_
36	33	values["elsize"] = elsize
37	34	values["sbits"] = sbits
38	35	values["codec"] = codec
39	36	values["shuffle"] = shuffle
40		values["safe"] = safe
41	37	# New run for nthreads
42	38	(ratios, speedsw, speedsr) = ([], [], [])
43	39	# Add a new entry for (ratios, speedw, speedr)

46	42	elif line.startswith('memcpy(write):'):
47	43	tmp = line.split(',')[1]
48	44	memcpyw = float(tmp.split(' ')[1])
49		values["memcpyw"].append(memcpyw)
	45	values["memcpyw"].append(memcpyw / 1024)
50	46	elif line.startswith('memcpy(read):'):
51	47	tmp = line.split(',')[1]
52	48	memcpyr = float(tmp.split(' ')[1])
53		values["memcpyr"].append(memcpyr)
	49	values["memcpyr"].append(memcpyr / 1024)
54	50	elif line.startswith('comp(write):'):
55	51	tmp = line.split(',')[1]
56	52	speedw = float(tmp.split(' ')[1])
57	53	ratio = float(line.split(':')[-1])
58		speedsw.append(speedw)
	54	speedsw.append(speedw / 1024)
59	55	ratios.append(ratio)
60	56	elif line.startswith('decomp(read):'):
61	57	tmp = line.split(',')[1]
62	58	speedr = float(tmp.split(' ')[1])
63		speedsr.append(speedr)
	59	speedsr.append(speedr / 1024)
64	60	if "OK" not in line:
65	61	print("WARNING! OK not found in decomp line!")
66	62

70	66
71	67	def show_plot(plots, yaxis, legends, gtitle, xmax=None, ymax=None):
72	68	xlabel('Compresssion ratio')
73		ylabel('Speed (MB/s)')
	69	ylabel('Speed (GB/s)')
74	70	title(gtitle)
75	71	xlim(0, xmax)
76	72	ylim(0, ymax)

189	185	if options.title:
190	186	plot_title = options.title
191	187	else:
192		plot_title += " (%(size).1f MB, %(elsize)d bytes, %(sbits)d bits), %(codec)s %(shuffle)s %(safe)s" % values
	188	plot_title += " (%(size).1f MB, %(elsize)d bytes, %(sbits)d bits), %(codec)s %(shuffle)s" % values
193	189
194	190	gtitle = plot_title
195	191

218	214	mean = np.mean(values["memcpyr"])
219	215	message = "memcpy (read from memory)"
220	216	plot_ = axhline(mean, linewidth=3, linestyle='-.', color='black')
221		text(4.0, mean+400, message)
	217	text(4.0, mean+.4, message)
222	218	plots.append(plot_)
223	219	show_plot(plots, yaxis, legends, gtitle,
224	220	xmax=int(options.xmax) if options.xmax else None,

-1

blosc/CMakeLists.txt less more

40	40	if (ZSTD_FOUND)
41	41	set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIR})
42	42	else (ZSTD_FOUND)
43		set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.4.4)
	43	set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.4.5)
44	44	set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_LOCAL_DIR} ${ZSTD_LOCAL_DIR}/common)
45	45	endif (ZSTD_FOUND)
46	46	endif (NOT DEACTIVATE_ZSTD)

85	85	else(LZ4_FOUND)
86	86	file(GLOB LZ4_FILES ${LZ4_LOCAL_DIR}/*.c)
87	87	set(SOURCES ${SOURCES} ${LZ4_FILES})
	88	source_group("LZ4" FILES ${LZ4_FILES})
88	89	endif(LZ4_FOUND)
89	90	endif(NOT DEACTIVATE_LZ4)
90	91

94	95	else(SNAPPY_FOUND)
95	96	file(GLOB SNAPPY_FILES ${SNAPPY_LOCAL_DIR}/*.cc)
96	97	set(SOURCES ${SOURCES} ${SNAPPY_FILES})
	98	source_group("Snappy" FILES ${SNAPPY_FILES})
97	99	endif(SNAPPY_FOUND)
98	100	endif(NOT DEACTIVATE_SNAPPY)
99	101

103	105	else(ZLIB_FOUND)
104	106	file(GLOB ZLIB_FILES ${ZLIB_LOCAL_DIR}/*.c)
105	107	set(SOURCES ${SOURCES} ${ZLIB_FILES})
	108	source_group("Zlib" FILES ${ZLIB_FILES})
106	109	endif(ZLIB_FOUND)
107	110	endif(NOT DEACTIVATE_ZLIB)
108	111

115	118	${ZSTD_LOCAL_DIR}/compress/*.c
116	119	${ZSTD_LOCAL_DIR}/decompress/*.c)
117	120	set(SOURCES ${SOURCES} ${ZSTD_FILES})
	121	source_group("Zstd" FILES ${ZSTD_FILES})
118	122	endif (ZSTD_FOUND)
119	123	endif (NOT DEACTIVATE_ZSTD)
120	124

-1

blosc/blosc-common.h less more

11	11	#include "blosc-export.h"
12	12	#include <string.h>
13	13
	14	#ifdef __GNUC__
	15	#define BLOSC_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
	16	#endif // __GNUC__
	17
14	18	/* Import standard integer type definitions */
15	19	#if defined(_WIN32) && !defined(__MINGW32__)
16
17	20	/* stdint.h only available in VS2010 (VC++ 16.0) and newer */
18	21	#if defined(_MSC_VER) && _MSC_VER < 1600
19	22	#include "win32/stdint-windows.h"

+64

-88

blosc/blosc.c less more

422	422	char* output, size_t maxout, int clevel)
423	423	{
424	424	int cbytes;
425		if (input_length > (size_t)(2<<30))
426		return -1; /* input larger than 1 GB is not supported */
	425	if (input_length > (size_t)(UINT32_C(2)<<30))
	426	return -1; /* input larger than 2 GB is not supported */
427	427	/* clevel for lz4hc goes up to 12, at least in LZ4 1.7.5
428		* but levels larger than 9 does not buy much compression. */
	428	* but levels larger than 9 do not buy much compression. */
429	429	cbytes = LZ4_compress_HC(input, output, (int)input_length, (int)maxout,
430	430	clevel);
431	431	return cbytes;

521	521	}
522	522	#endif /* HAVE_ZSTD */
523	523
524		static int initialize_decompress_func(struct blosc_context* context,
525		int unsafe) {
	524	static int initialize_decompress_func(struct blosc_context* context) {
526	525	int8_t header_flags = *(context->header_flags);
527	526	int32_t compformat = (header_flags & 0xe0) >> 5;
528	527	int compversion = context->compversion;

531	530	if (compversion != BLOSC_BLOSCLZ_VERSION_FORMAT) {
532	531	return -9;
533	532	}
534		context->decompress_func =
535		unsafe ? &blosclz_decompress_unsafe : &blosclz_decompress;
	533	context->decompress_func = &blosclz_decompress;
536	534	return 0;
537	535	}
538	536	#if defined(HAVE_LZ4)

651	649	}
652	650	}
653	651	if (context->compcode == BLOSC_BLOSCLZ) {
654		int doshuffle = (*(context->header_flags) & BLOSC_DOSHUFFLE) && (typesize > 1);
655	652	cbytes = blosclz_compress(context->clevel, _tmp+j*neblock, neblock,
656		dest, maxout, doshuffle);
	653	dest, maxout);
657	654	}
658	655	#if defined(HAVE_LZ4)
659	656	else if (context->compcode == BLOSC_LZ4) {

711	708	if ((ntbytes+neblock) > maxbytes) {
712	709	return 0; /* Non-compressible data */
713	710	}
714		blosc_internal_fastcopy(dest, _tmp + j * neblock, neblock);
	711	fastcopy(dest, _tmp + j * neblock, neblock);
715	712	cbytes = neblock;
716	713	}
717	714	_sw32(dest - 4, cbytes);

775	772	src = base_src + src_offset;
776	773	/* Uncompress */
777	774	if (cbytes == neblock) {
778		blosc_internal_fastcopy(_tmp, src, neblock);
	775	fastcopy(_tmp, src, neblock);
779	776	nbytes = neblock;
780	777	}
781	778	else {

829	826	if (context->compress) {
830	827	if (*(context->header_flags) & BLOSC_MEMCPYED) {
831	828	/* We want to memcpy only */
832		blosc_internal_fastcopy(context->dest + BLOSC_MAX_OVERHEAD + j * context->blocksize,
833		context->src + j * context->blocksize, bsize);
	829	fastcopy(context->dest + BLOSC_MAX_OVERHEAD + j * context->blocksize,
	830	context->src + j * context->blocksize, bsize);
834	831	cbytes = bsize;
835	832	}
836	833	else {

847	844	else {
848	845	if (*(context->header_flags) & BLOSC_MEMCPYED) {
849	846	/* We want to memcpy only */
850		blosc_internal_fastcopy(context->dest + j * context->blocksize,
851		context->src + BLOSC_MAX_OVERHEAD + j * context->blocksize, bsize);
	847	fastcopy(context->dest + j * context->blocksize,
	848	context->src + BLOSC_MAX_OVERHEAD + j * context->blocksize, bsize);
852	849	cbytes = bsize;
853	850	}
854	851	else {

879	876	(void)rc; // just to avoid 'unused-variable' warning
880	877
881	878	/* Check whether we need to restart threads */
882		blosc_set_nthreads_(context);
	879	if (blosc_set_nthreads_(context) < 0) {
	880	return -1;
	881	}
883	882
884	883	/* Set sentinels */
885	884	context->thread_giveup_code = 1;

1033	1032
1034	1033	/* Enlarge the blocksize for splittable codecs */
1035	1034	if (clevel > 0 && split_block(context->compcode, typesize, blocksize)) {
1036		if (blocksize > (1 << 16)) {
1037		/* Do not use a too large split buffer (> 64 KB) for splitting codecs */
1038		blocksize = (1 << 16);
	1035	if (blocksize > (1 << 18)) {
	1036	/* Do not use a too large split buffer (> 256 KB) for splitting codecs */
	1037	blocksize = (1 << 18);
1039	1038	}
1040	1039	blocksize *= typesize;
1041	1040	if (blocksize < (1 << 16)) {
1042	1041	/* Do not use a too small blocksize (< 64 KB) when typesize is small */
1043	1042	blocksize = (1 << 16);
1044	1043	}
	1044	if (blocksize > 1024 * 1024) {
	1045	/* But do not exceed 1 MB per thread (having this capacity in L3 is normal in modern CPUs) */
	1046	blocksize = 1024 * 1024;
	1047	}
	1048
1045	1049	}
1046	1050
1047	1051	/* Check that blocksize is not too large */

1069	1073	int32_t blocksize,
1070	1074	int32_t numthreads)
1071	1075	{
	1076	char *envvar = NULL;
	1077	int warnlvl = 0;
1072	1078	/* Set parameters */
1073	1079	context->compress = 1;
1074	1080	context->src = (const uint8_t*)src;

1082	1088	context->end_threads = 0;
1083	1089	context->clevel = clevel;
1084	1090
	1091	envvar = getenv("BLOSC_WARN");
	1092	if (envvar != NULL) {
	1093	warnlvl = strtol(envvar, NULL, 10);
	1094	}
	1095
1085	1096	/* Check buffer size limits */
1086	1097	if (sourcesize > BLOSC_MAX_BUFFERSIZE) {
1087		fprintf(stderr, "Input buffer size cannot exceed %d bytes\n",
1088		BLOSC_MAX_BUFFERSIZE);
1089		return -1;
	1098	if (warnlvl > 0) {
	1099	fprintf(stderr, "Input buffer size cannot exceed %d bytes\n",
	1100	BLOSC_MAX_BUFFERSIZE);
	1101	}
	1102	return 0;
1090	1103	}
1091	1104	if (destsize < BLOSC_MAX_OVERHEAD) {
1092		fprintf(stderr, "Output buffer size should be larger than %d bytes\n",
1093		BLOSC_MAX_OVERHEAD);
1094		return -1;
	1105	if (warnlvl > 0) {
	1106	fprintf(stderr, "Output buffer size should be larger than %d bytes\n",
	1107	BLOSC_MAX_OVERHEAD);
	1108	}
	1109	return 0;
1095	1110	}
1096	1111
1097	1112	/* Compression level */
1098	1113	if (clevel < 0 \|\| clevel > 9) {
1099		/* If clevel not in 0..9, print an error */
1100	1114	fprintf(stderr, "`clevel` parameter must be between 0 and 9!\n");
1101	1115	return -10;
1102	1116	}

1272	1286	nbytes, src, dest, destsize,
1273	1287	blosc_compname_to_compcode(compressor),
1274	1288	blocksize, numinternalthreads);
1275		if (error < 0) { return error; }
	1289	if (error <= 0) { return error; }
1276	1290
1277	1291	error = write_compression_header(&context, clevel, doshuffle);
1278		if (error < 0) { return error; }
	1292	if (error <= 0) { return error; }
1279	1293
1280	1294	result = blosc_compress_context(&context);
1281	1295

1394	1408	typesize, nbytes, src, dest, destsize,
1395	1409	g_compressor, g_force_blocksize,
1396	1410	g_threads);
1397		if (result < 0) { break; }
	1411	if (result <= 0) { break; }
1398	1412
1399	1413	result = write_compression_header(g_global_context, clevel, doshuffle);
1400		if (result < 0) { break; }
	1414	if (result <= 0) { break; }
1401	1415
1402	1416	result = blosc_compress_context(g_global_context);
1403	1417	} while (0);

1411	1425	const void* src,
1412	1426	void* dest,
1413	1427	size_t destsize,
1414		int numinternalthreads,
1415		int unsafe)
	1428	int numinternalthreads)
1416	1429	{
1417	1430	uint8_t version;
1418	1431	int32_t ntbytes;

1474	1487	return -1;
1475	1488	}
1476	1489	} else {
1477		ntbytes = initialize_decompress_func(context, unsafe);
	1490	ntbytes = initialize_decompress_func(context);
1478	1491	if (ntbytes != 0) return ntbytes;
1479	1492
1480	1493	/* Validate that compressed size is large enough to hold the bstarts array */

1493	1506	return ntbytes;
1494	1507	}
1495	1508
1496		/* Implementation of blosc_decompress_ctx{,_unsafe}. */
1497		static int blosc_decompress_ctx_impl(const void* src, void* dest,
1498		size_t destsize, int numinternalthreads,
1499		int unsafe) {
	1509	int blosc_decompress_ctx(const void* src, void* dest, size_t destsize,
	1510	int numinternalthreads) {
1500	1511	int result;
1501	1512	struct blosc_context context;
1502	1513
1503	1514	context.threads_started = 0;
1504	1515	result = blosc_run_decompression_with_context(&context, src, dest, destsize,
1505		numinternalthreads, unsafe);
	1516	numinternalthreads);
1506	1517
1507	1518	if (numinternalthreads > 1)
1508	1519	{

1512	1523	return result;
1513	1524	}
1514	1525
1515		int blosc_decompress_ctx(const void* src, void* dest, size_t destsize,
1516		int numinternalthreads) {
1517		return blosc_decompress_ctx_impl(src, dest, destsize, numinternalthreads,
1518		/unsafe=/0);
1519		}
1520
1521		int blosc_decompress_ctx_unsafe(const void* src, void* dest, size_t destsize,
1522		int numinternalthreads) {
1523		return blosc_decompress_ctx_impl(src, dest, destsize, numinternalthreads,
1524		/unsafe=/1);
1525		}
1526
1527		/* Implementation of blosc_decompress{,_unsafe}. */
1528		static int blosc_decompress_impl(const void* src, void* dest,
1529		size_t destsize, int unsafe) {
	1526	int blosc_decompress(const void* src, void* dest, size_t destsize) {
1530	1527	int result;
1531	1528	char* envvar;
1532	1529	long nthreads;

1556	1553	pthread_mutex_lock(global_comp_mutex);
1557	1554
1558	1555	result = blosc_run_decompression_with_context(g_global_context, src, dest,
1559		destsize, g_threads, unsafe);
	1556	destsize, g_threads);
1560	1557
1561	1558	pthread_mutex_unlock(global_comp_mutex);
1562	1559
1563	1560	return result;
1564	1561	}
1565	1562
1566		/* The public routine for decompression. See blosc.h for docstrings. */
1567		int blosc_decompress(const void src, void dest, size_t destsize) {
1568		return blosc_decompress_impl(src, dest, destsize, /unsafe=/0);
1569		}
1570
1571		int blosc_decompress_unsafe(const void src, void dest, size_t destsize) {
1572		return blosc_decompress_impl(src, dest, destsize, /unsafe=/1);
1573		}
1574
1575
1576		/* Implementation of blosc_getitem{,_unsafe}. */
1577		static int blosc_getitem_impl(const void* src, int start, int nitems,
1578		void* dest, int unsafe) {
	1563	int blosc_getitem(const void* src, int start, int nitems, void* dest) {
1579	1564	uint8_t _src=NULL; / current pos for source buffer */
1580	1565	uint8_t version, compversion; /* versions for compressed header */
1581	1566	uint8_t flags; /* flags for header */

1628	1613	return -1;
1629	1614	}
1630	1615	} else {
1631		ntbytes = initialize_decompress_func(&context, /unsafe=/unsafe);
	1616	ntbytes = initialize_decompress_func(&context);
1632	1617	if (ntbytes != 0) return ntbytes;
1633	1618
1634	1619	if (nblocks >= (compressedsize - 16) / 4) {

1681	1666	/* Do the actual data copy */
1682	1667	if (flags & BLOSC_MEMCPYED) {
1683	1668	/* We want to memcpy only */
1684		blosc_internal_fastcopy((uint8_t *) dest + ntbytes,
1685		(uint8_t ) src + BLOSC_MAX_OVERHEAD + j blocksize + startb, bsize2);
	1669	fastcopy((uint8_t *) dest + ntbytes,
	1670	(uint8_t ) src + BLOSC_MAX_OVERHEAD + j blocksize + startb, bsize2);
1686	1671	cbytes = bsize2;
1687	1672	}
1688	1673	else {

1695	1680	break;
1696	1681	}
1697	1682	/* Copy to destination */
1698		blosc_internal_fastcopy((uint8_t *) dest + ntbytes, tmp2 + startb, bsize2);
	1683	fastcopy((uint8_t *) dest + ntbytes, tmp2 + startb, bsize2);
1699	1684	cbytes = bsize2;
1700	1685	}
1701	1686	ntbytes += cbytes;

1704	1689	my_free(tmp);
1705	1690
1706	1691	return ntbytes;
1707		}
1708
1709		/* Specific routine optimized for decompression a small number of
1710		items out of a compressed chunk. This does not use threads because
1711		it would affect negatively to performance. */
1712		int blosc_getitem(const void src, int start, int nitems, void dest) {
1713		return blosc_getitem_impl(src, start, nitems, dest, /unsafe=/0);
1714		}
1715
1716		int blosc_getitem_unsafe(const void src, int start, int nitems, void dest) {
1717		return blosc_getitem_impl(src, start, nitems, dest, /unsafe=/1);
1718	1692	}
1719	1693
1720	1694	/* Decompress & unshuffle several blocks in a single thread */

1816	1790	if (compress) {
1817	1791	if (flags & BLOSC_MEMCPYED) {
1818	1792	/* We want to memcpy only */
1819		blosc_internal_fastcopy(dest + BLOSC_MAX_OVERHEAD + nblock_ * blocksize, src + nblock_ * blocksize,
1820		bsize);
	1793	fastcopy(dest + BLOSC_MAX_OVERHEAD + nblock_ * blocksize,
	1794	src + nblock_ * blocksize, bsize);
1821	1795	cbytes = bsize;
1822	1796	}
1823	1797	else {

1829	1803	else {
1830	1804	if (flags & BLOSC_MEMCPYED) {
1831	1805	/* We want to memcpy only */
1832		blosc_internal_fastcopy(dest + nblock_ * blocksize, src + BLOSC_MAX_OVERHEAD + nblock_ * blocksize,
1833		bsize);
	1806	fastcopy(dest + nblock_ * blocksize,
	1807	src + BLOSC_MAX_OVERHEAD + nblock_ * blocksize, bsize);
1834	1808	cbytes = bsize;
1835	1809	}
1836	1810	else {

1872	1846	/* End of critical section */
1873	1847
1874	1848	/* Copy the compressed buffer to destination */
1875		blosc_internal_fastcopy(dest + ntdest, tmp2, cbytes);
	1849	fastcopy(dest + ntdest, tmp2, cbytes);
1876	1850	}
1877	1851	else {
1878	1852	nblock_++;

2003	1977	/* Launch a new pool of threads */
2004	1978	if (context->numthreads > 1 && context->numthreads != context->threads_started) {
2005	1979	blosc_release_threadpool(context);
2006		init_threads(context);
	1980	if (init_threads(context) < 0) {
	1981	return -1;
	1982	}
2007	1983	}
2008	1984
2009	1985	/* We have now started the threads */

+10

-40

blosc/blosc.h less more

17	17
18	18	/* Version numbers */
19	19	#define BLOSC_VERSION_MAJOR 1 /* for major interface/format changes */
20		#define BLOSC_VERSION_MINOR 17 /* for minor interface/format changes */
	20	#define BLOSC_VERSION_MINOR 20 /* for minor interface/format changes */
21	21	#define BLOSC_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
22	22
23		#define BLOSC_VERSION_STRING "1.17.1" /* string version. Sync with above! */
	23	#define BLOSC_VERSION_STRING "1.20.1" /* string version. Sync with above! */
24	24	#define BLOSC_VERSION_REVISION "$Rev$" /* revision version */
25		#define BLOSC_VERSION_DATE "$Date:: 2019-12-12 #$" /* date version */
26
27		#define BLOSCLZ_VERSION_STRING "2.0.0" /* the internal compressor version */
	25	#define BLOSC_VERSION_DATE "$Date:: 2020-09-08 #$" /* date version */
	26
	27	#define BLOSCLZ_VERSION_STRING "2.3.0" /* the internal compressor version */
28	28
29	29	/* The _FORMAT symbols should be just 1-byte long /
30	30	#define BLOSC_VERSION_FORMAT 2 /* Blosc format version, starting at 1 */

214	214	This will call blosc_set_splitmode() with the different supported values.
215	215	See blosc_set_splitmode() docstrings for more info on each mode.
216	216
	217	BLOSC_WARN=(INTEGER): This will print some warning message on stderr
	218	showing more info in situations where data inputs cannot be compressed.
	219	The values can range from 1 (less verbose) to 10 (full verbose). 0 is
	220	the same as if the BLOSC_WARN envvar was not defined.
217	221	*/
218	222	BLOSC_EXPORT int blosc_compress(int clevel, int doshuffle, size_t typesize,
219	223	size_t nbytes, const void src, void dest,

277	281	BLOSC_EXPORT int blosc_decompress(const void src, void dest, size_t destsize);
278	282
279	283	/**
280		Same as `blosc_decompress`, except that this is not safe to run on
281		untrusted/possibly corrupted input (even after calling
282		`blosc_cbuffer_validate`).
283
284		This may be marginally faster than `blosc_decompress` due to skipping certain
285		bounds checking and validation.
286		*/
287		BLOSC_EXPORT int blosc_decompress_unsafe(const void* src, void* dest,
288		size_t destsize);
289
290		/**
291	284	Context interface to blosc decompression. This does not require a
292	285	call to blosc_init() and can be called from multithreaded
293	286	applications without the global lock being used, so allowing Blosc

310	303	size_t destsize, int numinternalthreads);
311	304
312	305	/**
313		Same as `blosc_decompress_ctx`, except that this is not safe to run on
314		untrusted/possibly corrupted input (even after calling
315		`blosc_cbuffer_validate`).
316
317		This may be marginally faster than `blosc_decompress_ctx` due to skipping
318		certain bounds checking and validation.
319		*/
320		BLOSC_EXPORT int blosc_decompress_ctx_unsafe(const void* src, void* dest,
321		size_t destsize,
322		int numinternalthreads);
323
324		/**
325	306	Get `nitems` (of typesize size) in `src` buffer starting in `start`.
326	307	The items are returned in `dest` buffer, which has to have enough
327	308	space for storing all items.

330	311	some error happens.
331	312	*/
332	313	BLOSC_EXPORT int blosc_getitem(const void src, int start, int nitems, void dest);
333
334		/**
335		Same as `blosc_getitem`, except that this is not safe to run on
336		untrusted/possibly corrupted input (even after calling
337		`blosc_cbuffer_validate`).
338
339		This may be marginally faster than `blosc_getitem` due to skipping certain
340		bounds checking and validation.
341		*/
342		BLOSC_EXPORT int blosc_getitem_unsafe(const void* src, int start, int nitems,
343		void* dest);
344	314
345	315	/**
346	316	Returns the current number of threads that are used for

368	338
369	339	/**
370	340	Select the compressor to be used. The supported ones are "blosclz",
371		"lz4", "lz4hc", "snappy", "zlib" and "ztsd". If this function is not
	341	"lz4", "lz4hc", "snappy", "zlib" and "zstd". If this function is not
372	342	called, then "blosclz" will be used by default.
373	343
374	344	In case the compressor is not recognized, or there is not support

+593

-312

blosc/blosclz.c less more

3	3	Author: Francesc Alted <francesc@blosc.org>
4	4	Creation date: 2009-05-20
5	5
6		See LICENSES/BLOSC.txt for details about copyright and rights to use.
	6	See LICENSE.txt for details about copyright and rights to use.
7	7	**********************************************************************/
8	8
9	9	/*********************************************************************

13	13
14	14
15	15	#include <stdio.h>
16		#include <stdlib.h>
17
18		#if defined(_WIN32) && !defined(__MINGW32__)
19		#include <windows.h>
20		/* stdint.h only available in VS2010 (VC++ 16.0) and newer */
21		#if defined(_MSC_VER) && _MSC_VER < 1600
22		#include "win32/stdint-windows.h"
23		#else
24		#include <stdint.h>
25		#endif
26		#else
27		#include <stdint.h>
28		#endif /* _WIN32 */
29
	16	#include <stdbool.h>
30	17	#include "blosclz.h"
31	18	#include "fastcopy.h"
32	19	#include "blosc-common.h"
33		#include "blosc-comp-features.h"
34	20
35	21
36	22	/*
37	23	* Give hints to the compiler for branch prediction optimization.
38	24	*/
39	25	#if defined(__GNUC__) && (__GNUC__ > 2)
40		#define BLOSCLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1))
41		#define BLOSCLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0))
	26	#define BLOSCLZ_LIKELY(c) (__builtin_expect((c), 1))
	27	#define BLOSCLZ_UNLIKELY(c) (__builtin_expect((c), 0))
42	28	#else
43		#define BLOSCLZ_EXPECT_CONDITIONAL(c) (c)
44		#define BLOSCLZ_UNEXPECT_CONDITIONAL(c) (c)
	29	#define BLOSCLZ_LIKELY(c) (c)
	30	#define BLOSCLZ_UNLIKELY(c) (c)
45	31	#endif
46	32
47	33	/*

56	42	#define MAX_FARDISTANCE (65535 + MAX_DISTANCE - 1)
57	43
58	44	#ifdef BLOSC_STRICT_ALIGN
59		#define BLOSCLZ_READU16(p) ((p)[0] \| (p)[1]<<8)
	45	#define BLOSCLZ_READU16(p) ((p)[0] \| (p)[1]<<8)
60	46	#define BLOSCLZ_READU32(p) ((p)[0] \| (p)[1]<<8 \| (p)[2]<<16 \| (p)[3]<<24)
61	47	#else
62		#define BLOSCLZ_READU16(p) ((const uint16_t)(p))
63		#define BLOSCLZ_READU32(p) ((const uint32_t)(p))
64		#endif
65
66		#define HASH_LOG (14U)
67
68		/* Simple, but pretty effective hash function for 3-byte sequence */
69		// This is the original hash function used in fastlz
70		//#define HASH_FUNCTION(v, p, h) { \
71		// v = BLOSCLZ_READU16(p); \
72		// v ^= BLOSCLZ_READU16(p + 1) ^ ( v >> (16 - h)); \
73		// v &= (1 << h) - 1; \
74		//}
	48	#define BLOSCLZ_READU16(p) ((const uint16_t)(p))
	49	#define BLOSCLZ_READU32(p) ((const uint32_t)(p))
	50	#endif
	51
	52	#define HASH_LOG (12U)
75	53
76	54	// This is used in LZ4 and seems to work pretty well here too
77		#define HASH_FUNCTION(v, p, h) \
78		v = ((BLOSCLZ_READU32(p) * 2654435761U) >> (32U - h))
79
80
81		#define LITERAL(ip, op, op_limit, anchor, copy) { \
82		if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + 2 > op_limit)) \
83		goto out; \
84		op++ = anchor++; \
85		ip = anchor; \
86		copy++; \
87		if (BLOSCLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY)) { \
88		copy = 0; \
89		*op++ = MAX_COPY-1; \
90		nmax_copies++; \
91		if (nmax_copies > max_nmax_copies) \
92		goto out; \
93		} \
94		continue; \
95		}
96
97		#define IP_BOUNDARY 2
98
	55	#define HASH_FUNCTION(v, s, h) { \
	56	v = (s * 2654435761U) >> (32U - h); \
	57	}
	58
	59
	60	#if defined(__AVX2__)
	61	static uint8_t get_run_32(uint8_t ip, const uint8_t ip_bound, const uint8_t ref) {
	62	uint8_t x = ip[-1];
	63	/* safe because the outer check against ip limit */
	64	if (ip < (ip_bound - sizeof(int64_t))) {
	65	int64_t value, value2;
	66	/* Broadcast the value for every byte in a 64-bit register */
	67	memset(&value, x, 8);
	68	#if defined(BLOSC_STRICT_ALIGN)
	69	memcpy(&value2, ref, 8);
	70	#else
	71	value2 = ((int64_t*)ref)[0];
	72	#endif
	73	if (value != value2) {
	74	/* Return the byte that starts to differ */
	75	while (*ref++ == x) ip++;
	76	return ip;
	77	}
	78	else {
	79	ip += 8;
	80	ref += 8;
	81	}
	82	}
	83	if (ip < (ip_bound - sizeof(__m128i))) {
	84	__m128i value, value2, cmp;
	85	/* Broadcast the value for every byte in a 128-bit register */
	86	memset(&value, x, sizeof(__m128i));
	87	value2 = _mm_loadu_si128((__m128i *) ref);
	88	cmp = _mm_cmpeq_epi32(value, value2);
	89	if (_mm_movemask_epi8(cmp) != 0xFFFF) {
	90	/* Return the byte that starts to differ */
	91	while (*ref++ == x) ip++;
	92	return ip;
	93	} else {
	94	ip += sizeof(__m128i);
	95	ref += sizeof(__m128i);
	96	}
	97	}
	98	while (ip < (ip_bound - (sizeof(__m256i)))) {
	99	__m256i value, value2, cmp;
	100	/* Broadcast the value for every byte in a 256-bit register */
	101	memset(&value, x, sizeof(__m256i));
	102	value2 = _mm256_loadu_si256((__m256i *)ref);
	103	cmp = _mm256_cmpeq_epi64(value, value2);
	104	if ((unsigned)_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) {
	105	/* Return the byte that starts to differ */
	106	while (*ref++ == x) ip++;
	107	return ip;
	108	}
	109	else {
	110	ip += sizeof(__m256i);
	111	ref += sizeof(__m256i);
	112	}
	113	}
	114	/* Look into the remainder */
	115	while ((ip < ip_bound) && (*ref++ == x)) ip++;
	116	return ip;
	117	}
	118
	119	#elif defined(__SSE2__)
	120
	121	static uint8_t get_run_16(uint8_t ip, const uint8_t ip_bound, const uint8_t ref) {
	122	uint8_t x = ip[-1];
	123
	124	if (ip < (ip_bound - sizeof(int64_t))) {
	125	int64_t value, value2;
	126	/* Broadcast the value for every byte in a 64-bit register */
	127	memset(&value, x, 8);
	128	#if defined(BLOSC_STRICT_ALIGN)
	129	memcpy(&value2, ref, 8);
	130	#else
	131	value2 = ((int64_t*)ref)[0];
	132	#endif
	133	if (value != value2) {
	134	/* Return the byte that starts to differ */
	135	while (*ref++ == x) ip++;
	136	return ip;
	137	}
	138	else {
	139	ip += 8;
	140	ref += 8;
	141	}
	142	}
	143	/* safe because the outer check against ip limit */
	144	while (ip < (ip_bound - sizeof(__m128i))) {
	145	__m128i value, value2, cmp;
	146	/* Broadcast the value for every byte in a 128-bit register */
	147	memset(&value, x, sizeof(__m128i));
	148	value2 = _mm_loadu_si128((__m128i *)ref);
	149	cmp = _mm_cmpeq_epi32(value, value2);
	150	if (_mm_movemask_epi8(cmp) != 0xFFFF) {
	151	/* Return the byte that starts to differ */
	152	while (*ref++ == x) ip++;
	153	return ip;
	154	}
	155	else {
	156	ip += sizeof(__m128i);
	157	ref += sizeof(__m128i);
	158	}
	159	}
	160	/* Look into the remainder */
	161	while ((ip < ip_bound) && (*ref++ == x)) ip++;
	162	return ip;
	163	}
	164
	165	#else
99	166
100	167	static uint8_t get_run(uint8_t ip, const uint8_t ip_bound, const uint8_t ref) {
101	168	uint8_t x = ip[-1];

124	191	return ip;
125	192	}
126	193
127		#ifdef __SSE2__
128		static uint8_t get_run_16(uint8_t ip, const uint8_t ip_bound, const uint8_t ref) {
129		uint8_t x = ip[-1];
130
131		if (ip < (ip_bound - sizeof(int64_t))) {
132		int64_t value, value2;
133		/* Broadcast the value for every byte in a 64-bit register */
134		memset(&value, x, 8);
135		#if defined(BLOSC_STRICT_ALIGN)
136		memcpy(&value2, ref, 8);
137		#else
138		value2 = ((int64_t*)ref)[0];
139		#endif
140		if (value != value2) {
141		/* Return the byte that starts to differ */
142		while (*ref++ == x) ip++;
143		return ip;
144		}
145		else {
146		ip += 8;
147		ref += 8;
148		}
149		}
150		/* safe because the outer check against ip limit */
151		while (ip < (ip_bound - sizeof(__m128i))) {
152		__m128i value, value2, cmp;
153		/* Broadcast the value for every byte in a 128-bit register */
154		memset(&value, x, sizeof(__m128i));
155		value2 = _mm_loadu_si128((__m128i *)ref);
156		cmp = _mm_cmpeq_epi32(value, value2);
157		if (_mm_movemask_epi8(cmp) != 0xFFFF) {
158		/* Return the byte that starts to differ */
159		while (*ref++ == x) ip++;
160		return ip;
161		}
162		else {
163		ip += sizeof(__m128i);
164		ref += sizeof(__m128i);
165		}
166		}
167		/* Look into the remainder */
168		while ((ip < ip_bound) && (*ref++ == x)) ip++;
169		return ip;
170		}
171		#endif
172
173
174		#ifdef __AVX2__
175		static uint8_t get_run_32(uint8_t ip, const uint8_t ip_bound, const uint8_t ref) {
176		uint8_t x = ip[-1];
177		/* safe because the outer check against ip limit */
178		if (ip < (ip_bound - sizeof(int64_t))) {
179		int64_t value, value2;
180		/* Broadcast the value for every byte in a 64-bit register */
181		memset(&value, x, 8);
182		#if defined(BLOSC_STRICT_ALIGN)
183		memcpy(&value2, ref, 8);
184		#else
185		value2 = ((int64_t*)ref)[0];
186		#endif
187		if (value != value2) {
188		/* Return the byte that starts to differ */
189		while (*ref++ == x) ip++;
190		return ip;
191		}
192		else {
193		ip += 8;
194		ref += 8;
195		}
196		}
197		if (ip < (ip_bound - sizeof(__m128i))) {
198		__m128i value, value2, cmp;
199		/* Broadcast the value for every byte in a 128-bit register */
200		memset(&value, x, sizeof(__m128i));
201		value2 = _mm_loadu_si128((__m128i *) ref);
202		cmp = _mm_cmpeq_epi32(value, value2);
203		if (_mm_movemask_epi8(cmp) != 0xFFFF) {
204		/* Return the byte that starts to differ */
205		while (*ref++ == x) ip++;
206		return ip;
207		} else {
208		ip += sizeof(__m128i);
209		ref += sizeof(__m128i);
210		}
211		}
212		while (ip < (ip_bound - (sizeof(__m256i)))) {
213		__m256i value, value2, cmp;
214		/* Broadcast the value for every byte in a 256-bit register */
215		memset(&value, x, sizeof(__m256i));
216		value2 = _mm256_loadu_si256((__m256i *)ref);
217		cmp = _mm256_cmpeq_epi64(value, value2);
218		if (_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) {
219		/* Return the byte that starts to differ */
220		while (*ref++ == x) ip++;
221		return ip;
222		}
223		else {
224		ip += sizeof(__m256i);
225		ref += sizeof(__m256i);
226		}
227		}
228		/* Look into the remainder */
229		while ((ip < ip_bound) && (*ref++ == x)) ip++;
230		return ip;
231		}
232	194	#endif
233	195
234	196

319	281	value = _mm256_loadu_si256((__m256i *) ip);
320	282	value2 = _mm256_loadu_si256((__m256i *)ref);
321	283	cmp = _mm256_cmpeq_epi64(value, value2);
322		if (_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) {
	284	if ((unsigned)_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) {
323	285	/* Return the byte that starts to differ */
324	286	while (ref++ == ip++) {}
325	287	return ip;

336	298	#endif
337	299
338	300
339		int blosclz_compress(const int opt_level, const void* input, int length,
340		void* output, int maxout, int shuffle) {
341		uint8_t* ip = (uint8_t*)input;
	301	static uint8_t* get_run_or_match(uint8_t* ip, uint8_t* ip_bound, const uint8_t* ref, bool run) {
	302	if (BLOSCLZ_UNLIKELY(run)) {
	303	#if defined(__AVX2__)
	304	ip = get_run_32(ip, ip_bound, ref);
	305	#elif defined(__SSE2__)
	306	ip = get_run_16(ip, ip_bound, ref);
	307	#else
	308	ip = get_run(ip, ip_bound, ref);
	309	#endif
	310	}
	311	else {
	312	#if defined(__AVX2__)
	313	ip = get_match_32(ip, ip_bound, ref);
	314	#elif defined(__SSE2__)
	315	ip = get_match_16(ip, ip_bound, ref);
	316	#else
	317	ip = get_match(ip, ip_bound, ref);
	318	#endif
	319	}
	320
	321	return ip;
	322	}
	323
	324
	325	#define LITERAL(ip, op, op_limit, anchor, copy) { \
	326	if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) \
	327	goto out; \
	328	op++ = anchor++; \
	329	ip = anchor; \
	330	copy++; \
	331	if (BLOSCLZ_UNLIKELY(copy == MAX_COPY)) { \
	332	copy = 0; \
	333	*op++ = MAX_COPY-1; \
	334	} \
	335	}
	336
	337	#define LITERAL2(ip, oc, anchor, copy) { \
	338	oc++; anchor++; \
	339	ip = anchor; \
	340	copy++; \
	341	if (BLOSCLZ_UNLIKELY(copy == MAX_COPY)) { \
	342	copy = 0; \
	343	oc++; \
	344	} \
	345	}
	346
	347	#define DISTANCE_SHORT(op, op_limit, len, distance) { \
	348	if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) \
	349	goto out; \
	350	*op++ = (uint8_t)((len << 5U) + (distance >> 8U)); \
	351	*op++ = (uint8_t)((distance & 255U)); \
	352	}
	353
	354	#define DISTANCE_LONG(op, op_limit, len, distance) { \
	355	if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \
	356	goto out; \
	357	*op++ = (uint8_t)((7U << 5U) + (distance >> 8U)); \
	358	for (len -= 7; len >= 255; len -= 255) { \
	359	if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \
	360	goto out; \
	361	*op++ = 255; \
	362	} \
	363	if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) \
	364	goto out; \
	365	*op++ = (uint8_t)len; \
	366	*op++ = (uint8_t)((distance & 255U)); \
	367	}
	368
	369	#define DISTANCE_SHORT_FAR(op, op_limit, len, distance) { \
	370	if (BLOSCLZ_UNLIKELY(op + 4 > op_limit)) \
	371	goto out; \
	372	*op++ = (uint8_t)((len << 5U) + 31); \
	373	*op++ = 255; \
	374	*op++ = (uint8_t)(distance >> 8U); \
	375	*op++ = (uint8_t)(distance & 255U); \
	376	}
	377
	378	#define DISTANCE_LONG_FAR(op, op_limit, len, distance) { \
	379	if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \
	380	goto out; \
	381	*op++ = (7U << 5U) + 31; \
	382	for (len -= 7; len >= 255; len -= 255) { \
	383	if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \
	384	goto out; \
	385	*op++ = 255; \
	386	} \
	387	if (BLOSCLZ_UNLIKELY(op + 4 > op_limit)) \
	388	goto out; \
	389	*op++ = (uint8_t)len; \
	390	*op++ = 255; \
	391	*op++ = (uint8_t)(distance >> 8U); \
	392	*op++ = (uint8_t)(distance & 255U); \
	393	}
	394
	395
	396	// Get the compressed size of a buffer. Useful for testing compression ratios for high clevels.
	397	static int get_csize(uint8_t* ibase, int maxlen, bool force_3b_shift) {
	398	uint8_t* ip = ibase;
	399	int32_t oc = 0;
	400	uint8_t* ip_bound = ibase + maxlen - 1;
	401	uint8_t* ip_limit = ibase + maxlen - 12;
	402	uint32_t htab[1U << (uint8_t)HASH_LOG];
	403	uint32_t hval;
	404	uint32_t seq;
	405	uint8_t copy;
	406
	407	// Initialize the hash table to distances of 0
	408	for (unsigned i = 0; i < (1U << HASH_LOG); i++) {
	409	htab[i] = 0;
	410	}
	411
	412	/* we start with literal copy */
	413	copy = 4;
	414	oc += 5;
	415
	416	/* main loop */
	417	while (BLOSCLZ_LIKELY(ip < ip_limit)) {
	418	const uint8_t* ref;
	419	unsigned distance;
	420	uint8_t* anchor = ip; /* comparison starting-point */
	421
	422	/* find potential match */
	423	seq = BLOSCLZ_READU32(ip);
	424	HASH_FUNCTION(hval, seq, HASH_LOG)
	425	ref = ibase + htab[hval];
	426
	427	/* calculate distance to the match */
	428	distance = anchor - ref;
	429
	430	/* update hash table */
	431	htab[hval] = (uint32_t) (anchor - ibase);
	432
	433	if (distance == 0 \|\| (distance >= MAX_FARDISTANCE)) {
	434	LITERAL2(ip, oc, anchor, copy)
	435	continue;
	436	}
	437
	438	/* is this a match? check the first 4 bytes */
	439	if (BLOSCLZ_UNLIKELY(BLOSCLZ_READU32(ref) == BLOSCLZ_READU32(ip))) {
	440	ref += 4;
	441	}
	442	else {
	443	/* no luck, copy as a literal */
	444	LITERAL2(ip, oc, anchor, copy)
	445	continue;
	446	}
	447
	448	/* last matched byte */
	449	ip = anchor + 4;
	450
	451	/* distance is biased */
	452	distance--;
	453
	454	/* get runs or matches; zero distance means a run */
	455	ip = get_run_or_match(ip, ip_bound, ref, !distance);
	456
	457	ip -= force_3b_shift ? 3 : 4;
	458	unsigned len = (int)(ip - anchor);
	459	// If match is close, let's reduce the minimum length to encode it
	460	unsigned minlen = (distance < MAX_DISTANCE) ? 3 : 4;
	461	// Encoding short lengths is expensive during decompression
	462	if (len < minlen) {
	463	LITERAL2(ip, oc, anchor, copy)
	464	continue;
	465	}
	466
	467	/* if we have'nt copied anything, adjust the output counter */
	468	if (!copy)
	469	oc--;
	470	/* reset literal counter */
	471	copy = 0;
	472
	473	/* encode the match */
	474	if (distance < MAX_DISTANCE) {
	475	if (len >= 7) {
	476	oc += ((len - 7) / 255) + 1;
	477	}
	478	oc += 2;
	479	}
	480	else {
	481	/* far away, but not yet in the another galaxy... */
	482	if (len >= 7) {
	483	oc += ((len - 7) / 255) + 1;
	484	}
	485	oc += 4;
	486	}
	487
	488	/* update the hash at match boundary */
	489	seq = BLOSCLZ_READU32(ip);
	490	HASH_FUNCTION(hval, seq, HASH_LOG)
	491	htab[hval] = (uint32_t) (ip++ - ibase);
	492	seq >>= 8U;
	493	HASH_FUNCTION(hval, seq, HASH_LOG)
	494	htab[hval] = (uint32_t) (ip++ - ibase);
	495	/* assuming literal copy */
	496	oc++;
	497
	498	}
	499
	500	/* if we have copied something, adjust the copy length */
	501	if (!copy)
	502	oc--;
	503
	504	return (int)oc;
	505	}
	506
	507
	508	int blosclz_compress(const int clevel, const void* input, int length,
	509	void* output, int maxout) {
342	510	uint8_t* ibase = (uint8_t*)input;
343		uint8_t* ip_bound = ip + length - IP_BOUNDARY;
344		uint8_t* ip_limit = ip + length - 12;
	511	uint8_t* ip = ibase;
	512	uint8_t* ip_bound = ibase + length - 1;
	513	uint8_t* ip_limit = ibase + length - 12;
345	514	uint8_t* op = (uint8_t*)output;
346	515	uint8_t* op_limit;
347		uint16_t htab[1U << (uint8_t)HASH_LOG];
348		int32_t hval;
	516	uint32_t htab[1U << (uint8_t)HASH_LOG];
	517	uint32_t hval;
	518	uint32_t seq;
349	519	uint8_t copy;
350		uint32_t nmax_copies = 0;
351		unsigned i;
352		uint8_t hashlog_[10] = {0, HASH_LOG - 4, HASH_LOG - 4, HASH_LOG - 3 , HASH_LOG - 2,
353		HASH_LOG - 1, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG};
354		uint8_t hashlog = hashlog_[opt_level];
355		// The maximum amount of consecutive MAX_COPY copies before giving up
356		// 0 means something very close to RLE
357		uint8_t max_nmax_copies_[10] = {255U, 0U, 8U, 8U, 16U, 32U, 32U, 32U, 32U, 64U}; // 255 never used
358		uint8_t max_nmax_copies = max_nmax_copies_[opt_level];
359		double maxlength_[10] = {-1, .1, .2, .3, .4, .6, .9, .95, 1.0, 1.0};
360		int32_t maxlength = (int32_t)(length * maxlength_[opt_level]);
361
362		if (maxlength > (int32_t)maxout) {
363		maxlength = (int32_t)maxout;
364		}
365		op_limit = op + maxlength;
366
	520
	521	op_limit = op + maxout;
	522
	523	// Minimum lengths for encoding
	524	unsigned minlen_[10] = {0, 12, 12, 11, 10, 9, 8, 7, 6, 5};
	525
	526	// Minimum compression ratios for initiate encoding
	527	double cratio_[10] = {0, 2, 2, 2, 2, 1.8, 1.6, 1.4, 1.2, 1.1};
	528
	529	uint8_t hashlog_[10] = {0, HASH_LOG - 2, HASH_LOG - 1, HASH_LOG, HASH_LOG,
	530	HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG};
	531	uint8_t hashlog = hashlog_[clevel];
367	532	// Initialize the hash table to distances of 0
368		for (i = 0; i < (1U << hashlog); i++) {
	533	for (unsigned i = 0; i < (1U << hashlog); i++) {
369	534	htab[i] = 0;
370	535	}
371	536
372		/* output buffer cannot be less than 66 bytes or we can get into trouble */
373		if (BLOSCLZ_UNEXPECT_CONDITIONAL(maxout < 66 \|\| length < 4)) {
	537	/* input and output buffer cannot be less than 16 and 66 bytes or we can get into trouble */
	538	if (length < 16 \|\| maxout < 66) {
374	539	return 0;
375	540	}
376	541
	542	/* When we go back in a match (shift), we obtain quite different compression properties.
	543	* It looks like 4 is more useful in combination with bitshuffle and small typesizes
	544	* (compress better and faster in e.g. `b2bench blosclz bitshuffle single 6 6291456 1 19`).
	545	* Fallback to 4 because it provides more consistent results on small itemsizes.
	546	*
	547	* In this block we also check cratios for the beginning of the buffers and
	548	* eventually discard those that are small (take too long to decompress).
	549	* This process is called _entropy probing_.
	550	*/
	551	int ipshift = 4;
	552	int maxlen; // maximum length for entropy probing
	553	int csize_3b;
	554	int csize_4b;
	555	double cratio = 0;
	556	switch (clevel) {
	557	case 1:
	558	case 2:
	559	case 3:
	560	maxlen = length / 8;
	561	csize_4b = get_csize(ibase, maxlen, false);
	562	cratio = (double)maxlen / csize_4b;
	563	break;
	564	case 4:
	565	case 5:
	566	case 6:
	567	case 7:
	568	case 8:
	569	maxlen = length / 8;
	570	csize_4b = get_csize(ibase, maxlen, false);
	571	cratio = (double)maxlen / csize_4b;
	572	break;
	573	case 9:
	574	// case 9 is special. we need to asses the optimal shift
	575	maxlen = length / 8;
	576	csize_3b = get_csize(ibase, maxlen, true);
	577	csize_4b = get_csize(ibase, maxlen, false);
	578	ipshift = (csize_3b < csize_4b) ? 3 : 4;
	579	cratio = (csize_3b < csize_4b) ? ((double)maxlen / csize_3b) : ((double)maxlen / csize_4b);
	580	break;
	581	default:
	582	break;
	583	}
	584	// discard probes with small compression ratios (too expensive)
	585	if (cratio < cratio_ [clevel]) {
	586	goto out;
	587	}
	588
377	589	/* we start with literal copy */
378		copy = 2;
	590	copy = 4;
379	591	*op++ = MAX_COPY - 1;
380	592	op++ = ip++;
381	593	op++ = ip++;
	594	op++ = ip++;
	595	op++ = ip++;
382	596
383	597	/* main loop */
384		while (BLOSCLZ_EXPECT_CONDITIONAL(ip < ip_limit)) {
	598	while (BLOSCLZ_LIKELY(ip < ip_limit)) {
385	599	const uint8_t* ref;
386		uint32_t distance;
387		uint32_t len = 3; /* minimum match length */
	600	unsigned distance;
388	601	uint8_t* anchor = ip; /* comparison starting-point */
389	602
390		/* check for a run */
391		if (ip[0] == ip[-1] && BLOSCLZ_READU16(ip - 1) == BLOSCLZ_READU16(ip + 1)) {
392		distance = 1;
393		ref = anchor - 1 + 3;
394		goto match;
395		}
396
397	603	/* find potential match */
398		HASH_FUNCTION(hval, ip, hashlog);
	604	seq = BLOSCLZ_READU32(ip);
	605	HASH_FUNCTION(hval, seq, hashlog)
399	606	ref = ibase + htab[hval];
400	607
401	608	/* calculate distance to the match */
402		distance = (int32_t)(anchor - ref);
403
404		/* update hash table if necessary */
405		/* not exactly sure why masking the distance works best, but this is what the experiments say */
406		if (!shuffle \|\| (distance & (MAX_COPY - 1)) == 0) {
407		htab[hval] = (uint16_t) (anchor - ibase);
408		}
	609	distance = anchor - ref;
	610
	611	/* update hash table */
	612	htab[hval] = (uint32_t) (anchor - ibase);
409	613
410	614	if (distance == 0 \|\| (distance >= MAX_FARDISTANCE)) {
411	615	LITERAL(ip, op, op_limit, anchor, copy)
	616	continue;
412	617	}
413	618
414	619	/* is this a match? check the first 4 bytes */
415		if (BLOSCLZ_READU32(ref) == BLOSCLZ_READU32(ip)) {
416		len = 4;
	620	if (BLOSCLZ_UNLIKELY(BLOSCLZ_READU32(ref) == BLOSCLZ_READU32(ip))) {
417	621	ref += 4;
418		}
419		/* check just the first 3 bytes */
420		else if (ref++ != ip++ \|\| ref++ != ip++ \|\| ref++ != ip) {
	622	} else {
421	623	/* no luck, copy as a literal */
422	624	LITERAL(ip, op, op_limit, anchor, copy)
423		}
424
425		match:
	625	continue;
	626	}
426	627
427	628	/* last matched byte */
428		ip = anchor + len;
	629	ip = anchor + 4;
429	630
430	631	/* distance is biased */
431	632	distance--;
432	633
433		if (!distance) {
434		/* zero distance means a run */
435		#if defined(__AVX2__)
436		ip = get_run_32(ip, ip_bound, ref);
437		#elif defined(__SSE2__)
438		ip = get_run_16(ip, ip_bound, ref);
439		#else
440		ip = get_run(ip, ip_bound, ref);
441		#endif
442		}
443		else {
444		#if defined(__AVX2__)
445		ip = get_match_32(ip, ip_bound + IP_BOUNDARY, ref);
446		#elif defined(__SSE2__)
447		ip = get_match_16(ip, ip_bound + IP_BOUNDARY, ref);
448		#else
449		ip = get_match(ip, ip_bound + IP_BOUNDARY, ref);
450		#endif
	634	/* get runs or matches; zero distance means a run */
	635	ip = get_run_or_match(ip, ip_bound, ref, !distance);
	636
	637	/* length is biased, '1' means a match of 3 bytes */
	638	ip -= ipshift;
	639
	640	unsigned len = (int)(ip - anchor);
	641	// If match is close, let's reduce the minimum length to encode it
	642	unsigned minlen = (clevel == 9) ? ipshift : minlen_[clevel];
	643
	644	// Encoding short lengths is expensive during decompression
	645	// Encode only for reasonable lengths (extensive experiments done)
	646	if (len < minlen \|\| (len <= 5 && distance >= MAX_DISTANCE)) {
	647	LITERAL(ip, op, op_limit, anchor, copy)
	648	continue;
451	649	}
452	650
453	651	/* if we have copied something, adjust the copy count */

457	655	else
458	656	/* back, to overwrite the copy count */
459	657	op--;
460
461	658	/* reset literal counter */
462	659	copy = 0;
463	660
464		/* length is biased, '1' means a match of 3 bytes */
465		ip -= 3;
466		len = (int32_t)(ip - anchor);
467
468		/* check that we have space enough to encode the match for all the cases */
469		if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + (len / 255) + 6 > op_limit)) goto out;
470
471	661	/* encode the match */
472	662	if (distance < MAX_DISTANCE) {
473		if (len < 7U) {
474		*op++ = (uint8_t)((len << 5U) + (distance >> 8U));
475		*op++ = (uint8_t)((distance & 255U));
476		}
477		else {
478		*op++ = (uint8_t)((7U << 5U) + (distance >> 8U));
479		for (len -= 7U; len >= 255U; len -= 255U)
480		*op++ = 255U;
481		*op++ = (uint8_t)len;
482		*op++ = (uint8_t)((distance & 255U));
483		}
484		}
485		else {
	663	if (len < 7) {
	664	DISTANCE_SHORT(op, op_limit, len, distance)
	665	} else {
	666	DISTANCE_LONG(op, op_limit, len, distance)
	667	}
	668	} else {
486	669	/* far away, but not yet in the another galaxy... */
487		if (len < 7U) {
488		distance -= MAX_DISTANCE;
489		*op++ = (uint8_t)((len << 5U) + 31U);
490		*op++ = 255U;
491		*op++ = (uint8_t)(distance >> 8U);
492		*op++ = (uint8_t)(distance & 255U);
493		}
494		else {
495		distance -= MAX_DISTANCE;
496		*op++ = (7U << 5U) + 31U;
497		for (len -= 7U; len >= 255U; len -= 255U)
498		*op++ = 255U;
499		*op++ = (uint8_t)len;
500		*op++ = 255U;
501		*op++ = (uint8_t)(distance >> 8U);
502		*op++ = (uint8_t)(distance & 255U);
	670	distance -= MAX_DISTANCE;
	671	if (len < 7) {
	672	DISTANCE_SHORT_FAR(op, op_limit, len, distance)
	673	} else {
	674	DISTANCE_LONG_FAR(op, op_limit, len, distance)
503	675	}
504	676	}
505	677
506	678	/* update the hash at match boundary */
507		if (ip < ip_limit) {
508		HASH_FUNCTION(hval, ip, hashlog);
509		htab[hval] = (uint16_t)(ip - ibase);
510		}
511		ip += 2;
	679	seq = BLOSCLZ_READU32(ip);
	680	HASH_FUNCTION(hval, seq, hashlog)
	681	htab[hval] = (uint32_t) (ip++ - ibase);
	682	seq >>= 8U;
	683	HASH_FUNCTION(hval, seq, hashlog)
	684	htab[hval] = (uint32_t) (ip++ - ibase);
512	685	/* assuming literal copy */
	686
	687	if (BLOSCLZ_UNLIKELY(op + 1 > op_limit))
	688	goto out;
513	689	*op++ = MAX_COPY - 1;
514
515		// reset the number of max copies
516		nmax_copies = 0;
517	690	}
518	691
519	692	/* left-over as literal copy */
520		ip_bound++;
521		while (ip <= ip_bound) {
522		if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + 2 > op_limit)) goto out;
	693	while (BLOSCLZ_UNLIKELY(ip <= ip_bound)) {
	694	if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) goto out;
523	695	op++ = ip++;
524	696	copy++;
525		if (copy == MAX_COPY) {
	697	if (BLOSCLZ_UNLIKELY(copy == MAX_COPY)) {
526	698	copy = 0;
527	699	*op++ = MAX_COPY - 1;
528	700	}

541	713
542	714	out:
543	715	return 0;
544
545	716	}
546	717
547	718	// See https://habr.com/en/company/yandex/blog/457612/

563	734
564	735	static const ALIGNED_TYPE_(uint8_t, 16) masks[] =
565	736	{
566		0, 1, 2, 1, 4, 1, 4, 2, 8, 7, 6, 5, 4, 3, 2, 1, // offset = 0, not used as mask, but for shift
567		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // offset = 1
568		0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
569		0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0,
570		0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
571		0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0,
572		0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
573		0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1,
574		0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
575		0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6,
576		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5,
577		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4,
578		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3,
579		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2,
580		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1,
581		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0,
582		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // offset = 16
	737	0, 1, 2, 1, 4, 1, 4, 2, 8, 7, 6, 5, 4, 3, 2, 1, // offset = 0, not used as mask, but for shift
	738	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // offset = 1
	739	0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
	740	0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0,
	741	0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
	742	0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0,
	743	0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
	744	0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1,
	745	0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
	746	0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6,
	747	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5,
	748	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4,
	749	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3,
	750	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2,
	751	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1,
	752	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0,
	753	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // offset = 16
583	754	};
584	755
585	756	_mm_storeu_si128((__m128i *)(op),

599	770	}
600	771	#endif
601	772
602
603		/**
604		Define blosc_decompress and blosc_decompress_unsafe.
605		*/
606		#define BLOSCLZ_SAFE
607		#include "blosclz_impl.inc"
608		#undef BLOSCLZ_SAFE
609		#define blosclz_decompress blosclz_decompress_unsafe
610		#include "blosclz_impl.inc"
611		#undef blosclz_decompress
	773	// LZ4 wildCopy which can reach excellent copy bandwidth (even if insecure)
	774	static inline void wild_copy(uint8_t out, const uint8_t from, uint8_t* end) {
	775	uint8_t* d = out;
	776	const uint8_t* s = from;
	777	uint8_t* const e = end;
	778
	779	do { memcpy(d,s,8); d+=8; s+=8; } while (d<e);
	780	}
	781
	782	int blosclz_decompress(const void* input, int length, void* output, int maxout) {
	783	const uint8_t* ip = (const uint8_t*)input;
	784	const uint8_t* ip_limit = ip + length;
	785	uint8_t* op = (uint8_t*)output;
	786	uint32_t ctrl;
	787	uint8_t* op_limit = op + maxout;
	788	if (BLOSCLZ_UNLIKELY(length == 0)) {
	789	return 0;
	790	}
	791	ctrl = (*ip++) & 31U;
	792
	793	while (1) {
	794	if (ctrl >= 32) {
	795	// match
	796	int32_t len = (ctrl >> 5U) - 1 ;
	797	int32_t ofs = (ctrl & 31U) << 8U;
	798	uint8_t code;
	799	const uint8_t* ref = op - ofs;
	800
	801	if (len == 7 - 1) {
	802	do {
	803	if (BLOSCLZ_UNLIKELY(ip + 1 >= ip_limit)) {
	804	return 0;
	805	}
	806	code = *ip++;
	807	len += code;
	808	} while (code == 255);
	809	}
	810	else {
	811	if (BLOSCLZ_UNLIKELY(ip + 1 >= ip_limit)) {
	812	return 0;
	813	}
	814	}
	815	code = *ip++;
	816	len += 3;
	817	ref -= code;
	818
	819	/* match from 16-bit distance */
	820	if (BLOSCLZ_UNLIKELY(code == 255)) {
	821	if (ofs == (31U << 8U)) {
	822	if (ip + 1 >= ip_limit) {
	823	return 0;
	824	}
	825	ofs = (*ip++) << 8U;
	826	ofs += *ip++;
	827	ref = op - ofs - MAX_DISTANCE;
	828	}
	829	}
	830
	831	if (BLOSCLZ_UNLIKELY(op + len > op_limit)) {
	832	return 0;
	833	}
	834
	835	if (BLOSCLZ_UNLIKELY(ref - 1 < (uint8_t*)output)) {
	836	return 0;
	837	}
	838
	839	if (BLOSCLZ_UNLIKELY(ip >= ip_limit)) break;
	840	ctrl = *ip++;
	841
	842	ref--;
	843	if (ref == op - 1) {
	844	/* optimized copy for a run */
	845	memset(op, *ref, len);
	846	op += len;
	847	}
	848	else if ((op - ref >= 8) && (op_limit - op >= len + 8)) {
	849	// copy with an overlap not larger than 8
	850	wild_copy(op, ref, op + len);
	851	op += len;
	852	}
	853	else {
	854	// general copy with any overlap
	855	#ifdef __AVX2__
	856	if (op - ref <= 16) {
	857	// This is not faster on a combination of compilers (clang, gcc, icc) or machines, but
	858	// it is not slower either. Let's activate here for experimentation.
	859	op = copy_match_16(op, ref, len);
	860	}
	861	else {
	862	#endif
	863	op = copy_match(op, ref, (unsigned) len);
	864	#ifdef __AVX2__
	865	}
	866	#endif
	867	}
	868	}
	869	else {
	870	// literal
	871	ctrl++;
	872	if (BLOSCLZ_UNLIKELY(op + ctrl > op_limit)) {
	873	return 0;
	874	}
	875	if (BLOSCLZ_UNLIKELY(ip + ctrl > ip_limit)) {
	876	return 0;
	877	}
	878
	879	memcpy(op, ip, ctrl); op += ctrl; ip += ctrl;
	880	// On GCC-6, fastcopy this is still faster than plain memcpy
	881	// However, using recent CLANG/LLVM 9.0, there is almost no difference
	882	// in performance.
	883	// And starting on CLANG/LLVM 10 and GCC 9, memcpy is generally faster.
	884	// op = fastcopy(op, ip, (unsigned) ctrl); ip += ctrl;
	885
	886	if (BLOSCLZ_UNLIKELY(ip >= ip_limit)) break;
	887	ctrl = *ip++;
	888	}
	889	}
	890
	891	return (int)(op - (uint8_t*)output);
	892	}

-10

blosc/blosclz.h less more

2	2
3	3	Author: Francesc Alted <francesc@blosc.org>
4	4
5		See LICENSES/BLOSC.txt for details about copyright and rights to use.
	5	See LICENSE.txt for details about copyright and rights to use.
6	6	**********************************************************************/
7	7
8	8	/*********************************************************************

18	18	#if defined (__cplusplus)
19	19	extern "C" {
20	20	#endif
	21
21	22
22	23	/**
23	24	Compress a block of data in the input buffer and returns the size of

39	40	The input buffer and the output buffer can not overlap.
40	41	*/
41	42
42		int blosclz_compress(const int opt_level, const void* input, int length,
43		void* output, int maxout, int shuffle);
	43	int blosclz_compress(int opt_level, const void* input, int length,
	44	void* output, int maxout);
44	45
45	46	/**
46	47	Decompress a block of compressed data and returns the size of the

56	57
57	58	int blosclz_decompress(const void* input, int length, void* output, int maxout);
58	59
59		/**
60		Same as above, except that it is not safe to run on invalid/untrusted input,
61		and may be slightly faster.
62		*/
63		int blosclz_decompress_unsafe(const void* input, int length, void* output,
64		int maxout);
65
66	60	#if defined (__cplusplus)
67	61	}
68	62	#endif

-126

~~blosc/blosclz_impl.inc~~ less more

0		/**
1		* Defines blosclz_decompress for a particular choice of BLOSCLZ_SAFE.
2		*
3		* This is included from blosclz.c once for each value of BLOSCLZ_SAFE.
4		*/
5
6		int blosclz_decompress(const void* input, int length, void* output, int maxout) {
7		const uint8_t* ip = (const uint8_t*)input;
8		const uint8_t* ip_limit = ip + length;
9		uint8_t* op = (uint8_t*)output;
10		int32_t ctrl;
11		int32_t loop = 1;
12		#ifdef BLOSCLZ_SAFE
13		uint8_t* op_limit = op + maxout;
14		if (BLOSCLZ_UNEXPECT_CONDITIONAL(length == 0)) {
15		return 0;
16		}
17		#endif
18		ctrl = (*ip++) & 31;
19
20		do {
21		uint8_t* ref = op;
22		int32_t len = ctrl >> 5;
23		int32_t ofs = (ctrl & 31) << 8;
24
25		if (ctrl >= 32) {
26		uint8_t code;
27		len--;
28		ref -= ofs;
29		if (len == 7 - 1) {
30		do {
31		#ifdef BLOSCLZ_SAFE
32		if (BLOSCLZ_UNEXPECT_CONDITIONAL(ip + 1 >= ip_limit)) {
33		return 0;
34		}
35		#endif
36		code = *ip++;
37		len += code;
38		} while (code == 255);
39		} else {
40		#ifdef BLOSCLZ_SAFE
41		if (BLOSCLZ_UNEXPECT_CONDITIONAL(ip >= ip_limit)) {
42		return 0;
43		}
44		#endif
45		}
46		code = *ip++;
47		ref -= code;
48
49		/* match from 16-bit distance */
50		if (BLOSCLZ_UNEXPECT_CONDITIONAL(code == 255)) if (BLOSCLZ_EXPECT_CONDITIONAL(ofs == (31 << 8))) {
51		#ifdef BLOSCLZ_SAFE
52		if (BLOSCLZ_UNEXPECT_CONDITIONAL(ip + 1 >= ip_limit)) {
53		return 0;
54		}
55		#endif
56		ofs = (*ip++) << 8;
57		ofs += *ip++;
58		ref = op - ofs - MAX_DISTANCE;
59		}
60
61		#ifdef BLOSCLZ_SAFE
62		if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + len + 3 > op_limit)) {
63		return 0;
64		}
65
66		if (BLOSCLZ_UNEXPECT_CONDITIONAL(ref - 1 < (uint8_t*)output)) {
67		return 0;
68		}
69		#endif
70
71		if (BLOSCLZ_EXPECT_CONDITIONAL(ip < ip_limit))
72		ctrl = *ip++;
73		else
74		loop = 0;
75
76		if (ref == op) {
77		/* optimized copy for a run */
78		uint8_t b = ref[-1];
79		memset(op, b, len + 3);
80		op += len + 3;
81		}
82		else {
83		/* copy from reference */
84		ref--;
85		len += 3;
86		#ifdef __AVX2__
87		if (op - ref <= 16) {
88		// This is not faster on a combination of compilers (clang, gcc, icc) or machines, but
89		// it is not slower either. Let's activate here for experimentation.
90		op = copy_match_16(op, ref, len);
91		}
92		else {
93		#endif
94		// We absolutely need a blosc_internal_copy_match here
95		op = blosc_internal_copy_match(op, ref, (unsigned) len);
96		#ifdef __AVX2__
97		}
98		#endif
99		}
100		}
101		else {
102		ctrl++;
103		#ifdef BLOSCLZ_SAFE
104		if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + ctrl > op_limit)) {
105		return 0;
106		}
107		if (BLOSCLZ_UNEXPECT_CONDITIONAL(ip + ctrl > ip_limit)) {
108		return 0;
109		}
110		#endif
111
112		// memcpy(op, ip, ctrl); op += ctrl; ip += ctrl;
113		// On GCC-6, blosc_internal_fastcopy this is still faster than plain memcpy
114		// However, using recent CLANG/LLVM 9.0, there is almost no difference
115		// in performance.
116		op = blosc_internal_fastcopy(op, ip, (unsigned) ctrl); ip += ctrl;
117
118		loop = (int32_t)BLOSCLZ_EXPECT_CONDITIONAL(ip < ip_limit);
119		if (loop)
120		ctrl = *ip++;
121		}
122		} while (BLOSCLZ_EXPECT_CONDITIONAL(loop));
123
124		return (int)(op - (uint8_t*)output);
125		}

+32

-31

blosc/fastcopy.c less more

111	111	__m256i chunk;
112	112	chunk = _mm256_loadu_si256((__m256i*)from);
113	113	_mm256_storeu_si256((__m256i*)out, chunk);
114		from += 32; out += 32;
	114	out += 32;
115	115	#elif defined(__SSE2__)
116	116	__m128i chunk;
117	117	chunk = _mm_loadu_si128((__m128i*)from);

138	138	return out;
139	139	}
140	140
141		#if defined(__AVX2__)
142		static inline unsigned char copy_32_bytes_aligned(unsigned char out, const unsigned char *from) {
143		__m256i chunk;
144		chunk = _mm256_load_si256((__m256i*)from);
145		_mm256_storeu_si256((__m256i*)out, chunk);
146		return out + 32;
147		}
148		#endif // __AVX2__
	141	// This is never used, so comment it out
	142	//#if defined(__AVX2__)
	143	//static inline unsigned char copy_32_bytes_aligned(unsigned char out, const unsigned char *from) {
	144	// __m256i chunk;
	145	// chunk = _mm256_load_si256((__m256i*)from);
	146	// _mm256_storeu_si256((__m256i*)out, chunk);
	147	// return out + 32;
	148	//}
	149	//#endif // __AVX2__
149	150
150	151	/* Copy LEN bytes (7 or fewer) from FROM into OUT. Return OUT + LEN. */
151	152	static inline unsigned char copy_bytes(unsigned char out, const unsigned char *from, unsigned len) {

207	208	case 7:
208	209	out = copy_8_bytes(out, from);
209	210	from += sz;
210		#ifdef AVOID_FALLTHROUGH_WARNING
	211	#ifdef AVOID_FALLTHROUGH_WARNING
211	212	__attribute__ ((fallthrough)); // Shut-up -Wimplicit-fallthrough warning in GCC
212		#endif
	213	#endif
213	214	case 6:
214	215	out = copy_8_bytes(out, from);
215	216	from += sz;
216		#ifdef AVOID_FALLTHROUGH_WARNING
	217	#ifdef AVOID_FALLTHROUGH_WARNING
217	218	__attribute__ ((fallthrough));
218		#endif
	219	#endif
219	220	case 5:
220	221	out = copy_8_bytes(out, from);
221	222	from += sz;
222		#ifdef AVOID_FALLTHROUGH_WARNING
	223	#ifdef AVOID_FALLTHROUGH_WARNING
223	224	__attribute__ ((fallthrough));
224		#endif
	225	#endif
225	226	case 4:
226	227	out = copy_8_bytes(out, from);
227	228	from += sz;
228		#ifdef AVOID_FALLTHROUGH_WARNING
	229	#ifdef AVOID_FALLTHROUGH_WARNING
229	230	__attribute__ ((fallthrough));
230		#endif
	231	#endif
231	232	case 3:
232	233	out = copy_8_bytes(out, from);
233	234	from += sz;
234		#ifdef AVOID_FALLTHROUGH_WARNING
	235	#ifdef AVOID_FALLTHROUGH_WARNING
235	236	__attribute__ ((fallthrough));
236		#endif
	237	#endif
237	238	case 2:
238	239	out = copy_8_bytes(out, from);
239	240	from += sz;
240		#ifdef AVOID_FALLTHROUGH_WARNING
	241	#ifdef AVOID_FALLTHROUGH_WARNING
241	242	__attribute__ ((fallthrough));
242		#endif
	243	#endif
243	244	case 1:
244	245	out = copy_8_bytes(out, from);
245	246	from += sz;
246		#ifdef AVOID_FALLTHROUGH_WARNING
	247	#ifdef AVOID_FALLTHROUGH_WARNING
247	248	__attribute__ ((fallthrough));
248		#endif
	249	#endif
249	250	default:
250	251	break;
251	252	}

491	492
492	493
493	494	/* Byte by byte semantics: copy LEN bytes from FROM and write them to OUT. Return OUT + LEN. */
494		unsigned char blosc_internal_fastcopy(unsigned char out, const unsigned char *from, unsigned len) {
	495	unsigned char fastcopy(unsigned char out, const unsigned char *from, unsigned len) {
495	496	switch (len) {
496	497	case 32:
497	498	return copy_32_bytes(out, from);

524	525
525	526
526	527	/* Copy a run */
527		unsigned char* blosc_internal_copy_match(unsigned char out, const unsigned char from, unsigned len) {
	528	unsigned char* copy_match(unsigned char out, const unsigned char from, unsigned len) {
528	529	#if defined(__AVX2__)
529	530	unsigned sz = sizeof(__m256i);
530	531	#elif defined(__SSE2__)

533	534	unsigned sz = sizeof(uint64_t);
534	535	#endif
535	536
536		// If out and from are away more than the size of the copy, then a blosc_internal_fastcopy is safe
537		unsigned overlap_dist = (unsigned) (out - from);
538		if (overlap_dist > sz) {
539		return blosc_internal_fastcopy(out, from, len);
540		}
541
542	537	#if ((defined(__GNUC__) && BLOSC_GCC_VERSION < 800) && !defined(__clang__) && !defined(__ICC) && !defined(__ICL))
543	538	// GCC < 8 in fully optimization mode seems to have problems with the code further below so stop here
544	539	for (; len > 0; len--) {

546	541	}
547	542	return out;
548	543	#endif
	544
	545	// If out and from are away more than the size of the copy, then a fastcopy is safe
	546	unsigned overlap_dist = (unsigned) (out - from);
	547	if (overlap_dist > sz) {
	548	return fastcopy(out, from, len);
	549	}
549	550
550	551	// Otherwise we need to be more careful so as not to overwrite destination
551	552	switch (overlap_dist) {

-5

blosc/fastcopy.h less more

3	3	Author: Francesc Alted <francesc@blosc.org>
4	4	Creation date: 2018-01-03
5	5
6		See LICENSES/BLOSC.txt for details about copyright and rights to use.
	6	See LICENSE.txt for details about copyright and rights to use.
7	7	**********************************************************************/
8	8
9	9	#ifndef BLOSC_FASTCOPY_H
10	10	#define BLOSC_FASTCOPY_H
11	11
12	12	/* Same semantics than memcpy() */
13		unsigned char blosc_internal_fastcopy(unsigned char out, const unsigned char *from, unsigned len);
	13	unsigned char fastcopy(unsigned char out, const unsigned char *from, unsigned len);
14	14
15		/* Same as blosc_internal_fastcopy() but without overwriting origin or destination when they overlap */
16		unsigned char* blosc_internal_copy_match(unsigned char out, const unsigned char from, unsigned len);
	15	/* Same as fastcopy() but without overwriting origin or destination when they overlap */
	16	unsigned char* copy_match(unsigned char out, const unsigned char from, unsigned len);
17	17
18		#endif /BLOSC_FASTCOPY_H/
	18	#endif //BLOSC_FASTCOPY_H

+33

-13

blosc/shuffle.c less more

7	7	**********************************************************************/
8	8
9	9	#include "shuffle.h"
	10	#include "blosc-common.h"
10	11	#include "shuffle-generic.h"
11	12	#include "bitshuffle-generic.h"
12	13	#include "blosc-comp-features.h"

171	172	#define blosc_internal_cpuid(cpuInfo, function_id) blosc_internal_cpuidex(cpuInfo, function_id, 0)
172	173
173	174	#define _XCR_XFEATURE_ENABLED_MASK 0
	175
	176	#if !(defined(_IMMINTRIN_H_INCLUDED) && (BLOSC_GCC_VERSION >= 900))
174	177
175	178	/* Reads the content of an extended control register.
176	179	https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family

191	194	return ((uint64_t)edx << 32) \| eax;
192	195	}
193	196
194		#endif /* defined(_MSC_FULL_VER) */
	197	#else
	198
	199	#define blosc_internal_xgetbv _xgetbv
	200
	201	#endif // !(defined(_IMMINTRIN_H_INCLUDED) && (BLOSC_GCC_VERSION >= 900))
	202	#endif /* defined(_MSC_FULL_VER) */
195	203
196	204	#ifndef _XCR_XFEATURE_ENABLED_MASK
197	205	#define _XCR_XFEATURE_ENABLED_MASK 0x0

403	411	/* Initialize the shuffle implementation if necessary. */
404	412	init_shuffle_implementation();
405	413
406		if ((size % 8) == 0)
	414	if ((size % 8) == 0) {
407	415	/* The number of elems is a multiple of 8 which is supported by
408	416	bitshuffle. */
409		return (int)(host_implementation.bitshuffle)((void)_src, (void)_dest,
410		blocksize / bytesoftype,
411		bytesoftype, (void*)_tmp);
412		else
413		memcpy((void)_dest, (void)_src, blocksize);
	417	int ret = (int)(host_implementation.bitshuffle)((void ) _src, (void ) _dest,
	418	blocksize / bytesoftype,
	419	bytesoftype, (void *) _tmp);
	420	/* Copy the leftovers */
	421	size_t offset = size * bytesoftype;
	422	memcpy((void ) (_dest + offset), (void ) (_src + offset), blocksize - offset);
	423	return ret;
	424	}
	425	else {
	426	memcpy((void ) _dest, (void ) _src, blocksize);
	427	}
414	428	return size;
415	429	}
416	430

424	438	/* Initialize the shuffle implementation if necessary. */
425	439	init_shuffle_implementation();
426	440
427		if ((size % 8) == 0)
	441	if ((size % 8) == 0) {
428	442	/* The number of elems is a multiple of 8 which is supported by
429	443	bitshuffle. */
430		return (int)(host_implementation.bitunshuffle)((void)_src, (void)_dest,
431		blocksize / bytesoftype,
432		bytesoftype, (void*)_tmp);
433		else
434		memcpy((void)_dest, (void)_src, blocksize);
	444	int ret = (int) (host_implementation.bitunshuffle)((void ) _src, (void ) _dest,
	445	blocksize / bytesoftype,
	446	bytesoftype, (void *) _tmp);
	447	/* Copy the leftovers */
	448	size_t offset = size * bytesoftype;
	449	memcpy((void ) (_dest + offset), (void ) (_src + offset), blocksize - offset);
	450	return ret;
	451	}
	452	else {
	453	memcpy((void ) _dest, (void ) _src, blocksize);
	454	}
435	455	return size;
436	456	}

-10

~~build.py~~ less more

0		from conan.packager import ConanMultiPackager
1		import os
2
3		if __name__ == "__main__":
4		version = os.getenv("TRAVIS_TAG") or os.getenv("APPVEYOR_REPO_TAG_NAME") or "dev"
5		reference = "c-blosc/%s" % version
6		upload = os.getenv("CONAN_UPLOAD") if (version != "dev") else False
7		builder = ConanMultiPackager(reference=reference, upload=upload)
8		builder.add_common_builds(shared_option_name="c-blosc:shared")
9		builder.run()

-0

code_of_conduct.md less more

	0	# Code of Conduct
	1
	2	The Blosc community has adopted a Code of Conduct that we expect project participants to adhere to.
	3	Please read the [full text](https://github.com/Blosc/community/blob/master/code_of_conduct.md)
	4	so that you can understand what actions will and will not be tolerated.

-0

compat/CMakeLists.txt less more

19	19	if (TEST_INCLUDE_COMPAT)
20	20	file(GLOB DATAFILES *.cdata)
21	21	foreach(datafile ${DATAFILES})
	22	# Don't test data if compressor is deactivated
	23	if((datafile MATCHES "lz4" AND DEACTIVATE_LZ4) OR
	24	(datafile MATCHES "snappy" AND DEACTIVATE_SNAPPY) OR
	25	(datafile MATCHES "zlib" AND DEACTIVATE_ZLIB) OR
	26	(datafile MATCHES "zstd" AND DEACTIVATE_ZSTD))
	27	continue()
	28	endif()
22	29	get_filename_component(fname ${datafile} NAME)
23	30	add_test(test_compat_${fname} filegen decompress ${datafile})
24	31	endforeach(datafile)

compat/blosc-1.18.0-blosclz-bitshuffle.cdata less more

Binary diff not shown

compat/blosc-1.18.0-blosclz.cdata less more

Binary diff not shown

compat/blosc-1.18.0-lz4-bitshuffle.cdata less more

Binary diff not shown

compat/blosc-1.18.0-lz4.cdata less more

Binary diff not shown

compat/blosc-1.18.0-lz4hc.cdata less more

Binary diff not shown

compat/blosc-1.18.0-zlib.cdata less more

Binary diff not shown

compat/blosc-1.18.0-zstd.cdata less more

Binary diff not shown

-74

~~conanfile.py~~ less more

0		import os
1		from conans import ConanFile, CMake, tools
2
3
4		class CbloscConan(ConanFile):
5		name = "c-blosc"
6		description = "An extremely fast, multi-threaded, meta-compressor library"
7		license = "BSD"
8		url = "https://github.com/Blosc/c-blosc"
9		settings = "os", "compiler", "build_type", "arch"
10		options = {"shared": [True, False]}
11		default_options = "shared=False"
12		generators = "cmake"
13		exports_sources = "", "!test_package/", "!appveyor", "!..yml", "!bench/plot-speeds.py", "!build.py", "!.*"
14
15		@property
16		def run_tests(self):
17		return "CONAN_RUN_TESTS" in os.environ
18
19		def build(self):
20		os.mkdir("build")
21		tools.replace_in_file("CMakeLists.txt", "project(blosc)", '''project(blosc)
22		include(${CMAKE_BINARY_DIR}/../conanbuildinfo.cmake)
23		conan_basic_setup(NO_OUTPUT_DIRS)''')
24		cmake = CMake(self)
25		cmake.definitions["BUILD_TESTS"] = "ON" if self.run_tests else "OFF"
26		cmake.definitions["BUILD_BENCHMARKS"] = "ON" if self.run_tests else "OFF"
27		cmake.definitions["BUILD_SHARED"] = "ON" if (self.options.shared or self.run_tests) else "OFF"
28		cmake.definitions["BUILD_STATIC"] = "OFF" if self.options.shared else "ON"
29		cmake.configure(build_folder="build")
30		cmake.build()
31
32		if self.run_tests:
33		self.output.warn("Running tests!!")
34		self.launch_tests()
35
36		def launch_tests(self):
37		"""Conan will remove rpaths from shared libs to be able to reuse the shared libs, we need
38		to tell the tests where to find the shared libs"""
39		test_args = "-VV" if tools.os_info.is_windows else ""
40		with tools.chdir("build"):
41		outdir = os.path.join(self.build_folder, "build", "blosc")
42		if tools.os_info.is_macos:
43		prefix = "DYLD_LIBRARY_PATH=%s" % outdir
44		elif tools.os_info.is_windows:
45		prefix = "PATH=%s;%%PATH%%" % outdir
46		elif tools.os_info.is_linux:
47		prefix = "LD_LIBRARY_PATH=%s" % outdir
48		else:
49		return
50		with tools.environment_append({'CTEST_OUTPUT_ON_FAILURE': '1'}):
51		self.run("%s ctest %s" % (prefix, test_args))
52
53		def package(self):
54		self.copy("blosc.h", dst="include", src="blosc")
55		self.copy("blosc-export.h", dst="include", src="blosc")
56		self.copy("*libblosc.a", dst="lib", keep_path=False)
57
58		if self.options.shared:
59		self.copy("*/blosc.lib", dst="lib", keep_path=False)
60		self.copy("*blosc.dll", dst="bin", keep_path=False)
61		self.copy("blosc.dylib*", dst="lib", keep_path=False, symlinks=True)
62		self.copy("blosc.so", dst="lib", keep_path=False, symlinks=True)
63		self.copy("*libblosc.dll.a", dst="lib", keep_path=False) # Mingw
64		else:
65		self.copy("*libblosc.lib", dst="lib", src="", keep_path=False)
66
67		def package_info(self):
68		if self.settings.compiler == "Visual Studio" and not self.options.shared:
69		self.cpp_info.libs = ["libblosc"]
70		else:
71		self.cpp_info.libs = ["blosc"]
72		if self.settings.os == "Linux":
73		self.cpp_info.libs.append("pthread")

-8

~~test_package/CMakeLists.txt~~ less more

0		project(PackageTest CXX)
1		cmake_minimum_required(VERSION 2.8.12)
2
3		include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
4		conan_basic_setup()
5
6		add_executable(example example.cpp)
7		target_link_libraries(example ${CONAN_LIBS})

-24

~~test_package/conanfile.py~~ less more

0		from conans import ConanFile, CMake, tools
1
2
3		class CbloscTestConan(ConanFile):
4		settings = "os", "compiler", "build_type", "arch"
5		generators = "cmake"
6
7		def build(self):
8		cmake = CMake(self)
9		cmake.configure()
10		cmake.build()
11
12		def imports(self):
13		self.copy("*.dll", dst="bin", src="bin" )
14		self.copy(".dylib", dst="bin", src="lib")
15		self.copy(".so", dst="bin", src="lib")
16
17		def test(self):
18		with tools.chdir("bin"):
19		if tools.os_info.is_windows:
20		self.run("example")
21		else:
22		prefix = "DYLD_LIBRARY_PATH=." if tools.os_info.is_macos else ""
23		self.run("%s ./example" % prefix)

-59

~~test_package/example.cpp~~ less more

0		#include <stdio.h>
1		#include <blosc.h>
2
3		#define SIZE 100100100
4
5		int main(){
6		static float data[SIZE];
7		static float data_out[SIZE];
8		static float data_dest[SIZE];
9		int isize = SIZEsizeof(float), osize = SIZEsizeof(float);
10		int dsize = SIZE*sizeof(float), csize;
11		int i;
12
13		for(i=0; i<SIZE; i++){
14		data[i] = i;
15		}
16
17		/* Register the filter with the library */
18		printf("Blosc version info: %s (%s)\n",
19		BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
20
21		/* Initialize the Blosc compressor */
22		blosc_init();
23
24		/* Compress with clevel=5 and shuffle active */
25		csize = blosc_compress(5, 1, sizeof(float), isize, data, data_out, osize);
26		if (csize == 0) {
27		printf("Buffer is uncompressible. Giving up.\n");
28		return 1;
29		}
30		else if (csize < 0) {
31		printf("Compression error. Error code: %d\n", csize);
32		return csize;
33		}
34
35		printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1.*isize) / csize);
36
37		/* Decompress */
38		dsize = blosc_decompress(data_out, data_dest, dsize);
39		if (dsize < 0) {
40		printf("Decompression error. Error code: %d\n", dsize);
41		return dsize;
42		}
43
44		printf("Decompression succesful!\n");
45
46		/* After using it, destroy the Blosc environment */
47		blosc_destroy();
48
49		for(i=0;i<SIZE;i++){
50		if(data[i] != data_dest[i]) {
51		printf("Decompressed data differs from original!\n");
52		return -1;
53		}
54		}
55
56		printf("Succesful roundtrip!\n");
57		return 0;
58		}

-2

tests/CMakeLists.txt less more

19	19	endif()
20	20	endif()
21	21
22		# test_compressor will be enabled only when LZ4 support is in
23		if(target STREQUAL test_compressor AND DEACTIVATE_LZ4)
	22	# Disable targets that use lz4 compressor when lz4 is deactivated
	23	if((target STREQUAL test_compressor) OR (target STREQUAL test_bitshuffle_leftovers) AND DEACTIVATE_LZ4)
24	24	message("Skipping ${target} on non-LZ4 builds")
25	25	continue()
26	26	endif()

+62

-0

tests/fuzz/CMakeLists.txt less more

	0	# flags
	1	link_directories(${PROJECT_BINARY_DIR}/blosc)
	2
	3	# look for fuzzing lib and link with it if found
	4	if(CMAKE_C_COMPILER_ID STREQUAL "Clang")
	5	enable_language(CXX)
	6
	7	if(DEFINED ENV{LIB_FUZZING_ENGINE})
	8	set(FUZZING_ENGINE $ENV{LIB_FUZZING_ENGINE})
	9	set(FUZZING_ENGINE_FOUND TRUE)
	10	else()
	11	find_library(FUZZING_ENGINE "FuzzingEngine")
	12	endif()
	13	endif()
	14
	15	# If fuzzing lib not found then create standalone fuzz runner
	16	if(NOT FUZZING_ENGINE_FOUND)
	17	set(FUZZER_SRC standalone.c)
	18	else()
	19	set(FUZZER_SRC)
	20	endif()
	21
	22	# sources
	23	file(GLOB SOURCES fuzz_*.c)
	24
	25	# targets and tests
	26	foreach(source ${SOURCES})
	27	get_filename_component(target ${source} NAME_WE)
	28
	29	# Enable support for testing accelerated shuffles
	30	if(COMPILER_SUPPORT_SSE2)
	31	# Define a symbol so tests for SSE2 shuffle/unshuffle will be compiled in.
	32	set_property(
	33	SOURCE ${source}
	34	APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_SSE2_ENABLED)
	35	endif(COMPILER_SUPPORT_SSE2)
	36	# if(COMPILER_SUPPORT_AVX2)
	37	# # Define a symbol so tests for AVX2 shuffle/unshuffle will be compiled in.
	38	# set_property(
	39	# SOURCE ${source}
	40	# APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_AVX2_ENABLED)
	41	# endif(COMPILER_SUPPORT_AVX2)
	42
	43	add_executable(${target} ${source} ${FUZZER_SRC})
	44
	45	# OSS-Fuzz expect fuzzers to end with _fuzzer
	46	string(REPLACE "fuzz_" "" output_name ${target})
	47	set_target_properties(${target} PROPERTIES OUTPUT_NAME ${output_name}_fuzzer)
	48
	49	if(FUZZING_ENGINE_FOUND)
	50	set_target_properties(${target} PROPERTIES LINKER_LANGUAGE CXX)
	51	target_link_libraries(${target} ${FUZZING_ENGINE})
	52	endif()
	53
	54	target_link_libraries(${target} blosc_static)
	55	add_dependencies(${target} blosc_static)
	56
	57	# run standalone fuzzer against each file
	58	file(GLOB COMPAT_FILES ${PROJECT_SOURCE_DIR}/compat/*.cdata)
	59	add_test(NAME ${target} COMMAND ${target} ${COMPAT_FILES})
	60
	61	endforeach(source)

+65

-0

tests/fuzz/fuzz_compress.c less more

	0	#include <stdint.h>
	1	#include <stdlib.h>
	2
	3	#include "blosc.h"
	4
	5	#ifdef __cplusplus
	6	extern "C" {
	7	#endif
	8
	9	int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
	10	const char *compressors[] = { "blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd" };
	11	int level = 9, filter = BLOSC_BITSHUFFLE, cindex = 0, i = 0;
	12	size_t nbytes, cbytes, blocksize;
	13	void output, input;
	14
	15	blosc_set_nthreads(1);
	16
	17	if (size > 0)
	18	level = data[0] % (9 + 1);
	19	if (size > 1)
	20	filter = data[1] % (BLOSC_BITSHUFFLE + 1);
	21	if (size > 2)
	22	cindex = data[2];
	23
	24	/* Find next available compressor */
	25	while (blosc_set_compressor(compressors[cindex % 6]) == -1 && i < 6) {
	26	cindex++, i++;
	27	}
	28	if (i == 6) {
	29	/* No compressors available */
	30	return 0;
	31	}
	32
	33	if (size > 3 && data[3] % 7 == 0)
	34	blosc_set_blocksize(4096);
	35
	36	if (size > 4)
	37	blosc_set_splitmode(data[4] % BLOSC_FORWARD_COMPAT_SPLIT + 1);
	38
	39	output = malloc(size + 1);
	40	if (output == NULL)
	41	return 0;
	42
	43	if (blosc_compress(level, filter, 1, size, data, output, size) == 0) {
	44	/* Cannot compress src buffer into dest */
	45	free(output);
	46	return 0;
	47	}
	48
	49	blosc_cbuffer_sizes(output, &nbytes, &cbytes, &blocksize);
	50
	51	input = malloc(cbytes);
	52	if (input != NULL) {
	53	blosc_decompress(output, input, cbytes);
	54	free(input);
	55	}
	56
	57	free(output);
	58
	59	return 0;
	60	}
	61
	62	#ifdef __cplusplus
	63	}
	64	#endif

+41

-0

tests/fuzz/fuzz_decompress.c less more

	0	#include <stdint.h>
	1	#include <stdlib.h>
	2
	3	#include "blosc.h"
	4
	5	#ifdef __cplusplus
	6	extern "C" {
	7	#endif
	8
	9	int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
	10	size_t nbytes, cbytes, blocksize;
	11	void *output;
	12
	13	if (size < BLOSC_MIN_HEADER_LENGTH) {
	14	return 0;
	15	}
	16
	17	blosc_cbuffer_sizes(data, &nbytes, &cbytes, &blocksize);
	18	if (cbytes != size) {
	19	return 0;
	20	}
	21	if (nbytes == 0) {
	22	return 0;
	23	}
	24
	25	if (blosc_cbuffer_validate(data, size, &nbytes) != 0) {
	26	/* Unexpected nbytes specified in blosc header */
	27	return 0;
	28	}
	29
	30	output = malloc(cbytes);
	31	if (output != NULL) {
	32	blosc_decompress(data, output, cbytes);
	33	free(output);
	34	}
	35	return 0;
	36	}
	37
	38	#ifdef __cplusplus
	39	}
	40	#endif

+44

-0

tests/fuzz/standalone.c less more

	0	#include <assert.h>
	1	#include <stdio.h>
	2	#include <stdlib.h>
	3
	4	extern int LLVMFuzzerTestOneInput(const unsigned char *data, size_t size);
	5
	6	int main(int argc, char **argv) {
	7	int i;
	8	fprintf(stderr, "Running %d inputs\n", argc - 1);
	9
	10	for (i = 1; i < argc; i++) {
	11	size_t len, err, n_read = 0;
	12	unsigned char *buf;
	13	FILE *f = NULL;
	14
	15	f = fopen(argv[i], "rb+");
	16	if (f == NULL) {
	17	/* Failed to open this file: it may be a directory. */
	18	fprintf(stderr, "Skipping: %s\n", argv[i]);
	19	continue;
	20	}
	21	fprintf(stderr, "Running: %s %s\n", argv[0], argv[i]);
	22
	23	fseek(f, 0, SEEK_END);
	24	len = ftell(f);
	25	fseek(f, 0, SEEK_SET);
	26
	27	buf = (unsigned char *)malloc(len);
	28	if (buf != NULL) {
	29	n_read = fread(buf, 1, len, f);
	30	assert(n_read == len);
	31	LLVMFuzzerTestOneInput(buf, len);
	32	free(buf);
	33	}
	34
	35	err = fclose(f);
	36	assert(err == 0);
	37	(void)err;
	38
	39	fprintf(stderr, "Done: %s: (%d bytes)\n", argv[i], (int)n_read);
	40	}
	41
	42	return 0;
	43	}

+140

-0

tests/test_bitshuffle_leftovers.c less more

	0	/*********************************************************************
	1	Blosc - Blocked Shuffling and Compression Library
	2
	3	Unit test for the bitshuffle with blocks that are not aligned.
	4	See https://github.com/Blosc/python-blosc/issues/220
	5	Probably related: https://github.com/Blosc/c-blosc/issues/240
	6
	7	Creation date: 2020-02-18
	8	Author: Francesc Alted <francesc@blosc.org>
	9
	10	See LICENSES/BLOSC.txt for details about copyright and rights to use.
	11	**********************************************************************/
	12
	13	#include "test_common.h"
	14
	15
	16	static int test_roundtrip_bitshuffle8(int size, void data, void data_out, void *data_dest) {
	17	/* Compress with bitshuffle active */
	18	int isize = size;
	19	int osize = size + BLOSC_MIN_HEADER_LENGTH;
	20	int csize = blosc_compress(9, BLOSC_BITSHUFFLE, 8, isize, data, data_out, osize);
	21	int dsize;
	22	int exit_code;
	23	FILE *fout = fopen("test-bitshuffle8-nomemcpy.cdata", "w");
	24
	25	if (csize == 0) {
	26	printf("Buffer is uncompressible. Giving up.\n");
	27	return 1;
	28	}
	29	else if (csize < 0) {
	30	printf("Compression error. Error code: %d\n", csize);
	31	return csize;
	32	}
	33	printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1.*isize) / csize);
	34
	35	fwrite(data_out, csize, 1, fout);
	36	fclose(fout);
	37
	38	/* Decompress */
	39	dsize = blosc_decompress(data_out, data_dest, isize);
	40	if (dsize < 0) {
	41	printf("Decompression error. Error code: %d\n", dsize);
	42	return dsize;
	43	}
	44
	45	printf("Decompression succesful!\n");
	46
	47	exit_code = memcmp(data, data_dest, size) ? EXIT_FAILURE : EXIT_SUCCESS;
	48
	49	if (exit_code == EXIT_SUCCESS)
	50	printf("Succesful roundtrip!\n");
	51	else
	52	printf("Decompressed data differs from original!\n");
	53
	54	return exit_code;
	55	}
	56
	57	static int test_roundtrip_bitshuffle4(int size, void data, void data_out, void *data_dest) {
	58	/* Compress with bitshuffle active */
	59	int isize = size;
	60	int osize = size + BLOSC_MIN_HEADER_LENGTH;
	61	int csize = blosc_compress(9, BLOSC_BITSHUFFLE, 4, isize, data, data_out, osize);
	62	int dsize;
	63	int exit_code;
	64	FILE *fout = fopen("test-bitshuffle4-memcpy.cdata", "w");
	65
	66	if (csize == 0) {
	67	printf("Buffer is uncompressible. Giving up.\n");
	68	return 1;
	69	}
	70	else if (csize < 0) {
	71	printf("Compression error. Error code: %d\n", csize);
	72	return csize;
	73	}
	74	printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1.*isize) / csize);
	75
	76	fwrite(data_out, csize, 1, fout);
	77	fclose(fout);
	78
	79	/* Decompress */
	80	dsize = blosc_decompress(data_out, data_dest, isize);
	81	if (dsize < 0) {
	82	printf("Decompression error. Error code: %d\n", dsize);
	83	return dsize;
	84	}
	85
	86	printf("Decompression succesful!\n");
	87
	88	exit_code = memcmp(data, data_dest, size) ? EXIT_FAILURE : EXIT_SUCCESS;
	89	if (exit_code == EXIT_SUCCESS)
	90	printf("Succesful roundtrip!\n");
	91	else
	92	printf("Decompressed data differs from original!\n");
	93
	94	return exit_code;
	95	}
	96
	97	int main() {
	98	/* `size` below is chosen so that it is not divisible by 8
	99	* (not supported by bitshuffle) and in addition, it is not
	100	* divisible by 8 (typesize) again.
	101	*/
	102	int size = 641091;
	103	int32_t *data = malloc(size);
	104	int32_t *data_out = malloc(size + BLOSC_MIN_HEADER_LENGTH);
	105	int32_t *data_dest = malloc(size);
	106	int result;
	107	int i;
	108
	109	/* Initialize data */
	110	for (i = 0; i < size / sizeof(int32_t); i++) {
	111	((uint32_t*)data)[i] = i;
	112	}
	113	/* leftovers */
	114	for (i = size / sizeof(int32_t) * sizeof(int32_t); i < size; i++) {
	115	((uint8_t*)data)[i] = i;
	116	}
	117
	118	blosc_init();
	119	blosc_set_nthreads(1);
	120	blosc_set_compressor("lz4");
	121	printf("Blosc version info: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
	122	result = test_roundtrip_bitshuffle4(size, data, data_out, data_dest);
	123	if (result != EXIT_SUCCESS) {
	124	goto fail;
	125	}
	126	result = test_roundtrip_bitshuffle8(size, data, data_out, data_dest);
	127	if (result != EXIT_SUCCESS) {
	128	goto fail;
	129	}
	130
	131	free(data);
	132	free(data_out);
	133	free(data_dest);
	134
	135	blosc_destroy();
	136
	137	fail:
	138	return result;
	139	}

-2

tests/test_compressor.c less more

150	150	int cbytes2;
151	151
152	152	/* Get a compressed buffer */
153		blosc_set_compressor("blosclz"); /* avoid lz4 here for now (see #BLOSC_MAX_OVERHEAD8) */
	153	blosc_set_compressor("lz4"); /* avoid lz4 here for now (see #BLOSC_MAX_OVERHEAD8) */
154	154	cbytes = blosc_compress(clevel, doshuffle, typesize, size, src,
155	155	dest, size + BLOSC_MAX_OVERHEAD);
156	156	mu_assert("ERROR: cbytes is not 0", cbytes < size);

159	159	setenv("BLOSC_SHUFFLE", "BITSHUFFLE", 0);
160	160	cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src,
161	161	dest, size + BLOSC_MAX_OVERHEAD);
	162	/* For some reason, shuffle is unreasonably efficient here (much more than bitshuffle) */
162	163	mu_assert("ERROR: BLOSC_SHUFFLE=BITSHUFFLE does not work correctly",
163		cbytes2 < cbytes * 1.5);
	164	cbytes2 < cbytes * 20);
164	165
165	166	/* Reset env var */
166	167	unsetenv("BLOSC_SHUFFLE");

+33

-12

tests/test_maxout.c less more

21	21	size_t size = 1000; /* must be divisible by 4 */
22	22
23	23
	24	/* Check input size > BLOSC_MAX_BUFFERSIZE */
	25	static const char *test_input_too_large(void) {
	26
	27	/* Get a compressed buffer */
	28	cbytes = blosc_compress(clevel, doshuffle, typesize, BLOSC_MAX_BUFFERSIZE + 1, src,
	29	dest, size + BLOSC_MAX_OVERHEAD - 1);
	30	mu_assert("ERROR: cbytes is not 0", cbytes == 0);
	31
	32	return 0;
	33	}
	34
	35
24	36	/* Check maxout with maxout < size */
25	37	static const char *test_maxout_less(void) {
26	38
27	39	/* Get a compressed buffer */
28		cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD - 1);
	40	cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest,
	41	size + BLOSC_MAX_OVERHEAD - 1);
29	42	mu_assert("ERROR: cbytes is not 0", cbytes == 0);
30	43
31	44	return 0;

36	49	static const char *test_maxout_less_memcpy(void) {
37	50
38	51	/* Get a compressed buffer */
39		cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD - 1);
	52	cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
	53	size + BLOSC_MAX_OVERHEAD - 1);
40	54	mu_assert("ERROR: cbytes is not 0", cbytes == 0);
41	55
42	56	return 0;

47	61	static const char *test_maxout_equal(void) {
48	62
49	63	/* Get a compressed buffer */
50		cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD);
	64	cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest,
	65	size + BLOSC_MAX_OVERHEAD);
51	66	mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);
52	67
53	68	/* Decompress the buffer */

62	77	static const char *test_maxout_equal_memcpy(void) {
63	78
64	79	/* Get a compressed buffer */
65		cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD);
	80	cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
	81	size + BLOSC_MAX_OVERHEAD);
66	82	mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);
67	83
68	84	/* Decompress the buffer */

76	92	/* Check maxout with maxout > size */
77	93	static const char *test_maxout_great(void) {
78	94	/* Get a compressed buffer */
79		cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD + 1);
	95	cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest,
	96	size + BLOSC_MAX_OVERHEAD + 1);
80	97	mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);
81	98
82	99	/* Decompress the buffer */

90	107	/* Check maxout with maxout > size (memcpy version) */
91	108	static const char *test_maxout_great_memcpy(void) {
92	109	/* Get a compressed buffer */
93		cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD + 1);
	110	cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
	111	size + BLOSC_MAX_OVERHEAD + 1);
94	112	mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD);
95	113
96	114	/* Decompress the buffer */

103	121	/* Check maxout with maxout < BLOSC_MAX_OVERHEAD */
104	122	static const char *test_max_overhead(void) {
105	123	blosc_init();
106		cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 1);
107		mu_assert("ERROR: cbytes is not correct", cbytes < 0);
	124	cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
	125	BLOSC_MAX_OVERHEAD - 1);
	126	mu_assert("ERROR: cbytes is not correct", cbytes == 0);
108	127	blosc_destroy();
109	128
110	129	blosc_init();
111		cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 2);
112		mu_assert("ERROR: cbytes is not correct", cbytes < 0);
	130	cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest,
	131	BLOSC_MAX_OVERHEAD - 2);
	132	mu_assert("ERROR: cbytes is not correct", cbytes == 0);
113	133	blosc_destroy();
114	134
115	135	blosc_init();
116	136	cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, 0);
117		mu_assert("ERROR: cbytes is not correct", cbytes < 0);
	137	mu_assert("ERROR: cbytes is not correct", cbytes == 0);
118	138	blosc_destroy();
119	139
120	140	return 0;

122	142
123	143
124	144	static const char *all_tests(void) {
	145	mu_run_test(test_input_too_large);
125	146	mu_run_test(test_maxout_less);
126	147	mu_run_test(test_maxout_less_memcpy);
127	148	mu_run_test(test_maxout_equal);

174	195	blosc_destroy();
175	196
176	197	return result != 0;
177		}
	198	}⏎

2	2	\|--------\|---------\|-----\|
3	3	\| Blosc Development Team \| blosc@blosc.org \| http://www.blosc.org \|
4	4
5		\| Gitter \| Travis CI \| Appveyor \| NumFOCUS \|
6		\|--------\|-----------\|----------\|----------\|
7		\| [![Build Status](https://badges.gitter.im/Blosc/c-blosc.svg)](https://gitter.im/Blosc/c-blosc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) \| [![Build Status](https://travis-ci.org/Blosc/c-blosc.svg?branch=master)](https://travis-ci.org/Blosc/c-blosc) \| [![Build Status](https://ci.appveyor.com/api/projects/status/3mlyjc1ak0lbkmte?svg=true)](https://ci.appveyor.com/project/FrancescAlted/c-blosc/branch/master) \| [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](http://numfocus.org) \|
8
	5	\| Gitter \| GH Actions \| NumFOCUS \| Code of Conduct \|
	6	\|--------\|------------\|----------\|-----------------\|
	7	\| [![Gitter](https://badges.gitter.im/Blosc/c-blosc.svg)](https://gitter.im/Blosc/c-blosc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) \| [![CI CMake](https://github.com/Blosc/c-blosc/workflows/CI%20CMake/badge.svg)](https://github.com/Blosc/c-blosc/actions?query=workflow%3A%22CI+CMake%22) \| [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](http://numfocus.org) \| [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](code_of_conduct.md) \|
9	8
10	9	## What is it?
11	10