Commit 90be034bf1c2a5d7964cf138a3ebb9c927ffc5f2 - python-msgpack

Merge tag '1.0.0' into debian/victoria Thomas Goirand 3 years ago

46 changed file(s) with 2082 addition(s) and 1383 deletion(s). Raw diff Collapse all Expand all

+21

-0

.github/workflows/black.yaml less more

	0	name: Black
	1
	2	on: ["push", "pull_request"]
	3
	4	jobs:
	5	black:
	6	runs-on: ubuntu-latest
	7	steps:
	8	- name: Setup Python
	9	uses: actions/setup-python@v1
	10	with:
	11	python-version: '3.x'
	12	architecture: 'x64'
	13
	14	- name: Checkout
	15	uses: actions/checkout@v1
	16
	17	- name: Black Code Formatter
	18	run: \|
	19	pip install black
	20	black --diff --check msgpack/ test/ setup.py

+70

-0

.github/workflows/windows.yaml less more

	0	name: Build and test windows wheels
	1	on:
	2	push:
	3	branches:
	4	- master
	5	- test
	6	pull_request:
	7	create:
	8
	9	jobs:
	10	build:
	11	runs-on: windows-latest
	12	steps:
	13	- name: Checkout
	14	uses: actions/checkout@v1
	15
	16	- name: Cythonize
	17	shell: bash
	18	run: \|
	19	pip install -U Cython
	20	make cython
	21	#python setup.py sdist
	22
	23	- name: Python 3.6 (amd64)
	24	env:
	25	PYTHON: "py -3.6-64"
	26	shell: bash
	27	run: \|
	28	ci/runtests.sh
	29
	30	- name: Python 3.6 (x86)
	31	env:
	32	PYTHON: "py -3.6-32"
	33	shell: bash
	34	run: \|
	35	ci/runtests.sh
	36
	37	- name: Python 3.7 (amd64)
	38	env:
	39	PYTHON: "py -3.7-64"
	40	shell: bash
	41	run: \|
	42	ci/runtests.sh
	43
	44	- name: Python 3.7 (x86)
	45	env:
	46	PYTHON: "py -3.7-32"
	47	shell: bash
	48	run: \|
	49	ci/runtests.sh
	50
	51	- name: Python 3.8 (amd64)
	52	env:
	53	PYTHON: "py -3.8-64"
	54	shell: bash
	55	run: \|
	56	ci/runtests.sh
	57
	58	- name: Python 3.8 (x86)
	59	env:
	60	PYTHON: "py -3.8-32"
	61	shell: bash
	62	run: \|
	63	ci/runtests.sh
	64
	65	- name: Upload Wheels
	66	uses: actions/upload-artifact@v1
	67	with:
	68	name: win-wheels
	69	path: ./dist

+47

-17

.travis.yml less more

	0	version: ~> 1.0
0	1	dist: xenial
1	2	language: python
2	3	cache: pip
3
	4	arch:
	5	- amd64
	6	- arm64
4	7	python:
5	8	# Available Python (PyPy) can be listed by:
6	9	#
7	10	# $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/
8		- "2.7"
9	11	- "3.4"
10	12	- "3.5"
11	13	- "3.6"
12	14	- "3.7"
13		- "3.8-dev"
	15	- "3.8"
	16	- "3.9-dev"
	17
	18
	19	_pure: &pure
	20	install:
	21	- pip install -U pip
	22	- pip install -U pytest pytest-cov codecov
	23	- pip install .
	24	script:
	25	- pytest --cov=msgpack -v test
14	26
15	27	matrix:
16	28	include:
17	29	- name: 32bit build
18		sudo: required
19	30	language: python
20	31	services:
21	32	- docker

28	39	- docker pull $DOCKER_IMAGE
29	40	script:
30	41	- docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
	42
	43	- arch: arm64
	44	name: arm64 32bit build
	45	language: python
	46	services:
	47	- docker
	48	env:
	49	- DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64
	50	install:
	51	- pip install -U pip
	52	- pip install -r requirements.txt
	53	- make cython
	54	- docker pull $DOCKER_IMAGE
	55	script:
	56	- docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
	57
	58	- name: "Python 2 (fallback)"
	59	python: "2.7"
	60	<<: *pure
	61
31	62	- name: "pypy2.7"
32	63	python: "pypy2.7-7.1.1"
33		install:
34		- pip install -e .
35		script:
36		- py.test -v test
	64	<<: *pure
	65
37	66	- name: "pypy3"
38	67	python: "pypy3.6-7.1.1"
39		install:
40		- pip install -e .
41		script:
42		- pytest -v test
43
	68	<<: *pure
44	69
45	70	install:
46	71	- pip install -U pip
47		- pip install -U pytest
48		- pip install -r requirements.txt
	72	- pip install -U pytest pytest-cov codecov
	73	- pip install -r requirements.txt # Cython
49	74	- make cython
50	75	- pip install -e .
51	76
52	77	script:
53	78	- python -c 'import sys; print(hex(sys.maxsize))'
54	79	- python -c 'from msgpack import _cmsgpack'
55		- pytest -v test
56		- MSGPACK_PUREPYTHON=x pytest -v test
	80	- pytest --cov=msgpack -v test
	81	- MSGPACK_PUREPYTHON=x pytest --cov=msgpack -v test
	82
	83	after_success:
	84	- if [ -f .coverage ]; then
	85	codecov;
	86	fi
57	87
58	88	# vim: sw=2 ts=2

+18

-0

ChangeLog.rst less more

	0	1.0.0
	1	=====
	2
	3	Release Date: 2020-02-17
	4
	5	* Remove Python 2 support from the ``msgpack/_cmsgpack``.
	6	``msgpack/fallback`` still supports Python 2.
	7	* Remove ``encoding`` option from the Packer and Unpacker.
	8	* Unpacker: The default value of ``max_buffer_type`` is changed to 100MiB.
	9	* Unpacker: ``strict_map_key`` is True by default now.
	10	* Unpacker: String map keys are interned.
	11	* Drop old buffer protocol support.
	12	* Support Timestamp type.
	13	* Support serializing and decerializing ``datetime`` object
	14	with tzinfo.
	15	* Unpacker: ``Fix Unpacker.read_bytes()`` in fallback implementation. (#352)
	16
	17
0	18	0.6.2
1	19	=====
2	20

-0

DEVELOP.md less more

	0	# Developer's note
	1
	2	## Wheels
	3
	4	Wheels for macOS and Linux are built on Travis and AppVeyr, in
	5	[methane/msgpack-wheels](https://github.com/methane/msgpack-wheels) repository.
	6
	7	Wheels for Windows are built on Github Actions in this repository.
	8

-1

MANIFEST.in less more

0	0	include setup.py
1	1	include COPYING
2		include README.rst
	2	include README.md
3	3	recursive-include msgpack .h .c .pyx .cpp
4	4	recursive-include test *.py

-1

Makefile less more

0	0	.PHONY: all
1	1	all: cython
2	2	python setup.py build_ext -i -f
	3
	4	.PHONY: black
	5	black:
	6	black msgpack/ test/ setup.py
3	7
4	8	.PHONY: cython
5	9	cython:

18	22	.PHONY: clean
19	23	clean:
20	24	rm -rf build
21		rm -f msgpack/_msgpack.cpp
	25	rm -f msgpack/_cmsgpack.cpp
	26	rm -f msgpack/_cmsgpack.*.so
22	27	rm -rf msgpack/__pycache__
23	28	rm -rf test/__pycache__
24	29

+284

-0

README.md less more

	0	# MessagePack for Python
	1
	2	[![Build Status](https://travis-ci.org/msgpack/msgpack-python.svg?branch=master)](https://travis-ci.org/msgpack/msgpack-python)
	3	[![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest)
	4
	5	## What's this
	6
	7	[MessagePack](https://msgpack.org/) is an efficient binary serialization format.
	8	It lets you exchange data among multiple languages like JSON.
	9	But it's faster and smaller.
	10	This package provides CPython bindings for reading and writing MessagePack data.
	11
	12
	13	## Very important notes for existing users
	14
	15	### PyPI package name
	16
	17	TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`.
	18	Do `pip uninstall msgpack-python; pip install msgpack` instead.
	19
	20	Package name on PyPI was changed to msgpack from 0.5.
	21	I upload transitional package (msgpack-python 0.5 which depending on msgpack)
	22	for smooth transition from msgpack-python to msgpack.
	23
	24	Sadly, this doesn't work for upgrade install. After `pip install -U msgpack-python`,
	25	msgpack is removed, and `import msgpack` fail.
	26
	27
	28	### Compatibility with the old format
	29
	30	You can use `use_bin_type=False` option to pack `bytes`
	31	object into raw type in the old msgpack spec, instead of bin type in new msgpack spec.
	32
	33	You can unpack old msgpack format using `raw=True` option.
	34	It unpacks str (raw) type in msgpack into Python bytes.
	35
	36	See note below for detail.
	37
	38
	39	### Major breaking changes in msgpack 1.0
	40
	41	* Python 2
	42
	43	* The extension module does not support Python 2 anymore.
	44	The pure Python implementation (`msgpack.fallback`) is used for Python 2.
	45
	46	* Packer
	47
	48	* `use_bin_type=True` by default. bytes are encoded in bin type in msgpack.
	49	If you are still sing Python 2, you must use unicode for all string types.
	50	You can use `use_bin_type=False` to encode into old msgpack format.
	51	* `encoding` option is removed. UTF-8 is used always.
	52
	53	* Unpacker
	54
	55	* `raw=False` by default. It assumes str types are valid UTF-8 string
	56	and decode them to Python str (unicode) object.
	57	* `encoding` option is removed. You can use `raw=True` to support old format.
	58	* Default value of `max_buffer_size` is changed from 0 to 100 MiB.
	59	* Default value of `strict_map_key` is changed to True to avoid hashdos.
	60	You need to pass `strict_map_key=False` if you have data which contain map keys
	61	which type is not bytes or str.
	62
	63
	64	## Install
	65
	66
	67	$ pip install msgpack
	68
	69
	70	### Pure Python implementation
	71
	72	The extension module in msgpack (`msgpack._cmsgpack`) does not support
	73	Python 2 and PyPy.
	74
	75	But msgpack provides a pure Python implementation (`msgpack.fallback`)
	76	for PyPy and Python 2.
	77
	78	Since the [pip](https://pip.pypa.io/) uses the pure Python implementation,
	79	Python 2 support will not be dropped in the foreseeable future.
	80
	81
	82	### Windows
	83
	84	When you can't use a binary distribution, you need to install Visual Studio
	85	or Windows SDK on Windows.
	86	Without extension, using pure Python implementation on CPython runs slowly.
	87
	88
	89	## How to use
	90
	91	NOTE: In examples below, I use `raw=False` and `use_bin_type=True` for users
	92	using msgpack < 1.0. These options are default from msgpack 1.0 so you can omit them.
	93
	94
	95	### One-shot pack & unpack
	96
	97	Use `packb` for packing and `unpackb` for unpacking.
	98	msgpack provides `dumps` and `loads` as an alias for compatibility with
	99	`json` and `pickle`.
	100
	101	`pack` and `dump` packs to a file-like object.
	102	`unpack` and `load` unpacks from a file-like object.
	103
	104	```pycon
	105	>>> import msgpack
	106	>>> msgpack.packb([1, 2, 3], use_bin_type=True)
	107	'\x93\x01\x02\x03'
	108	>>> msgpack.unpackb(_, raw=False)
	109	[1, 2, 3]
	110	```
	111
	112	`unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple:
	113
	114	```pycon
	115	>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False)
	116	(1, 2, 3)
	117	```
	118
	119	You should always specify the `use_list` keyword argument for backward compatibility.
	120	See performance issues relating to `use_list option`_ below.
	121
	122	Read the docstring for other options.
	123
	124
	125	### Streaming unpacking
	126
	127	`Unpacker` is a "streaming unpacker". It unpacks multiple objects from one
	128	stream (or from bytes provided through its `feed` method).
	129
	130	```py
	131	import msgpack
	132	from io import BytesIO
	133
	134	buf = BytesIO()
	135	for i in range(100):
	136	buf.write(msgpack.packb(i, use_bin_type=True))
	137
	138	buf.seek(0)
	139
	140	unpacker = msgpack.Unpacker(buf, raw=False)
	141	for unpacked in unpacker:
	142	print(unpacked)
	143	```
	144
	145
	146	### Packing/unpacking of custom data type
	147
	148	It is also possible to pack/unpack custom data types. Here is an example for
	149	`datetime.datetime`.
	150
	151	```py
	152	import datetime
	153	import msgpack
	154
	155	useful_dict = {
	156	"id": 1,
	157	"created": datetime.datetime.now(),
	158	}
	159
	160	def decode_datetime(obj):
	161	if b'__datetime__' in obj:
	162	obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f")
	163	return obj
	164
	165	def encode_datetime(obj):
	166	if isinstance(obj, datetime.datetime):
	167	return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")}
	168	return obj
	169
	170
	171	packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
	172	this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False)
	173	```
	174
	175	`Unpacker`'s `object_hook` callback receives a dict; the
	176	`object_pairs_hook` callback may instead be used to receive a list of
	177	key-value pairs.
	178
	179
	180	### Extended types
	181
	182	It is also possible to pack/unpack custom data types using the ext type.
	183
	184	```pycon
	185	>>> import msgpack
	186	>>> import array
	187	>>> def default(obj):
	188	... if isinstance(obj, array.array) and obj.typecode == 'd':
	189	... return msgpack.ExtType(42, obj.tostring())
	190	... raise TypeError("Unknown type: %r" % (obj,))
	191	...
	192	>>> def ext_hook(code, data):
	193	... if code == 42:
	194	... a = array.array('d')
	195	... a.fromstring(data)
	196	... return a
	197	... return ExtType(code, data)
	198	...
	199	>>> data = array.array('d', [1.2, 3.4])
	200	>>> packed = msgpack.packb(data, default=default, use_bin_type=True)
	201	>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False)
	202	>>> data == unpacked
	203	True
	204	```
	205
	206
	207	### Advanced unpacking control
	208
	209	As an alternative to iteration, `Unpacker` objects provide `unpack`,
	210	`skip`, `read_array_header` and `read_map_header` methods. The former two
	211	read an entire message from the stream, respectively de-serialising and returning
	212	the result, or ignoring it. The latter two methods return the number of elements
	213	in the upcoming container, so that each element in an array, or key-value pair
	214	in a map, can be unpacked or skipped individually.
	215
	216
	217	## Notes
	218
	219	### string and binary type
	220
	221	Early versions of msgpack didn't distinguish string and binary types.
	222	The type for representing both string and binary types was named raw.
	223
	224	You can pack into and unpack from this old spec using `use_bin_type=False`
	225	and `raw=True` options.
	226
	227	```pycon
	228	>>> import msgpack
	229	>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True)
	230	[b'spam', b'eggs']
	231	>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False)
	232	[b'spam', 'eggs']
	233	```
	234
	235	### ext type
	236
	237	To use the ext type, pass `msgpack.ExtType` object to packer.
	238
	239	```pycon
	240	>>> import msgpack
	241	>>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy'))
	242	>>> msgpack.unpackb(packed)
	243	ExtType(code=42, data='xyzzy')
	244	```
	245
	246	You can use it with `default` and `ext_hook`. See below.
	247
	248
	249	### Security
	250
	251	To unpacking data received from unreliable source, msgpack provides
	252	two security options.
	253
	254	`max_buffer_size` (default: `10010241024`) limits the internal buffer size.
	255	It is used to limit the preallocated list size too.
	256
	257	`strict_map_key` (default: `True`) limits the type of map keys to bytes and str.
	258	While msgpack spec doesn't limit the types of the map keys,
	259	there is a risk of the hashdos.
	260	If you need to support other types for map keys, use `strict_map_key=False`.
	261
	262
	263	### Performance tips
	264
	265	CPython's GC starts when growing allocated object.
	266	This means unpacking may cause useless GC.
	267	You can use `gc.disable()` when unpacking large message.
	268
	269	List is the default sequence type of Python.
	270	But tuple is lighter than list.
	271	You can use `use_list=False` while unpacking when performance is important.
	272
	273
	274	## Development
	275
	276	### Test
	277
	278	MessagePack uses `pytest` for testing.
	279	Run test with following command:
	280
	281	```
	282	$ make test
	283	```

-336

~~README.rst~~ less more

0		======================
1		MessagePack for Python
2		======================
3
4		.. image:: https://travis-ci.org/msgpack/msgpack-python.svg?branch=master
5		:target: https://travis-ci.org/msgpack/msgpack-python
6		:alt: Build Status
7
8		.. image:: https://readthedocs.org/projects/msgpack-python/badge/?version=latest
9		:target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest
10		:alt: Documentation Status
11
12
13		What's this
14		-----------
15
16		`MessagePack <https://msgpack.org/>`_ is an efficient binary serialization format.
17		It lets you exchange data among multiple languages like JSON.
18		But it's faster and smaller.
19		This package provides CPython bindings for reading and writing MessagePack data.
20
21
22		Very important notes for existing users
23		---------------------------------------
24
25		PyPI package name
26		^^^^^^^^^^^^^^^^^
27
28		TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`.
29		Do `pip uninstall msgpack-python; pip install msgpack` instead.
30
31		Package name on PyPI was changed to msgpack from 0.5.
32		I upload transitional package (msgpack-python 0.5 which depending on msgpack)
33		for smooth transition from msgpack-python to msgpack.
34
35		Sadly, this doesn't work for upgrade install. After `pip install -U msgpack-python`,
36		msgpack is removed and `import msgpack` fail.
37
38
39		Deprecating encoding option
40		^^^^^^^^^^^^^^^^^^^^^^^^^^^^
41
42		encoding and unicode_errors options are deprecated.
43
44		In case of packer, use UTF-8 always. Storing other than UTF-8 is not recommended.
45
46		For backward compatibility, you can use ``use_bin_type=False`` and pack ``bytes``
47		object into msgpack raw type.
48
49		In case of unpacker, there is new ``raw`` option. It is ``True`` by default
50		for backward compatibility, but it is changed to ``False`` in near future.
51		You can use ``raw=False`` instead of ``encoding='utf-8'``.
52
53		Planned backward incompatible changes
54		^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
55
56		When msgpack 1.0, I planning these breaking changes:
57
58		* packer and unpacker: Remove ``encoding`` and ``unicode_errors`` option.
59		* packer: Change default of ``use_bin_type`` option from False to True.
60		* unpacker: Change default of ``raw`` option from True to False.
61		* unpacker: Reduce all ``max_xxx_len`` options for typical usage.
62		* unpacker: Remove ``write_bytes`` option from all methods.
63
64		To avoid these breaking changes breaks your application, please:
65
66		* Don't use deprecated options.
67		* Pass ``use_bin_type`` and ``raw`` options explicitly.
68		* If your application handle large (>1MB) data, specify ``max_xxx_len`` options too.
69
70
71		Install
72		-------
73
74		::
75
76		$ pip install msgpack
77
78		PyPy
79		^^^^
80
81		msgpack provides a pure Python implementation. PyPy can use this.
82
83		Windows
84		^^^^^^^
85
86		When you can't use a binary distribution, you need to install Visual Studio
87		or Windows SDK on Windows.
88		Without extension, using pure Python implementation on CPython runs slowly.
89
90		For Python 2.7, `Microsoft Visual C++ Compiler for Python 2.7 <https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_
91		is recommended solution.
92
93		For Python 3.5, `Microsoft Visual Studio 2015 <https://www.visualstudio.com/en-us/products/vs-2015-product-editions.aspx>`_
94		Community Edition or Express Edition can be used to build extension module.
95
96
97		How to use
98		----------
99
100		One-shot pack & unpack
101		^^^^^^^^^^^^^^^^^^^^^^
102
103		Use ``packb`` for packing and ``unpackb`` for unpacking.
104		msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
105		``json`` and ``pickle``.
106
107		``pack`` and ``dump`` packs to a file-like object.
108		``unpack`` and ``load`` unpacks from a file-like object.
109
110		.. code-block:: pycon
111
112		>>> import msgpack
113		>>> msgpack.packb([1, 2, 3], use_bin_type=True)
114		'\x93\x01\x02\x03'
115		>>> msgpack.unpackb(_, raw=False)
116		[1, 2, 3]
117
118		``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple:
119
120		.. code-block:: pycon
121
122		>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False)
123		(1, 2, 3)
124
125		You should always specify the ``use_list`` keyword argument for backward compatibility.
126		See performance issues relating to `use_list option`_ below.
127
128		Read the docstring for other options.
129
130
131		Streaming unpacking
132		^^^^^^^^^^^^^^^^^^^
133
134		``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one
135		stream (or from bytes provided through its ``feed`` method).
136
137		.. code-block:: python
138
139		import msgpack
140		from io import BytesIO
141
142		buf = BytesIO()
143		for i in range(100):
144		buf.write(msgpack.packb(i, use_bin_type=True))
145
146		buf.seek(0)
147
148		unpacker = msgpack.Unpacker(buf, raw=False)
149		for unpacked in unpacker:
150		print(unpacked)
151
152
153		Packing/unpacking of custom data type
154		^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
155
156		It is also possible to pack/unpack custom data types. Here is an example for
157		``datetime.datetime``.
158
159		.. code-block:: python
160
161		import datetime
162		import msgpack
163
164		useful_dict = {
165		"id": 1,
166		"created": datetime.datetime.now(),
167		}
168
169		def decode_datetime(obj):
170		if b'__datetime__' in obj:
171		obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f")
172		return obj
173
174		def encode_datetime(obj):
175		if isinstance(obj, datetime.datetime):
176		return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")}
177		return obj
178
179
180		packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
181		this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False)
182
183		``Unpacker``'s ``object_hook`` callback receives a dict; the
184		``object_pairs_hook`` callback may instead be used to receive a list of
185		key-value pairs.
186
187
188		Extended types
189		^^^^^^^^^^^^^^
190
191		It is also possible to pack/unpack custom data types using the ext type.
192
193		.. code-block:: pycon
194
195		>>> import msgpack
196		>>> import array
197		>>> def default(obj):
198		... if isinstance(obj, array.array) and obj.typecode == 'd':
199		... return msgpack.ExtType(42, obj.tostring())
200		... raise TypeError("Unknown type: %r" % (obj,))
201		...
202		>>> def ext_hook(code, data):
203		... if code == 42:
204		... a = array.array('d')
205		... a.fromstring(data)
206		... return a
207		... return ExtType(code, data)
208		...
209		>>> data = array.array('d', [1.2, 3.4])
210		>>> packed = msgpack.packb(data, default=default, use_bin_type=True)
211		>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False)
212		>>> data == unpacked
213		True
214
215
216		Advanced unpacking control
217		^^^^^^^^^^^^^^^^^^^^^^^^^^
218
219		As an alternative to iteration, ``Unpacker`` objects provide ``unpack``,
220		``skip``, ``read_array_header`` and ``read_map_header`` methods. The former two
221		read an entire message from the stream, respectively de-serialising and returning
222		the result, or ignoring it. The latter two methods return the number of elements
223		in the upcoming container, so that each element in an array, or key-value pair
224		in a map, can be unpacked or skipped individually.
225
226		Each of these methods may optionally write the packed data it reads to a
227		callback function:
228
229		.. code-block:: python
230
231		from io import BytesIO
232
233		def distribute(unpacker, get_worker):
234		nelems = unpacker.read_map_header()
235		for i in range(nelems):
236		# Select a worker for the given key
237		key = unpacker.unpack()
238		worker = get_worker(key)
239
240		# Send the value as a packed message to worker
241		bytestream = BytesIO()
242		unpacker.skip(bytestream.write)
243		worker.send(bytestream.getvalue())
244
245
246		Notes
247		-----
248
249		string and binary type
250		^^^^^^^^^^^^^^^^^^^^^^
251
252		Early versions of msgpack didn't distinguish string and binary types (like Python 1).
253		The type for representing both string and binary types was named raw.
254
255		For backward compatibility reasons, msgpack-python will still default all
256		strings to byte strings, unless you specify the ``use_bin_type=True`` option in
257		the packer. If you do so, it will use a non-standard type called bin to
258		serialize byte arrays, and raw becomes to mean str. If you want to
259		distinguish bin and raw in the unpacker, specify ``raw=False``.
260
261		Note that Python 2 defaults to byte-arrays over Unicode strings:
262
263		.. code-block:: pycon
264
265		>>> import msgpack
266		>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
267		['spam', 'eggs']
268		>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
269		raw=False)
270		['spam', u'eggs']
271
272		This is the same code in Python 3 (same behaviour, but Python 3 has a
273		different default):
274
275		.. code-block:: pycon
276
277		>>> import msgpack
278		>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
279		[b'spam', b'eggs']
280		>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
281		raw=False)
282		[b'spam', 'eggs']
283
284
285		ext type
286		^^^^^^^^
287
288		To use the ext type, pass ``msgpack.ExtType`` object to packer.
289
290		.. code-block:: pycon
291
292		>>> import msgpack
293		>>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy'))
294		>>> msgpack.unpackb(packed)
295		ExtType(code=42, data='xyzzy')
296
297		You can use it with ``default`` and ``ext_hook``. See below.
298
299
300		Note about performance
301		----------------------
302
303		GC
304		^^
305
306		CPython's GC starts when growing allocated object.
307		This means unpacking may cause useless GC.
308		You can use ``gc.disable()`` when unpacking large message.
309
310		use_list option
311		^^^^^^^^^^^^^^^
312
313		List is the default sequence type of Python.
314		But tuple is lighter than list.
315		You can use ``use_list=False`` while unpacking when performance is important.
316
317		Python's dict can't use list as key and MessagePack allows array for key of mapping.
318		``use_list=False`` allows unpacking such message.
319		Another way to unpacking such object is using ``object_pairs_hook``.
320
321
322		Development
323		-----------
324
325		Test
326		^^^^
327
328		MessagePack uses `pytest` for testing.
329		Run test with following command:
330
331		$ make test
332
333
334		..
335		vim: filetype=rst

-0

appveyor.yml less more

28	28	- ci\\runtests.bat
29	29	- set PYTHON="C:\\Python37-x64"
30	30	- ci\\runtests.bat
	31	- set PYTHON="C:\\Python38"
	32	- ci\\runtests.bat
	33	- set PYTHON="C:\\Python38-x64"
	34	- ci\\runtests.bat
31	35
32	36	after_test:
33	37	# This step builds your wheels.

+12

-8

benchmark/benchmark.py less more

0	0	from msgpack import fallback
	1
1	2	try:
2	3	from msgpack import _unpacker, _packer
	4
3	5	has_ext = True
4	6	except ImportError:
5	7	has_ext = False

8	10
9	11	def profile(name, func):
10	12	times = timeit.repeat(func, number=1000, repeat=4)
11		times = ', '.join(["%8f" % t for t in times])
	13	times = ", ".join(["%8f" % t for t in times])
12	14	print("%-30s %40s" % (name, times))
13	15
14	16

17	19	packer = _packer.Packer()
18	20	profile("packing %s (ext)" % name, lambda: packer.pack(data))
19	21	packer = fallback.Packer()
20		profile('packing %s (fallback)' % name, lambda: packer.pack(data))
	22	profile("packing %s (fallback)" % name, lambda: packer.pack(data))
21	23
22	24	data = packer.pack(data)
23	25	if has_ext:
24		profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data))
25		profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data))
	26	profile("unpacking %s (ext)" % name, lambda: _unpacker.unpackb(data))
	27	profile("unpacking %s (fallback)" % name, lambda: fallback.unpackb(data))
	28
26	29
27	30	def main():
28		simple("integers", [7]*10000)
29		simple("bytes", [b'x'n for n in range(100)]10)
30		simple("lists", [[]]*10000)
31		simple("dicts", [{}]*10000)
	31	simple("integers", [7] * 10000)
	32	simple("bytes", [b"x" * n for n in range(100)] * 10)
	33	simple("lists", [[]] * 10000)
	34	simple("dicts", [{}] * 10000)
	35
32	36
33	37	main()

-1

ci/runtests.bat less more

1	1	%PYTHON%\python.exe setup.py build_ext -i
2	2	%PYTHON%\python.exe setup.py install
3	3	%PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))"
4		%PYTHON%\python.exe -c "from msgpack import _packer, _unpacker"
	4	%PYTHON%\python.exe -c "from msgpack import _cmsgpack"
5	5	%PYTHON%\python.exe setup.py bdist_wheel
6	6	%PYTHON%\python.exe -m pytest -v test
7	7	SET EL=%ERRORLEVEL%

-0

ci/runtests.sh less more

	0	#!/bin/bash
	1	set -ex
	2	${PYTHON} -VV
	3	${PYTHON} -m pip install setuptools wheel pytest
	4	${PYTHON} setup.py build_ext -if
	5	${PYTHON} -c "from msgpack import _cmsgpack"
	6	${PYTHON} setup.py bdist_wheel
	7	${PYTHON} -m pytest -v test

-1

docker/buildwheel.sh less more

0	0	#!/bin/bash
	1	DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
	2	source "$DOCKER_DIR/shared.env"
	3
1	4	set -e -x
2	5
3	6	ARCH=`uname -p`
4	7	echo "arch=$ARCH"
5	8
6		for V in cp37-cp37m cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
	9	for V in "${PYTHON_VERSIONS[@]}"; do
7	10	PYBIN=/opt/python/$V/bin
8	11	rm -rf build/ # Avoid lib build by narrow Python is used by wide python
9	12	$PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH}

-1

docker/runtests.sh less more

0	0	#!/bin/bash
	1	DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
	2	source "$DOCKER_DIR/shared.env"
	3
1	4	set -e -x
2	5
3		for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
	6	for V in "${PYTHON_VERSIONS[@]}"; do
4	7	PYBIN=/opt/python/$V/bin
5	8	$PYBIN/python setup.py install
6	9	rm -rf build/ # Avoid lib build by narrow Python is used by wide python

-0

docker/shared.env less more

	0	PYTHON_VERSIONS=(
	1	cp38-cp38
	2	cp37-cp37m
	3	cp36-cp36m
	4	cp35-cp35m
	5	)

-0

docs/api.rst less more

26	26
27	27	.. autoclass:: ExtType
28	28
	29	.. autoclass:: Timestamp
	30	:members:
	31	:special-members: __init__
	32
29	33	exceptions
30	34	----------
31	35

+82

-82

docs/conf.py less more

15	15	# If extensions (or modules to document with autodoc) are in another directory,
16	16	# add these directories to sys.path here. If the directory is relative to the
17	17	# documentation root, use os.path.abspath to make it absolute, like shown here.
18		#sys.path.insert(0, os.path.abspath('.'))
	18	# sys.path.insert(0, os.path.abspath('.'))
19	19
20	20	# -- General configuration -----------------------------------------------------
21	21
22	22	# If your documentation needs a minimal Sphinx version, state it here.
23		#needs_sphinx = '1.0'
	23	# needs_sphinx = '1.0'
24	24
25	25	# Add any Sphinx extension module names here, as strings. They can be extensions
26	26	# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
27		extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
	27	extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode"]
28	28
29	29	# Add any paths that contain templates here, relative to this directory.
30		templates_path = ['_templates']
	30	templates_path = ["_templates"]
31	31
32	32	# The suffix of source filenames.
33		source_suffix = '.rst'
	33	source_suffix = ".rst"
34	34
35	35	# The encoding of source files.
36		#source_encoding = 'utf-8-sig'
	36	# source_encoding = 'utf-8-sig'
37	37
38	38	# The master toctree document.
39		master_doc = 'index'
	39	master_doc = "index"
40	40
41	41	# General information about the project.
42		project = u'msgpack'
43		copyright = u'2013, INADA Naoki'
	42	project = u"msgpack"
	43	copyright = u"2013, INADA Naoki"
44	44
45	45	# The version info for the project you're documenting, acts as replacement for
46	46	# \|version\| and \|release\|, also used in various other places throughout the

48	48	#
49	49	# The short X.Y version.
50	50	# The full version, including alpha/beta/rc tags.
51		version = release = '0.5'
	51	version = release = "0.5"
52	52
53	53	# The language for content autogenerated by Sphinx. Refer to documentation
54	54	# for a list of supported languages.
55		#language = None
	55	# language = None
56	56
57	57	# There are two options for replacing \|today\|: either, you set today to some
58	58	# non-false value, then it is used:
59		#today = ''
	59	# today = ''
60	60	# Else, today_fmt is used as the format for a strftime call.
61		#today_fmt = '%B %d, %Y'
	61	# today_fmt = '%B %d, %Y'
62	62	today_fmt = "%Y-%m-%d"
63	63
64	64	# List of patterns, relative to source directory, that match files and
65	65	# directories to ignore when looking for source files.
66		exclude_patterns = ['_build']
	66	exclude_patterns = ["_build"]
67	67
68	68	# The reST default role (used for this markup: `text`) to use for all documents.
69		#default_role = None
	69	# default_role = None
70	70
71	71	# If true, '()' will be appended to :func: etc. cross-reference text.
72		#add_function_parentheses = True
	72	# add_function_parentheses = True
73	73
74	74	# If true, the current module name will be prepended to all description
75	75	# unit titles (such as .. function::).
76		#add_module_names = True
	76	# add_module_names = True
77	77
78	78	# If true, sectionauthor and moduleauthor directives will be shown in the
79	79	# output. They are ignored by default.
80		#show_authors = False
	80	# show_authors = False
81	81
82	82	# The name of the Pygments (syntax highlighting) style to use.
83		pygments_style = 'sphinx'
	83	pygments_style = "sphinx"
84	84
85	85	# A list of ignored prefixes for module index sorting.
86		#modindex_common_prefix = []
	86	# modindex_common_prefix = []
87	87
88	88
89	89	# -- Options for HTML output ---------------------------------------------------
90	90
91	91	# The theme to use for HTML and HTML Help pages. See the documentation for
92	92	# a list of builtin themes.
93		html_theme = 'sphinxdoc'
	93	html_theme = "sphinxdoc"
94	94
95	95	# Theme options are theme-specific and customize the look and feel of a theme
96	96	# further. For a list of options available for each theme, see the
97	97	# documentation.
98		#html_theme_options = {}
	98	# html_theme_options = {}
99	99
100	100	# Add any paths that contain custom themes here, relative to this directory.
101		#html_theme_path = []
	101	# html_theme_path = []
102	102
103	103	# The name for this set of Sphinx documents. If None, it defaults to
104	104	# "<project> v<release> documentation".
105		#html_title = None
	105	# html_title = None
106	106
107	107	# A shorter title for the navigation bar. Default is the same as html_title.
108		#html_short_title = None
	108	# html_short_title = None
109	109
110	110	# The name of an image file (relative to this directory) to place at the top
111	111	# of the sidebar.
112		#html_logo = None
	112	# html_logo = None
113	113
114	114	# The name of an image file (within the static path) to use as favicon of the
115	115	# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
116	116	# pixels large.
117		#html_favicon = None
	117	# html_favicon = None
118	118
119	119	# Add any paths that contain custom static files (such as style sheets) here,
120	120	# relative to this directory. They are copied after the builtin static files,
121	121	# so a file named "default.css" will overwrite the builtin "default.css".
122		html_static_path = ['_static']
	122	html_static_path = ["_static"]
123	123
124	124	# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
125	125	# using the given strftime format.
126		#html_last_updated_fmt = '%b %d, %Y'
	126	# html_last_updated_fmt = '%b %d, %Y'
127	127
128	128	# If true, SmartyPants will be used to convert quotes and dashes to
129	129	# typographically correct entities.
130		#html_use_smartypants = True
	130	# html_use_smartypants = True
131	131
132	132	# Custom sidebar templates, maps document names to template names.
133		#html_sidebars = {}
	133	# html_sidebars = {}
134	134
135	135	# Additional templates that should be rendered to pages, maps page names to
136	136	# template names.
137		#html_additional_pages = {}
	137	# html_additional_pages = {}
138	138
139	139	# If false, no module index is generated.
140		#html_domain_indices = True
	140	# html_domain_indices = True
141	141
142	142	# If false, no index is generated.
143		#html_use_index = True
	143	# html_use_index = True
144	144
145	145	# If true, the index is split into individual pages for each letter.
146		#html_split_index = False
	146	# html_split_index = False
147	147
148	148	# If true, links to the reST sources are added to the pages.
149		#html_show_sourcelink = True
	149	# html_show_sourcelink = True
150	150
151	151	# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
152		#html_show_sphinx = True
	152	# html_show_sphinx = True
153	153
154	154	# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
155		#html_show_copyright = True
	155	# html_show_copyright = True
156	156
157	157	# If true, an OpenSearch description file will be output, and all pages will
158	158	# contain a <link> tag referring to it. The value of this option must be the
159	159	# base URL from which the finished HTML is served.
160		#html_use_opensearch = ''
	160	# html_use_opensearch = ''
161	161
162	162	# This is the file name suffix for HTML files (e.g. ".xhtml").
163		#html_file_suffix = None
	163	# html_file_suffix = None
164	164
165	165	# Output file base name for HTML help builder.
166		htmlhelp_basename = 'msgpackdoc'
	166	htmlhelp_basename = "msgpackdoc"
167	167
168	168
169	169	# -- Options for LaTeX output --------------------------------------------------
170	170
171	171	latex_elements = {
172		# The paper size ('letterpaper' or 'a4paper').
173		#'papersize': 'letterpaper',
174
175		# The font size ('10pt', '11pt' or '12pt').
176		#'pointsize': '10pt',
177
178		# Additional stuff for the LaTeX preamble.
179		#'preamble': '',
	172	# The paper size ('letterpaper' or 'a4paper').
	173	#'papersize': 'letterpaper',
	174	# The font size ('10pt', '11pt' or '12pt').
	175	#'pointsize': '10pt',
	176	# Additional stuff for the LaTeX preamble.
	177	#'preamble': '',
180	178	}
181	179
182	180	# Grouping the document tree into LaTeX files. List of tuples
183	181	# (source start file, target name, title, author, documentclass [howto/manual]).
184	182	latex_documents = [
185		('index', 'msgpack.tex', u'msgpack Documentation',
186		u'Author', 'manual'),
	183	("index", "msgpack.tex", u"msgpack Documentation", u"Author", "manual"),
187	184	]
188	185
189	186	# The name of an image file (relative to this directory) to place at the top of
190	187	# the title page.
191		#latex_logo = None
	188	# latex_logo = None
192	189
193	190	# For "manual" documents, if this is true, then toplevel headings are parts,
194	191	# not chapters.
195		#latex_use_parts = False
	192	# latex_use_parts = False
196	193
197	194	# If true, show page references after internal links.
198		#latex_show_pagerefs = False
	195	# latex_show_pagerefs = False
199	196
200	197	# If true, show URL addresses after external links.
201		#latex_show_urls = False
	198	# latex_show_urls = False
202	199
203	200	# Documents to append as an appendix to all manuals.
204		#latex_appendices = []
	201	# latex_appendices = []
205	202
206	203	# If false, no module index is generated.
207		#latex_domain_indices = True
	204	# latex_domain_indices = True
208	205
209	206
210	207	# -- Options for manual page output --------------------------------------------
211	208
212	209	# One entry per manual page. List of tuples
213	210	# (source start file, name, description, authors, manual section).
214		man_pages = [
215		('index', 'msgpack', u'msgpack Documentation',
216		[u'Author'], 1)
217		]
	211	man_pages = [("index", "msgpack", u"msgpack Documentation", [u"Author"], 1)]
218	212
219	213	# If true, show URL addresses after external links.
220		#man_show_urls = False
	214	# man_show_urls = False
221	215
222	216
223	217	# -- Options for Texinfo output ------------------------------------------------

226	220	# (source start file, target name, title, author,
227	221	# dir menu entry, description, category)
228	222	texinfo_documents = [
229		('index', 'msgpack', u'msgpack Documentation',
230		u'Author', 'msgpack', 'One line description of project.',
231		'Miscellaneous'),
	223	(
	224	"index",
	225	"msgpack",
	226	u"msgpack Documentation",
	227	u"Author",
	228	"msgpack",
	229	"One line description of project.",
	230	"Miscellaneous",
	231	),
232	232	]
233	233
234	234	# Documents to append as an appendix to all manuals.
235		#texinfo_appendices = []
	235	# texinfo_appendices = []
236	236
237	237	# If false, no module index is generated.
238		#texinfo_domain_indices = True
	238	# texinfo_domain_indices = True
239	239
240	240	# How to display URL addresses: 'footnote', 'no', or 'inline'.
241		#texinfo_show_urls = 'footnote'
	241	# texinfo_show_urls = 'footnote'
242	242
243	243
244	244	# -- Options for Epub output ---------------------------------------------------
245	245
246	246	# Bibliographic Dublin Core info.
247		epub_title = u'msgpack'
248		epub_author = u'Author'
249		epub_publisher = u'Author'
250		epub_copyright = u'2013, Author'
	247	epub_title = u"msgpack"
	248	epub_author = u"Author"
	249	epub_publisher = u"Author"
	250	epub_copyright = u"2013, Author"
251	251
252	252	# The language of the text. It defaults to the language option
253	253	# or en if the language is not set.
254		#epub_language = ''
	254	# epub_language = ''
255	255
256	256	# The scheme of the identifier. Typical schemes are ISBN or URL.
257		#epub_scheme = ''
	257	# epub_scheme = ''
258	258
259	259	# The unique identifier of the text. This can be a ISBN number
260	260	# or the project homepage.
261		#epub_identifier = ''
	261	# epub_identifier = ''
262	262
263	263	# A unique identification for the text.
264		#epub_uid = ''
	264	# epub_uid = ''
265	265
266	266	# A tuple containing the cover image and cover page html template filenames.
267		#epub_cover = ()
	267	# epub_cover = ()
268	268
269	269	# HTML files that should be inserted before the pages created by sphinx.
270	270	# The format is a list of tuples containing the path and title.
271		#epub_pre_files = []
	271	# epub_pre_files = []
272	272
273	273	# HTML files shat should be inserted after the pages created by sphinx.
274	274	# The format is a list of tuples containing the path and title.
275		#epub_post_files = []
	275	# epub_post_files = []
276	276
277	277	# A list of files that should not be packed into the epub file.
278		#epub_exclude_files = []
	278	# epub_exclude_files = []
279	279
280	280	# The depth of the table of contents in toc.ncx.
281		#epub_tocdepth = 3
	281	# epub_tocdepth = 3
282	282
283	283	# Allow duplicate toc entries.
284		#epub_tocdup = True
	284	# epub_tocdup = True

-15

msgpack/__init__.py less more

0	0	# coding: utf-8
1	1	from ._version import version
2	2	from .exceptions import *
	3	from .ext import ExtType, Timestamp
3	4
4		from collections import namedtuple
	5	import os
	6	import sys
5	7
6	8
7		class ExtType(namedtuple('ExtType', 'code data')):
8		"""ExtType represents ext type in msgpack."""
9		def __new__(cls, code, data):
10		if not isinstance(code, int):
11		raise TypeError("code must be int")
12		if not isinstance(data, bytes):
13		raise TypeError("data must be bytes")
14		if not 0 <= code <= 127:
15		raise ValueError("code must be 0~127")
16		return super(ExtType, cls).__new__(cls, code, data)
17
18
19		import os
20		if os.environ.get('MSGPACK_PUREPYTHON'):
	9	if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2:
21	10	from .fallback import Packer, unpackb, Unpacker
22	11	else:
23	12	try:

-0

msgpack/_cmsgpack.pyx less more

0	0	# coding: utf-8
1	1	#cython: embedsignature=True, c_string_encoding=ascii, language_level=3
	2	from cpython.datetime cimport import_datetime, datetime_new
	3	import_datetime()
	4
	5	import datetime
	6	cdef object utc = datetime.timezone.utc
	7	cdef object epoch = datetime_new(1970, 1, 1, 0, 0, 0, 0, tz=utc)
	8
2	9	include "_packer.pyx"
3	10	include "_unpacker.pyx"

+36

-34

msgpack/_packer.pyx less more

1	1
2	2	from cpython cimport *
3	3	from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact
	4	from cpython.datetime cimport (
	5	PyDateTime_CheckExact, PyDelta_CheckExact,
	6	datetime_tzinfo, timedelta_days, timedelta_seconds, timedelta_microseconds,
	7	)
4	8
5	9	cdef ExtType
6
7		from . import ExtType
	10	cdef Timestamp
	11
	12	from .ext import ExtType, Timestamp
8	13
9	14
10	15	cdef extern from "Python.h":

35	40	int msgpack_pack_bin(msgpack_packer* pk, size_t l)
36	41	int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
37	42	int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
	43	int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds);
38	44	int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)
39	45
40	46	cdef extern from "buff_converter.h":

77	83
78	84	:param bool use_bin_type:
79	85	Use bin type introduced in msgpack spec 2.0 for bytes.
80		It also enables str8 type for unicode.
81		Current default value is false, but it will be changed to true
82		in future version. You should specify it explicitly.
	86	It also enables str8 type for unicode. (default: True)
83	87
84	88	:param bool strict_types:
85	89	If set to true, types will be checked to be exact. Derived classes

90	94	for python types.
91	95
92	96	:param str unicode_errors:
93		Error handler for encoding unicode. (default: 'strict')
94
95		:param str encoding:
96		(deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
	97	The error handler for encoding unicode. (default: 'strict')
	98	DO NOT USE THIS!! This option is kept for very specific usage.
97	99	"""
98	100	cdef msgpack_packer pk
99	101	cdef object _default
100		cdef object _bencoding
101	102	cdef object _berrors
102		cdef const char *encoding
103	103	cdef const char *unicode_errors
104	104	cdef bint strict_types
105		cdef bool use_float
	105	cdef bint use_float
106	106	cdef bint autoreset
	107	cdef bint datetime
107	108
108	109	def __cinit__(self):
109	110	cdef int buf_size = 1024*1024

113	114	self.pk.buf_size = buf_size
114	115	self.pk.length = 0
115	116
116		def __init__(self, default=None, encoding=None, unicode_errors=None,
117		bint use_single_float=False, bint autoreset=True, bint use_bin_type=False,
118		bint strict_types=False):
119		if encoding is not None:
120		PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1)
	117	def __init__(self, *, default=None,
	118	bint use_single_float=False, bint autoreset=True, bint use_bin_type=True,
	119	bint strict_types=False, bint datetime=False, unicode_errors=None):
121	120	self.use_float = use_single_float
122	121	self.strict_types = strict_types
123	122	self.autoreset = autoreset
	123	self.datetime = datetime
124	124	self.pk.use_bin_type = use_bin_type
125	125	if default is not None:
126	126	if not PyCallable_Check(default):
127	127	raise TypeError("default must be a callable.")
128	128	self._default = default
129	129
130		self._bencoding = encoding
131		if encoding is None:
132		if PY_MAJOR_VERSION < 3:
133		self.encoding = 'utf-8'
134		else:
135		self.encoding = NULL
136		else:
137		self.encoding = self._bencoding
138
139	130	self._berrors = unicode_errors
140	131	if unicode_errors is None:
141	132	self.unicode_errors = NULL

149	140	cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
150	141	cdef long long llval
151	142	cdef unsigned long long ullval
	143	cdef unsigned long ulval
152	144	cdef long longval
153	145	cdef float fval
154	146	cdef double dval

166	158	while True:
167	159	if o is None:
168	160	ret = msgpack_pack_nil(&self.pk)
169		elif PyBool_Check(o) if strict_types else isinstance(o, bool):
170		if o:
171		ret = msgpack_pack_true(&self.pk)
172		else:
173		ret = msgpack_pack_false(&self.pk)
	161	elif o is True:
	162	ret = msgpack_pack_true(&self.pk)
	163	elif o is False:
	164	ret = msgpack_pack_false(&self.pk)
174	165	elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o):
175	166	# PyInt_Check(long) is True for Python 3.
176	167	# So we should test long before int.

207	198	if ret == 0:
208	199	ret = msgpack_pack_raw_body(&self.pk, rawval, L)
209	200	elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
210		if self.encoding == NULL and self.unicode_errors == NULL:
	201	if self.unicode_errors == NULL:
211	202	ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
212	203	if ret == -2:
213	204	raise ValueError("unicode string is too large")
214	205	else:
215		o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
	206	o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors)
216	207	L = Py_SIZE(o)
217	208	if L > ITEM_LIMIT:
218	209	raise ValueError("unicode string is too large")

252	243	raise ValueError("EXT data is too large")
253	244	ret = msgpack_pack_ext(&self.pk, longval, L)
254	245	ret = msgpack_pack_raw_body(&self.pk, rawval, L)
	246	elif type(o) is Timestamp:
	247	llval = o.seconds
	248	ulval = o.nanoseconds
	249	ret = msgpack_pack_timestamp(&self.pk, llval, ulval)
255	250	elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
256	251	L = Py_SIZE(o)
257	252	if L > ITEM_LIMIT:

272	267	if ret == 0:
273	268	ret = msgpack_pack_raw_body(&self.pk, <char*>view.buf, L)
274	269	PyBuffer_Release(&view);
	270	elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None:
	271	delta = o - epoch
	272	if not PyDelta_CheckExact(delta):
	273	raise ValueError("failed to calculate delta")
	274	llval = timedelta_days(delta) * <long long>(246060) + timedelta_seconds(delta)
	275	ulval = timedelta_microseconds(delta) * 1000
	276	ret = msgpack_pack_timestamp(&self.pk, llval, ulval)
275	277	elif not default_used and self._default:
276	278	o = self._default(o)
277	279	default_used = 1

350	352	def reset(self):
351	353	"""Reset internal buffer.
352	354
353		This method is usaful only when autoreset=False.
	355	This method is useful only when autoreset=False.
354	356	"""
355	357	self.pk.length = 0
356	358

+74

-108

msgpack/_unpacker.pyx less more

0	0	# coding: utf-8
1	1
2	2	from cpython cimport *
3
4	3	cdef extern from "Python.h":
5	4	ctypedef struct PyObject
6	5	cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1

18	17	FormatError,
19	18	StackError,
20	19	)
21		from . import ExtType
	20	from .ext import ExtType, Timestamp
	21
	22	cdef object giga = 1_000_000_000
22	23
23	24
24	25	cdef extern from "unpack.h":

27	28	bint raw
28	29	bint has_pairs_hook # call object_hook with k-v pairs
29	30	bint strict_map_key
	31	int timestamp
30	32	PyObject* object_hook
31	33	PyObject* list_hook
32	34	PyObject* ext_hook
33		char *encoding
	35	PyObject* timestamp_t
	36	PyObject *giga;
	37	PyObject *utc;
34	38	char *unicode_errors
35	39	Py_ssize_t max_str_len
36	40	Py_ssize_t max_bin_len

56	60	cdef inline init_ctx(unpack_context *ctx,
57	61	object object_hook, object object_pairs_hook,
58	62	object list_hook, object ext_hook,
59		bint use_list, bint raw, bint strict_map_key,
60		const char* encoding, const char* unicode_errors,
	63	bint use_list, bint raw, int timestamp,
	64	bint strict_map_key,
	65	const char* unicode_errors,
61	66	Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
62	67	Py_ssize_t max_array_len, Py_ssize_t max_map_len,
63	68	Py_ssize_t max_ext_len):

98	103	raise TypeError("ext_hook must be a callable.")
99	104	ctx.user.ext_hook = <PyObject*>ext_hook
100	105
101		ctx.user.encoding = encoding
	106	if timestamp < 0 or 3 < timestamp:
	107	raise ValueError("timestamp must be 0..3")
	108
	109	# Add Timestamp type to the user object so it may be used in unpack.h
	110	ctx.user.timestamp = timestamp
	111	ctx.user.timestamp_t = <PyObject*>Timestamp
	112	ctx.user.giga = <PyObject*>giga
	113	ctx.user.utc = <PyObject*>utc
102	114	ctx.user.unicode_errors = unicode_errors
103	115
104	116	def default_read_extended_type(typecode, data):

107	119	cdef inline int get_data_from_buffer(object obj,
108	120	Py_buffer *view,
109	121	char **buf,
110		Py_ssize_t *buffer_len,
111		int *new_protocol) except 0:
	122	Py_ssize_t *buffer_len) except 0:
112	123	cdef object contiguous
113	124	cdef Py_buffer tmp
114		if PyObject_CheckBuffer(obj):
115		new_protocol[0] = 1
116		if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1:
117		raise
118		if view.itemsize != 1:
119		PyBuffer_Release(view)
120		raise BufferError("cannot unpack from multi-byte object")
121		if PyBuffer_IsContiguous(view, b'A') == 0:
122		PyBuffer_Release(view)
123		# create a contiguous copy and get buffer
124		contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C')
125		PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE)
126		# view must hold the only reference to contiguous,
127		# so memory is freed when view is released
128		Py_DECREF(contiguous)
129		buffer_len[0] = view.len
130		buf[0] = <char*> view.buf
131		return 1
132		else:
133		new_protocol[0] = 0
134		if PyObject_AsReadBuffer(obj, <const void**> buf, buffer_len) == -1:
135		raise BufferError("could not get memoryview")
136		PyErr_WarnEx(RuntimeWarning,
137		"using old buffer interface to unpack %s; "
138		"this leads to unpacking errors if slicing is used and "
139		"will be removed in a future version" % type(obj),
140		1)
141		return 1
142
143		def unpackb(object packed, object object_hook=None, object list_hook=None,
144		bint use_list=True, bint raw=True, bint strict_map_key=False,
145		encoding=None, unicode_errors=None,
	125	if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1:
	126	raise
	127	if view.itemsize != 1:
	128	PyBuffer_Release(view)
	129	raise BufferError("cannot unpack from multi-byte object")
	130	if PyBuffer_IsContiguous(view, b'A') == 0:
	131	PyBuffer_Release(view)
	132	# create a contiguous copy and get buffer
	133	contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C')
	134	PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE)
	135	# view must hold the only reference to contiguous,
	136	# so memory is freed when view is released
	137	Py_DECREF(contiguous)
	138	buffer_len[0] = view.len
	139	buf[0] = <char*> view.buf
	140	return 1
	141
	142
	143	def unpackb(object packed, *, object object_hook=None, object list_hook=None,
	144	bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True,
	145	unicode_errors=None,
146	146	object_pairs_hook=None, ext_hook=ExtType,
147	147	Py_ssize_t max_str_len=-1,
148	148	Py_ssize_t max_bin_len=-1,

169	169	cdef Py_buffer view
170	170	cdef char* buf = NULL
171	171	cdef Py_ssize_t buf_len
172		cdef const char* cenc = NULL
173	172	cdef const char* cerr = NULL
174		cdef int new_protocol = 0
175
176		if encoding is not None:
177		PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
178		cenc = encoding
179	173
180	174	if unicode_errors is not None:
181	175	cerr = unicode_errors
182	176
183		get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
	177	get_data_from_buffer(packed, &view, &buf, &buf_len)
184	178
185	179	if max_str_len == -1:
186	180	max_str_len = buf_len

195	189
196	190	try:
197	191	init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
198		use_list, raw, strict_map_key, cenc, cerr,
	192	use_list, raw, timestamp, strict_map_key, cerr,
199	193	max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
200	194	ret = unpack_construct(&ctx, buf, buf_len, &off)
201	195	finally:
202		if new_protocol:
203		PyBuffer_Release(&view);
	196	PyBuffer_Release(&view);
204	197
205	198	if ret == 1:
206	199	obj = unpack_data(&ctx)

217	210	raise ValueError("Unpack failed: error = %d" % (ret,))
218	211
219	212
220		def unpack(object stream, **kwargs):
221		PyErr_WarnEx(
222		DeprecationWarning,
223		"Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1)
224		data = stream.read()
225		return unpackb(data, **kwargs)
226
227
228	213	cdef class Unpacker(object):
229	214	"""Streaming unpacker.
230	215

242	227	Otherwise, unpack to Python tuple. (default: True)
243	228
244	229	:param bool raw:
245		If true, unpack msgpack raw to Python bytes (default).
246		Otherwise, unpack to Python str (or unicode on Python 2) by decoding
247		with UTF-8 encoding (recommended).
248		Currently, the default is true, but it will be changed to false in
249		near future. So you must specify it explicitly for keeping backward
250		compatibility.
251
252		encoding option which is deprecated overrides this option.
	230	If true, unpack msgpack raw to Python bytes.
	231	Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
253	232
254	233	:param bool strict_map_key:
255		If true, only str or bytes are accepted for map (dict) keys.
256		It's False by default for backward-compatibility.
257		But it will be True from msgpack 1.0.
	234	If true (default), only str or bytes are accepted for map (dict) keys.
258	235
259	236	:param callable object_hook:
260	237	When specified, it should be callable.

267	244	(See also simplejson)
268	245
269	246	:param int max_buffer_size:
270		Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
	247	Limits size of data waiting unpacked. 0 means system's INT_MAX.
	248	The default value is 10010241024 (100MiB).
271	249	Raises `BufferFull` exception when it is insufficient.
272	250	You should set this parameter when unpacking data from untrusted source.
273	251
274	252	:param int max_str_len:
275	253	Deprecated, use max_buffer_size instead.
276		Limits max length of str. (default: max_buffer_size or 1024*1024)
	254	Limits max length of str. (default: max_buffer_size)
277	255
278	256	:param int max_bin_len:
279	257	Deprecated, use max_buffer_size instead.
280		Limits max length of bin. (default: max_buffer_size or 1024*1024)
	258	Limits max length of bin. (default: max_buffer_size)
281	259
282	260	:param int max_array_len:
283		Limits max length of array. (default: max_buffer_size or 128*1024)
	261	Limits max length of array. (default: max_buffer_size)
284	262
285	263	:param int max_map_len:
286		Limits max length of map. (default: max_buffer_size//2 or 32*1024)
	264	Limits max length of map. (default: max_buffer_size//2)
287	265
288	266	:param int max_ext_len:
289	267	Deprecated, use max_buffer_size instead.
290		Limits max size of ext type. (default: max_buffer_size or 1024*1024)
291
292		:param str encoding:
293		Deprecated, use ``raw=False`` instead.
294		Encoding used for decoding msgpack raw.
295		If it is None (default), msgpack raw is deserialized to Python bytes.
	268	Limits max size of ext type. (default: max_buffer_size)
296	269
297	270	:param str unicode_errors:
298	271	Error handler used for decoding str type. (default: `'strict'`)

300	273
301	274	Example of streaming deserialize from file-like object::
302	275
303		unpacker = Unpacker(file_like, raw=False, max_buffer_size=1010241024)
	276	unpacker = Unpacker(file_like)
304	277	for o in unpacker:
305	278	process(o)
306	279
307	280	Example of streaming deserialize from socket::
308	281
309		unpacker = Unpacker(raw=False, max_buffer_size=1010241024)
	282	unpacker = Unpacker()
310	283	while True:
311	284	buf = sock.recv(1024**2)
312	285	if not buf:

329	302	cdef Py_ssize_t read_size
330	303	# To maintain refcnt.
331	304	cdef object object_hook, object_pairs_hook, list_hook, ext_hook
332		cdef object encoding, unicode_errors
	305	cdef object unicode_errors
333	306	cdef Py_ssize_t max_buffer_size
334	307	cdef uint64_t stream_offset
335	308

340	313	PyMem_Free(self.buf)
341	314	self.buf = NULL
342	315
343		def __init__(self, file_like=None, Py_ssize_t read_size=0,
344		bint use_list=True, bint raw=True, bint strict_map_key=False,
	316	def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
	317	bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True,
345	318	object object_hook=None, object object_pairs_hook=None, object list_hook=None,
346		encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0,
	319	unicode_errors=None, Py_ssize_t max_buffer_size=10010241024,
347	320	object ext_hook=ExtType,
348	321	Py_ssize_t max_str_len=-1,
349	322	Py_ssize_t max_bin_len=-1,
350	323	Py_ssize_t max_array_len=-1,
351	324	Py_ssize_t max_map_len=-1,
352	325	Py_ssize_t max_ext_len=-1):
353		cdef const char *cenc=NULL,
354	326	cdef const char *cerr=NULL
355	327
356	328	self.object_hook = object_hook

364	336	if not PyCallable_Check(self.file_like_read):
365	337	raise TypeError("`file_like.read` must be a callable.")
366	338
367		if max_str_len == -1:
368		max_str_len = max_buffer_size or 1024*1024
369		if max_bin_len == -1:
370		max_bin_len = max_buffer_size or 1024*1024
371		if max_array_len == -1:
372		max_array_len = max_buffer_size or 128*1024
373		if max_map_len == -1:
374		max_map_len = max_buffer_size//2 or 32*1024
375		if max_ext_len == -1:
376		max_ext_len = max_buffer_size or 1024*1024
377
378	339	if not max_buffer_size:
379	340	max_buffer_size = INT_MAX
	341	if max_str_len == -1:
	342	max_str_len = max_buffer_size
	343	if max_bin_len == -1:
	344	max_bin_len = max_buffer_size
	345	if max_array_len == -1:
	346	max_array_len = max_buffer_size
	347	if max_map_len == -1:
	348	max_map_len = max_buffer_size//2
	349	if max_ext_len == -1:
	350	max_ext_len = max_buffer_size
	351
380	352	if read_size > max_buffer_size:
381	353	raise ValueError("read_size should be less or equal to max_buffer_size")
382	354	if not read_size:
383	355	read_size = min(max_buffer_size, 1024**2)
	356
384	357	self.max_buffer_size = max_buffer_size
385	358	self.read_size = read_size
386	359	self.buf = <char*>PyMem_Malloc(read_size)

391	364	self.buf_tail = 0
392	365	self.stream_offset = 0
393	366
394		if encoding is not None:
395		PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
396		self.encoding = encoding
397		cenc = encoding
398
399	367	if unicode_errors is not None:
400	368	self.unicode_errors = unicode_errors
401	369	cerr = unicode_errors
402	370
403	371	init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
404		ext_hook, use_list, raw, strict_map_key, cenc, cerr,
	372	ext_hook, use_list, raw, timestamp, strict_map_key, cerr,
405	373	max_str_len, max_bin_len, max_array_len,
406	374	max_map_len, max_ext_len)
407	375
408	376	def feed(self, object next_bytes):
409	377	"""Append `next_bytes` to internal buffer."""
410	378	cdef Py_buffer pybuff
411		cdef int new_protocol = 0
412	379	cdef char* buf
413	380	cdef Py_ssize_t buf_len
414	381

416	383	raise AssertionError(
417	384	"unpacker.feed() is not be able to use with `file_like`.")
418	385
419		get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol)
	386	get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len)
420	387	try:
421	388	self.append_buffer(buf, buf_len)
422	389	finally:
423		if new_protocol:
424		PyBuffer_Release(&pybuff)
	390	PyBuffer_Release(&pybuff)
425	391
426	392	cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
427	393	cdef:

-1

msgpack/_version.py less more

0		version = (0, 6, 2)
	0	version = (1, 0, 0)

-20

msgpack/buff_converter.h less more

0	0	#include "Python.h"
1	1
2	2	/* cython does not support this preprocessor check => write it in raw C */
3		#if PY_MAJOR_VERSION == 2
4		static PyObject *
5		buff_to_buff(char *buff, Py_ssize_t size)
6		{
7		return PyBuffer_FromMemory(buff, size);
8		}
9
10		#elif (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION >= 3)
11	3	static PyObject *
12	4	buff_to_buff(char *buff, Py_ssize_t size)
13	5	{
14	6	return PyMemoryView_FromMemory(buff, size, PyBUF_READ);
15	7	}
16		#else
17		static PyObject *
18		buff_to_buff(char *buff, Py_ssize_t size)
19		{
20		Py_buffer pybuf;
21		if (PyBuffer_FillInfo(&pybuf, NULL, buff, size, 1, PyBUF_FULL_RO) == -1) {
22		return NULL;
23		}
24
25		return PyMemoryView_FromBuffer(&pybuf);
26		}
27		#endif

+191

-0

msgpack/ext.py less more

	0	# coding: utf-8
	1	from collections import namedtuple
	2	import datetime
	3	import sys
	4	import struct
	5
	6
	7	PY2 = sys.version_info[0] == 2
	8
	9	if PY2:
	10	int_types = (int, long)
	11	_utc = None
	12	else:
	13	int_types = int
	14	try:
	15	_utc = datetime.timezone.utc
	16	except AttributeError:
	17	_utc = datetime.timezone(datetime.timedelta(0))
	18
	19
	20	class ExtType(namedtuple("ExtType", "code data")):
	21	"""ExtType represents ext type in msgpack."""
	22
	23	def __new__(cls, code, data):
	24	if not isinstance(code, int):
	25	raise TypeError("code must be int")
	26	if not isinstance(data, bytes):
	27	raise TypeError("data must be bytes")
	28	if not 0 <= code <= 127:
	29	raise ValueError("code must be 0~127")
	30	return super(ExtType, cls).__new__(cls, code, data)
	31
	32
	33	class Timestamp(object):
	34	"""Timestamp represents the Timestamp extension type in msgpack.
	35
	36	When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python
	37	msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`.
	38
	39	This class is immutable: Do not override seconds and nanoseconds.
	40	"""
	41
	42	__slots__ = ["seconds", "nanoseconds"]
	43
	44	def __init__(self, seconds, nanoseconds=0):
	45	"""Initialize a Timestamp object.
	46
	47	:param int seconds:
	48	Number of seconds since the UNIX epoch (00:00:00 UTC Jan 1 1970, minus leap seconds).
	49	May be negative.
	50
	51	:param int nanoseconds:
	52	Number of nanoseconds to add to `seconds` to get fractional time.
	53	Maximum is 999_999_999. Default is 0.
	54
	55	Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns.
	56	"""
	57	if not isinstance(seconds, int_types):
	58	raise TypeError("seconds must be an interger")
	59	if not isinstance(nanoseconds, int_types):
	60	raise TypeError("nanoseconds must be an integer")
	61	if not (0 <= nanoseconds < 10 ** 9):
	62	raise ValueError(
	63	"nanoseconds must be a non-negative integer less than 999999999."
	64	)
	65	self.seconds = seconds
	66	self.nanoseconds = nanoseconds
	67
	68	def __repr__(self):
	69	"""String representation of Timestamp."""
	70	return "Timestamp(seconds={0}, nanoseconds={1})".format(
	71	self.seconds, self.nanoseconds
	72	)
	73
	74	def __eq__(self, other):
	75	"""Check for equality with another Timestamp object"""
	76	if type(other) is self.__class__:
	77	return (
	78	self.seconds == other.seconds and self.nanoseconds == other.nanoseconds
	79	)
	80	return False
	81
	82	def __ne__(self, other):
	83	"""not-equals method (see :func:`__eq__()`)"""
	84	return not self.__eq__(other)
	85
	86	def __hash__(self):
	87	return hash((self.seconds, self.nanoseconds))
	88
	89	@staticmethod
	90	def from_bytes(b):
	91	"""Unpack bytes into a `Timestamp` object.
	92
	93	Used for pure-Python msgpack unpacking.
	94
	95	:param b: Payload from msgpack ext message with code -1
	96	:type b: bytes
	97
	98	:returns: Timestamp object unpacked from msgpack ext payload
	99	:rtype: Timestamp
	100	"""
	101	if len(b) == 4:
	102	seconds = struct.unpack("!L", b)[0]
	103	nanoseconds = 0
	104	elif len(b) == 8:
	105	data64 = struct.unpack("!Q", b)[0]
	106	seconds = data64 & 0x00000003FFFFFFFF
	107	nanoseconds = data64 >> 34
	108	elif len(b) == 12:
	109	nanoseconds, seconds = struct.unpack("!Iq", b)
	110	else:
	111	raise ValueError(
	112	"Timestamp type can only be created from 32, 64, or 96-bit byte objects"
	113	)
	114	return Timestamp(seconds, nanoseconds)
	115
	116	def to_bytes(self):
	117	"""Pack this Timestamp object into bytes.
	118
	119	Used for pure-Python msgpack packing.
	120
	121	:returns data: Payload for EXT message with code -1 (timestamp type)
	122	:rtype: bytes
	123	"""
	124	if (self.seconds >> 34) == 0: # seconds is non-negative and fits in 34 bits
	125	data64 = self.nanoseconds << 34 \| self.seconds
	126	if data64 & 0xFFFFFFFF00000000 == 0:
	127	# nanoseconds is zero and seconds < 2**32, so timestamp 32
	128	data = struct.pack("!L", data64)
	129	else:
	130	# timestamp 64
	131	data = struct.pack("!Q", data64)
	132	else:
	133	# timestamp 96
	134	data = struct.pack("!Iq", self.nanoseconds, self.seconds)
	135	return data
	136
	137	@staticmethod
	138	def from_unix(unix_sec):
	139	"""Create a Timestamp from posix timestamp in seconds.
	140
	141	:param unix_float: Posix timestamp in seconds.
	142	:type unix_float: int or float.
	143	"""
	144	seconds = int(unix_sec // 1)
	145	nanoseconds = int((unix_sec % 1) * 10 ** 9)
	146	return Timestamp(seconds, nanoseconds)
	147
	148	def to_unix(self):
	149	"""Get the timestamp as a floating-point value.
	150
	151	:returns: posix timestamp
	152	:rtype: float
	153	"""
	154	return self.seconds + self.nanoseconds / 1e9
	155
	156	@staticmethod
	157	def from_unix_nano(unix_ns):
	158	"""Create a Timestamp from posix timestamp in nanoseconds.
	159
	160	:param int unix_ns: Posix timestamp in nanoseconds.
	161	:rtype: Timestamp
	162	"""
	163	return Timestamp(divmod(unix_ns, 10 * 9))
	164
	165	def to_unix_nano(self):
	166	"""Get the timestamp as a unixtime in nanoseconds.
	167
	168	:returns: posix timestamp in nanoseconds
	169	:rtype: int
	170	"""
	171	return self.seconds * 10 ** 9 + self.nanoseconds
	172
	173	def to_datetime(self):
	174	"""Get the timestamp as a UTC datetime.
	175
	176	Python 2 is not supported.
	177
	178	:rtype: datetime.
	179	"""
	180	return datetime.datetime.fromtimestamp(self.to_unix(), _utc)
	181
	182	@staticmethod
	183	def from_datetime(dt):
	184	"""Create a Timestamp from datetime with tzinfo.
	185
	186	Python 2 is not supported.
	187
	188	:rtype: Timestamp
	189	"""
	190	return Timestamp.from_unix(dt.timestamp())

+307

-271

msgpack/fallback.py less more

0	0	"""Fallback pure Python implementation of msgpack"""
1	1
	2	from datetime import datetime as _DateTime
2	3	import sys
3	4	import struct
4		import warnings
5
6
7		if sys.version_info[0] == 2:
8		PY2 = True
	5
	6
	7	PY2 = sys.version_info[0] == 2
	8	if PY2:
9	9	int_types = (int, long)
	10
10	11	def dict_iteritems(d):
11	12	return d.iteritems()
	13
	14
12	15	else:
13		PY2 = False
14	16	int_types = int
15	17	unicode = str
16	18	xrange = range
	19
17	20	def dict_iteritems(d):
18	21	return d.items()
	22
19	23
20	24	if sys.version_info < (3, 5):
21	25	# Ugly hack...
22	26	RecursionError = RuntimeError
23	27
24	28	def _is_recursionerror(e):
25		return len(e.args) == 1 and isinstance(e.args[0], str) and \
26		e.args[0].startswith('maximum recursion depth exceeded')
	29	return (
	30	len(e.args) == 1
	31	and isinstance(e.args[0], str)
	32	and e.args[0].startswith("maximum recursion depth exceeded")
	33	)
	34
	35
27	36	else:
	37
28	38	def _is_recursionerror(e):
29	39	return True
30	40
31		if hasattr(sys, 'pypy_version_info'):
32		# cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own
	41
	42	if hasattr(sys, "pypy_version_info"):
	43	# StringIO is slow on PyPy, StringIO is faster. However: PyPy's own
33	44	# StringBuilder is fastest.
34	45	from __pypy__ import newlist_hint
	46
35	47	try:
36	48	from __pypy__.builders import BytesBuilder as StringBuilder
37	49	except ImportError:
38	50	from __pypy__.builders import StringBuilder
39	51	USING_STRINGBUILDER = True
	52
40	53	class StringIO(object):
41		def __init__(self, s=b''):
	54	def __init__(self, s=b""):
42	55	if s:
43	56	self.builder = StringBuilder(len(s))
44	57	self.builder.append(s)
45	58	else:
46	59	self.builder = StringBuilder()
	60
47	61	def write(self, s):
48	62	if isinstance(s, memoryview):
49	63	s = s.tobytes()
50	64	elif isinstance(s, bytearray):
51	65	s = bytes(s)
52	66	self.builder.append(s)
	67
53	68	def getvalue(self):
54	69	return self.builder.build()
	70
	71
55	72	else:
56	73	USING_STRINGBUILDER = False
57	74	from io import BytesIO as StringIO
	75
58	76	newlist_hint = lambda size: []
59	77
60	78
61		from .exceptions import (
62		BufferFull,
63		OutOfData,
64		ExtraData,
65		FormatError,
66		StackError,
67		)
68
69		from . import ExtType
70
71
72		EX_SKIP = 0
73		EX_CONSTRUCT = 1
74		EX_READ_ARRAY_HEADER = 2
75		EX_READ_MAP_HEADER = 3
76
77		TYPE_IMMEDIATE = 0
78		TYPE_ARRAY = 1
79		TYPE_MAP = 2
80		TYPE_RAW = 3
81		TYPE_BIN = 4
82		TYPE_EXT = 5
	79	from .exceptions import BufferFull, OutOfData, ExtraData, FormatError, StackError
	80
	81	from .ext import ExtType, Timestamp
	82
	83
	84	EX_SKIP = 0
	85	EX_CONSTRUCT = 1
	86	EX_READ_ARRAY_HEADER = 2
	87	EX_READ_MAP_HEADER = 3
	88
	89	TYPE_IMMEDIATE = 0
	90	TYPE_ARRAY = 1
	91	TYPE_MAP = 2
	92	TYPE_RAW = 3
	93	TYPE_BIN = 4
	94	TYPE_EXT = 5
83	95
84	96	DEFAULT_RECURSE_LIMIT = 511
85	97

92	104
93	105
94	106	def _get_data_from_buffer(obj):
95		try:
96		view = memoryview(obj)
97		except TypeError:
98		# try to use legacy buffer protocol if 2.7, otherwise re-raise
99		if PY2:
100		view = memoryview(buffer(obj))
101		warnings.warn("using old buffer interface to unpack %s; "
102		"this leads to unpacking errors if slicing is used and "
103		"will be removed in a future version" % type(obj),
104		RuntimeWarning, stacklevel=3)
105		else:
106		raise
	107	view = memoryview(obj)
107	108	if view.itemsize != 1:
108	109	raise ValueError("cannot unpack from multi-byte object")
109	110	return view
110
111
112		def unpack(stream, **kwargs):
113		warnings.warn(
114		"Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
115		DeprecationWarning, stacklevel=2)
116		data = stream.read()
117		return unpackb(data, **kwargs)
118	111
119	112
120	113	def unpackb(packed, **kwargs):

145	138
146	139
147	140	if sys.version_info < (2, 7, 6):
	141
148	142	def _unpack_from(f, b, o=0):
149		"""Explicit typcast for legacy struct.unpack_from"""
	143	"""Explicit type cast for legacy struct.unpack_from"""
150	144	return struct.unpack_from(f, bytes(b), o)
	145
	146
151	147	else:
152	148	_unpack_from = struct.unpack_from
153	149

155	151	class Unpacker(object):
156	152	"""Streaming unpacker.
157	153
158		arguments:
	154	Arguments:
159	155
160	156	:param file_like:
161	157	File-like object having `.read(n)` method.

169	165	Otherwise, unpack to Python tuple. (default: True)
170	166
171	167	:param bool raw:
172		If true, unpack msgpack raw to Python bytes (default).
173		Otherwise, unpack to Python str (or unicode on Python 2) by decoding
174		with UTF-8 encoding (recommended).
175		Currently, the default is true, but it will be changed to false in
176		near future. So you must specify it explicitly for keeping backward
177		compatibility.
178
179		encoding option which is deprecated overrides this option.
	168	If true, unpack msgpack raw to Python bytes.
	169	Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
	170
	171	:param int timestamp:
	172	Control how timestamp type is unpacked:
	173
	174	0 - Timestamp
	175	1 - float (Seconds from the EPOCH)
	176	2 - int (Nanoseconds from the EPOCH)
	177	3 - datetime.datetime (UTC). Python 2 is not supported.
180	178
181	179	:param bool strict_map_key:
182		If true, only str or bytes are accepted for map (dict) keys.
183		It's False by default for backward-compatibility.
184		But it will be True from msgpack 1.0.
	180	If true (default), only str or bytes are accepted for map (dict) keys.
185	181
186	182	:param callable object_hook:
187	183	When specified, it should be callable.

193	189	Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
194	190	(See also simplejson)
195	191
196		:param str encoding:
197		Encoding used for decoding msgpack raw.
198		If it is None (default), msgpack raw is deserialized to Python bytes.
199
200	192	:param str unicode_errors:
201		(deprecated) Used for decoding msgpack raw with encoding.
202		(default: `'strict'`)
	193	The error handler for decoding unicode. (default: 'strict')
	194	This option should be used only when you have msgpack data which
	195	contains invalid UTF-8 string.
203	196
204	197	:param int max_buffer_size:
205		Limits size of data waiting unpacked. 0 means system's INT_MAX (default).
	198	Limits size of data waiting unpacked. 0 means 2**32-1.
	199	The default value is 10010241024 (100MiB).
206	200	Raises `BufferFull` exception when it is insufficient.
207	201	You should set this parameter when unpacking data from untrusted source.
208	202
209	203	:param int max_str_len:
210	204	Deprecated, use max_buffer_size instead.
211		Limits max length of str. (default: max_buffer_size or 1024*1024)
	205	Limits max length of str. (default: max_buffer_size)
212	206
213	207	:param int max_bin_len:
214	208	Deprecated, use max_buffer_size instead.
215		Limits max length of bin. (default: max_buffer_size or 1024*1024)
	209	Limits max length of bin. (default: max_buffer_size)
216	210
217	211	:param int max_array_len:
218	212	Limits max length of array.
219		(default: max_buffer_size or 128*1024)
	213	(default: max_buffer_size)
220	214
221	215	:param int max_map_len:
222	216	Limits max length of map.
223		(default: max_buffer_size//2 or 32*1024)
	217	(default: max_buffer_size//2)
224	218
225	219	:param int max_ext_len:
226	220	Deprecated, use max_buffer_size instead.
227		Limits max size of ext type. (default: max_buffer_size or 1024*1024)
	221	Limits max size of ext type. (default: max_buffer_size)
228	222
229	223	Example of streaming deserialize from file-like object::
230	224
231		unpacker = Unpacker(file_like, raw=False, max_buffer_size=1010241024)
	225	unpacker = Unpacker(file_like)
232	226	for o in unpacker:
233	227	process(o)
234	228
235	229	Example of streaming deserialize from socket::
236	230
237		unpacker = Unpacker(raw=False, max_buffer_size=1010241024)
	231	unpacker = Unpacker(max_buffer_size)
238	232	while True:
239	233	buf = sock.recv(1024**2)
240	234	if not buf:

250	244	Other exceptions can be raised during unpacking.
251	245	"""
252	246
253		def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False,
254		object_hook=None, object_pairs_hook=None, list_hook=None,
255		encoding=None, unicode_errors=None, max_buffer_size=0,
256		ext_hook=ExtType,
257		max_str_len=-1,
258		max_bin_len=-1,
259		max_array_len=-1,
260		max_map_len=-1,
261		max_ext_len=-1):
262		if encoding is not None:
263		warnings.warn(
264		"encoding is deprecated, Use raw=False instead.",
265		DeprecationWarning, stacklevel=2)
266
	247	def __init__(
	248	self,
	249	file_like=None,
	250	read_size=0,
	251	use_list=True,
	252	raw=False,
	253	timestamp=0,
	254	strict_map_key=True,
	255	object_hook=None,
	256	object_pairs_hook=None,
	257	list_hook=None,
	258	unicode_errors=None,
	259	max_buffer_size=100 * 1024 * 1024,
	260	ext_hook=ExtType,
	261	max_str_len=-1,
	262	max_bin_len=-1,
	263	max_array_len=-1,
	264	max_map_len=-1,
	265	max_ext_len=-1,
	266	):
267	267	if unicode_errors is None:
268		unicode_errors = 'strict'
	268	unicode_errors = "strict"
269	269
270	270	if file_like is None:
271	271	self._feeding = True

289	289	# state, which _buf_checkpoint records.
290	290	self._buf_checkpoint = 0
291	291
	292	if not max_buffer_size:
	293	max_buffer_size = 2 ** 31 - 1
292	294	if max_str_len == -1:
293		max_str_len = max_buffer_size or 1024*1024
	295	max_str_len = max_buffer_size
294	296	if max_bin_len == -1:
295		max_bin_len = max_buffer_size or 1024*1024
	297	max_bin_len = max_buffer_size
296	298	if max_array_len == -1:
297		max_array_len = max_buffer_size or 128*1024
	299	max_array_len = max_buffer_size
298	300	if max_map_len == -1:
299		max_map_len = max_buffer_size//2 or 32*1024
	301	max_map_len = max_buffer_size // 2
300	302	if max_ext_len == -1:
301		max_ext_len = max_buffer_size or 1024*1024
302
303		self._max_buffer_size = max_buffer_size or 2**31-1
	303	max_ext_len = max_buffer_size
	304
	305	self._max_buffer_size = max_buffer_size
304	306	if read_size > self._max_buffer_size:
305	307	raise ValueError("read_size must be smaller than max_buffer_size")
306		self._read_size = read_size or min(self._max_buffer_size, 16*1024)
	308	self._read_size = read_size or min(self._max_buffer_size, 16 * 1024)
307	309	self._raw = bool(raw)
308	310	self._strict_map_key = bool(strict_map_key)
309		self._encoding = encoding
310	311	self._unicode_errors = unicode_errors
311	312	self._use_list = use_list
	313	if not (0 <= timestamp <= 3):
	314	raise ValueError("timestamp must be 0..3")
	315	self._timestamp = timestamp
312	316	self._list_hook = list_hook
313	317	self._object_hook = object_hook
314	318	self._object_pairs_hook = object_pairs_hook

321	325	self._stream_offset = 0
322	326
323	327	if list_hook is not None and not callable(list_hook):
324		raise TypeError('`list_hook` is not callable')
	328	raise TypeError("`list_hook` is not callable")
325	329	if object_hook is not None and not callable(object_hook):
326		raise TypeError('`object_hook` is not callable')
	330	raise TypeError("`object_hook` is not callable")
327	331	if object_pairs_hook is not None and not callable(object_pairs_hook):
328		raise TypeError('`object_pairs_hook` is not callable')
	332	raise TypeError("`object_pairs_hook` is not callable")
329	333	if object_hook is not None and object_pairs_hook is not None:
330		raise TypeError("object_pairs_hook and object_hook are mutually "
331		"exclusive")
	334	raise TypeError(
	335	"object_pairs_hook and object_hook are mutually " "exclusive"
	336	)
332	337	if not callable(ext_hook):
333	338	raise TypeError("`ext_hook` is not callable")
334	339
335	340	def feed(self, next_bytes):
336	341	assert self._feeding
337	342	view = _get_data_from_buffer(next_bytes)
338		if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size):
	343	if len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size:
339	344	raise BufferFull
340	345
341	346	# Strip buffer before checkpoint before reading file.
342	347	if self._buf_checkpoint > 0:
343		del self._buffer[:self._buf_checkpoint]
	348	del self._buffer[: self._buf_checkpoint]
344	349	self._buff_i -= self._buf_checkpoint
345	350	self._buf_checkpoint = 0
346	351

356	361	return self._buff_i < len(self._buffer)
357	362
358	363	def _get_extradata(self):
359		return self._buffer[self._buff_i:]
	364	return self._buffer[self._buff_i :]
360	365
361	366	def read_bytes(self, n):
362		return self._read(n)
	367	ret = self._read(n)
	368	self._consume()
	369	return ret
363	370
364	371	def _read(self, n):
365	372	# (int) -> bytearray
366	373	self._reserve(n)
367	374	i = self._buff_i
368		self._buff_i = i+n
369		return self._buffer[i:i+n]
	375	self._buff_i = i + n
	376	return self._buffer[i : i + n]
370	377
371	378	def _reserve(self, n):
372	379	remain_bytes = len(self._buffer) - self._buff_i - n

381	388
382	389	# Strip buffer before checkpoint before reading file.
383	390	if self._buf_checkpoint > 0:
384		del self._buffer[:self._buf_checkpoint]
	391	del self._buffer[: self._buf_checkpoint]
385	392	self._buff_i -= self._buf_checkpoint
386	393	self._buf_checkpoint = 0
387	394

410	417	if b & 0b10000000 == 0:
411	418	obj = b
412	419	elif b & 0b11100000 == 0b11100000:
413		obj = -1 - (b ^ 0xff)
	420	obj = -1 - (b ^ 0xFF)
414	421	elif b & 0b11100000 == 0b10100000:
415	422	n = b & 0b00011111
416	423	typ = TYPE_RAW

427	434	typ = TYPE_MAP
428	435	if n > self._max_map_len:
429	436	raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
430		elif b == 0xc0:
	437	elif b == 0xC0:
431	438	obj = None
432		elif b == 0xc2:
	439	elif b == 0xC2:
433	440	obj = False
434		elif b == 0xc3:
	441	elif b == 0xC3:
435	442	obj = True
436		elif b == 0xc4:
	443	elif b == 0xC4:
437	444	typ = TYPE_BIN
438	445	self._reserve(1)
439	446	n = self._buffer[self._buff_i]

441	448	if n > self._max_bin_len:
442	449	raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
443	450	obj = self._read(n)
444		elif b == 0xc5:
	451	elif b == 0xC5:
445	452	typ = TYPE_BIN
446	453	self._reserve(2)
447	454	n = _unpack_from(">H", self._buffer, self._buff_i)[0]

449	456	if n > self._max_bin_len:
450	457	raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
451	458	obj = self._read(n)
452		elif b == 0xc6:
	459	elif b == 0xC6:
453	460	typ = TYPE_BIN
454	461	self._reserve(4)
455	462	n = _unpack_from(">I", self._buffer, self._buff_i)[0]

457	464	if n > self._max_bin_len:
458	465	raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
459	466	obj = self._read(n)
460		elif b == 0xc7: # ext 8
	467	elif b == 0xC7: # ext 8
461	468	typ = TYPE_EXT
462	469	self._reserve(2)
463		L, n = _unpack_from('Bb', self._buffer, self._buff_i)
	470	L, n = _unpack_from("Bb", self._buffer, self._buff_i)
464	471	self._buff_i += 2
465	472	if L > self._max_ext_len:
466	473	raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
467	474	obj = self._read(L)
468		elif b == 0xc8: # ext 16
	475	elif b == 0xC8: # ext 16
469	476	typ = TYPE_EXT
470	477	self._reserve(3)
471		L, n = _unpack_from('>Hb', self._buffer, self._buff_i)
	478	L, n = _unpack_from(">Hb", self._buffer, self._buff_i)
472	479	self._buff_i += 3
473	480	if L > self._max_ext_len:
474	481	raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
475	482	obj = self._read(L)
476		elif b == 0xc9: # ext 32
	483	elif b == 0xC9: # ext 32
477	484	typ = TYPE_EXT
478	485	self._reserve(5)
479		L, n = _unpack_from('>Ib', self._buffer, self._buff_i)
	486	L, n = _unpack_from(">Ib", self._buffer, self._buff_i)
480	487	self._buff_i += 5
481	488	if L > self._max_ext_len:
482	489	raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
483	490	obj = self._read(L)
484		elif b == 0xca:
	491	elif b == 0xCA:
485	492	self._reserve(4)
486	493	obj = _unpack_from(">f", self._buffer, self._buff_i)[0]
487	494	self._buff_i += 4
488		elif b == 0xcb:
	495	elif b == 0xCB:
489	496	self._reserve(8)
490	497	obj = _unpack_from(">d", self._buffer, self._buff_i)[0]
491	498	self._buff_i += 8
492		elif b == 0xcc:
	499	elif b == 0xCC:
493	500	self._reserve(1)
494	501	obj = self._buffer[self._buff_i]
495	502	self._buff_i += 1
496		elif b == 0xcd:
	503	elif b == 0xCD:
497	504	self._reserve(2)
498	505	obj = _unpack_from(">H", self._buffer, self._buff_i)[0]
499	506	self._buff_i += 2
500		elif b == 0xce:
	507	elif b == 0xCE:
501	508	self._reserve(4)
502	509	obj = _unpack_from(">I", self._buffer, self._buff_i)[0]
503	510	self._buff_i += 4
504		elif b == 0xcf:
	511	elif b == 0xCF:
505	512	self._reserve(8)
506	513	obj = _unpack_from(">Q", self._buffer, self._buff_i)[0]
507	514	self._buff_i += 8
508		elif b == 0xd0:
	515	elif b == 0xD0:
509	516	self._reserve(1)
510	517	obj = _unpack_from("b", self._buffer, self._buff_i)[0]
511	518	self._buff_i += 1
512		elif b == 0xd1:
	519	elif b == 0xD1:
513	520	self._reserve(2)
514	521	obj = _unpack_from(">h", self._buffer, self._buff_i)[0]
515	522	self._buff_i += 2
516		elif b == 0xd2:
	523	elif b == 0xD2:
517	524	self._reserve(4)
518	525	obj = _unpack_from(">i", self._buffer, self._buff_i)[0]
519	526	self._buff_i += 4
520		elif b == 0xd3:
	527	elif b == 0xD3:
521	528	self._reserve(8)
522	529	obj = _unpack_from(">q", self._buffer, self._buff_i)[0]
523	530	self._buff_i += 8
524		elif b == 0xd4: # fixext 1
	531	elif b == 0xD4: # fixext 1
525	532	typ = TYPE_EXT
526	533	if self._max_ext_len < 1:
527	534	raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
528	535	self._reserve(2)
529	536	n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
530	537	self._buff_i += 2
531		elif b == 0xd5: # fixext 2
	538	elif b == 0xD5: # fixext 2
532	539	typ = TYPE_EXT
533	540	if self._max_ext_len < 2:
534	541	raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
535	542	self._reserve(3)
536	543	n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
537	544	self._buff_i += 3
538		elif b == 0xd6: # fixext 4
	545	elif b == 0xD6: # fixext 4
539	546	typ = TYPE_EXT
540	547	if self._max_ext_len < 4:
541	548	raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
542	549	self._reserve(5)
543	550	n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
544	551	self._buff_i += 5
545		elif b == 0xd7: # fixext 8
	552	elif b == 0xD7: # fixext 8
546	553	typ = TYPE_EXT
547	554	if self._max_ext_len < 8:
548	555	raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
549	556	self._reserve(9)
550	557	n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
551	558	self._buff_i += 9
552		elif b == 0xd8: # fixext 16
	559	elif b == 0xD8: # fixext 16
553	560	typ = TYPE_EXT
554	561	if self._max_ext_len < 16:
555	562	raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
556	563	self._reserve(17)
557	564	n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
558	565	self._buff_i += 17
559		elif b == 0xd9:
	566	elif b == 0xD9:
560	567	typ = TYPE_RAW
561	568	self._reserve(1)
562	569	n = self._buffer[self._buff_i]

564	571	if n > self._max_str_len:
565	572	raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
566	573	obj = self._read(n)
567		elif b == 0xda:
	574	elif b == 0xDA:
568	575	typ = TYPE_RAW
569	576	self._reserve(2)
570		n, = _unpack_from(">H", self._buffer, self._buff_i)
	577	(n,) = _unpack_from(">H", self._buffer, self._buff_i)
571	578	self._buff_i += 2
572	579	if n > self._max_str_len:
573	580	raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
574	581	obj = self._read(n)
575		elif b == 0xdb:
	582	elif b == 0xDB:
576	583	typ = TYPE_RAW
577	584	self._reserve(4)
578		n, = _unpack_from(">I", self._buffer, self._buff_i)
	585	(n,) = _unpack_from(">I", self._buffer, self._buff_i)
579	586	self._buff_i += 4
580	587	if n > self._max_str_len:
581	588	raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
582	589	obj = self._read(n)
583		elif b == 0xdc:
	590	elif b == 0xDC:
584	591	typ = TYPE_ARRAY
585	592	self._reserve(2)
586		n, = _unpack_from(">H", self._buffer, self._buff_i)
	593	(n,) = _unpack_from(">H", self._buffer, self._buff_i)
587	594	self._buff_i += 2
588	595	if n > self._max_array_len:
589	596	raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
590		elif b == 0xdd:
	597	elif b == 0xDD:
591	598	typ = TYPE_ARRAY
592	599	self._reserve(4)
593		n, = _unpack_from(">I", self._buffer, self._buff_i)
	600	(n,) = _unpack_from(">I", self._buffer, self._buff_i)
594	601	self._buff_i += 4
595	602	if n > self._max_array_len:
596	603	raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
597		elif b == 0xde:
	604	elif b == 0xDE:
598	605	self._reserve(2)
599		n, = _unpack_from(">H", self._buffer, self._buff_i)
	606	(n,) = _unpack_from(">H", self._buffer, self._buff_i)
600	607	self._buff_i += 2
601	608	if n > self._max_map_len:
602	609	raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
603	610	typ = TYPE_MAP
604		elif b == 0xdf:
	611	elif b == 0xDF:
605	612	self._reserve(4)
606		n, = _unpack_from(">I", self._buffer, self._buff_i)
	613	(n,) = _unpack_from(">I", self._buffer, self._buff_i)
607	614	self._buff_i += 4
608	615	if n > self._max_map_len:
609	616	raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)

646	653	return
647	654	if self._object_pairs_hook is not None:
648	655	ret = self._object_pairs_hook(
649		(self._unpack(EX_CONSTRUCT),
650		self._unpack(EX_CONSTRUCT))
651		for _ in xrange(n))
	656	(self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT))
	657	for _ in xrange(n)
	658	)
652	659	else:
653	660	ret = {}
654	661	for _ in xrange(n):
655	662	key = self._unpack(EX_CONSTRUCT)
656	663	if self._strict_map_key and type(key) not in (unicode, bytes):
657		raise ValueError("%s is not allowed for map key" % str(type(key)))
	664	raise ValueError(
	665	"%s is not allowed for map key" % str(type(key))
	666	)
	667	if not PY2 and type(key) is str:
	668	key = sys.intern(key)
658	669	ret[key] = self._unpack(EX_CONSTRUCT)
659	670	if self._object_hook is not None:
660	671	ret = self._object_hook(ret)

662	673	if execute == EX_SKIP:
663	674	return
664	675	if typ == TYPE_RAW:
665		if self._encoding is not None:
666		obj = obj.decode(self._encoding, self._unicode_errors)
667		elif self._raw:
	676	if self._raw:
668	677	obj = bytes(obj)
669	678	else:
670		obj = obj.decode('utf_8')
	679	obj = obj.decode("utf_8", self._unicode_errors)
671	680	return obj
672		if typ == TYPE_EXT:
673		return self._ext_hook(n, bytes(obj))
674	681	if typ == TYPE_BIN:
675	682	return bytes(obj)
	683	if typ == TYPE_EXT:
	684	if n == -1: # timestamp
	685	ts = Timestamp.from_bytes(bytes(obj))
	686	if self._timestamp == 1:
	687	return ts.to_unix()
	688	elif self._timestamp == 2:
	689	return ts.to_unix_nano()
	690	elif self._timestamp == 3:
	691	return ts.to_datetime()
	692	else:
	693	return ts
	694	else:
	695	return self._ext_hook(n, bytes(obj))
676	696	assert typ == TYPE_IMMEDIATE
677	697	return obj
678	698

722	742	"""
723	743	MessagePack Packer
724	744
725		usage:
	745	Usage:
726	746
727	747	packer = Packer()
728	748	astream.write(packer.pack(a))

743	763
744	764	:param bool use_bin_type:
745	765	Use bin type introduced in msgpack spec 2.0 for bytes.
746		It also enables str8 type for unicode.
	766	It also enables str8 type for unicode. (default: True)
747	767
748	768	:param bool strict_types:
749	769	If set to true, types will be checked to be exact. Derived classes
750		from serializeable types will not be serialized and will be
	770	from serializable types will not be serialized and will be
751	771	treated as unsupported type and forwarded to default.
752	772	Additionally tuples will not be serialized as lists.
753	773	This is useful when trying to implement accurate serialization
754	774	for python types.
755	775
756		:param str encoding:
757		(deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
	776	:param bool datetime:
	777	If set to true, datetime with tzinfo is packed into Timestamp type.
	778	Note that the tzinfo is stripped in the timestamp.
	779	You can get UTC datetime with `timestamp=3` option of the Unpacker.
	780	(Python 2 is not supported).
758	781
759	782	:param str unicode_errors:
760		Error handler for encoding unicode. (default: 'strict')
	783	The error handler for encoding unicode. (default: 'strict')
	784	DO NOT USE THIS!! This option is kept for very specific usage.
761	785	"""
762		def __init__(self, default=None, encoding=None, unicode_errors=None,
763		use_single_float=False, autoreset=True, use_bin_type=False,
764		strict_types=False):
765		if encoding is None:
766		encoding = 'utf_8'
767		else:
768		warnings.warn(
769		"encoding is deprecated, Use raw=False instead.",
770		DeprecationWarning, stacklevel=2)
771
772		if unicode_errors is None:
773		unicode_errors = 'strict'
774
	786
	787	def __init__(
	788	self,
	789	default=None,
	790	use_single_float=False,
	791	autoreset=True,
	792	use_bin_type=True,
	793	strict_types=False,
	794	datetime=False,
	795	unicode_errors=None,
	796	):
775	797	self._strict_types = strict_types
776	798	self._use_float = use_single_float
777	799	self._autoreset = autoreset
778	800	self._use_bin_type = use_bin_type
779		self._encoding = encoding
780		self._unicode_errors = unicode_errors
781	801	self._buffer = StringIO()
	802	if PY2 and datetime:
	803	raise ValueError("datetime is not supported in Python 2")
	804	self._datetime = bool(datetime)
	805	self._unicode_errors = unicode_errors or "strict"
782	806	if default is not None:
783	807	if not callable(default):
784	808	raise TypeError("default must be callable")
785	809	self._default = default
786	810
787		def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT,
788		check=isinstance, check_type_strict=_check_type_strict):
	811	def _pack(
	812	self,
	813	obj,
	814	nest_limit=DEFAULT_RECURSE_LIMIT,
	815	check=isinstance,
	816	check_type_strict=_check_type_strict,
	817	):
789	818	default_used = False
790	819	if self._strict_types:
791	820	check = check_type_strict

806	835	return self._buffer.write(struct.pack("B", obj))
807	836	if -0x20 <= obj < 0:
808	837	return self._buffer.write(struct.pack("b", obj))
809		if 0x80 <= obj <= 0xff:
810		return self._buffer.write(struct.pack("BB", 0xcc, obj))
	838	if 0x80 <= obj <= 0xFF:
	839	return self._buffer.write(struct.pack("BB", 0xCC, obj))
811	840	if -0x80 <= obj < 0:
812		return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
813		if 0xff < obj <= 0xffff:
814		return self._buffer.write(struct.pack(">BH", 0xcd, obj))
	841	return self._buffer.write(struct.pack(">Bb", 0xD0, obj))
	842	if 0xFF < obj <= 0xFFFF:
	843	return self._buffer.write(struct.pack(">BH", 0xCD, obj))
815	844	if -0x8000 <= obj < -0x80:
816		return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
817		if 0xffff < obj <= 0xffffffff:
818		return self._buffer.write(struct.pack(">BI", 0xce, obj))
	845	return self._buffer.write(struct.pack(">Bh", 0xD1, obj))
	846	if 0xFFFF < obj <= 0xFFFFFFFF:
	847	return self._buffer.write(struct.pack(">BI", 0xCE, obj))
819	848	if -0x80000000 <= obj < -0x8000:
820		return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
821		if 0xffffffff < obj <= 0xffffffffffffffff:
822		return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
	849	return self._buffer.write(struct.pack(">Bi", 0xD2, obj))
	850	if 0xFFFFFFFF < obj <= 0xFFFFFFFFFFFFFFFF:
	851	return self._buffer.write(struct.pack(">BQ", 0xCF, obj))
823	852	if -0x8000000000000000 <= obj < -0x80000000:
824		return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
	853	return self._buffer.write(struct.pack(">Bq", 0xD3, obj))
825	854	if not default_used and self._default is not None:
826	855	obj = self._default(obj)
827	856	default_used = True

829	858	raise OverflowError("Integer value out of range")
830	859	if check(obj, (bytes, bytearray)):
831	860	n = len(obj)
832		if n >= 2**32:
	861	if n >= 2 ** 32:
833	862	raise ValueError("%s is too large" % type(obj).__name__)
834	863	self._pack_bin_header(n)
835	864	return self._buffer.write(obj)
836	865	if check(obj, unicode):
837		if self._encoding is None:
838		raise TypeError(
839		"Can't encode unicode string: "
840		"no encoding is specified")
841		obj = obj.encode(self._encoding, self._unicode_errors)
	866	obj = obj.encode("utf-8", self._unicode_errors)
842	867	n = len(obj)
843		if n >= 2**32:
	868	if n >= 2 ** 32:
844	869	raise ValueError("String is too large")
845	870	self._pack_raw_header(n)
846	871	return self._buffer.write(obj)
847	872	if check(obj, memoryview):
848	873	n = len(obj) * obj.itemsize
849		if n >= 2**32:
	874	if n >= 2 ** 32:
850	875	raise ValueError("Memoryview is too large")
851	876	self._pack_bin_header(n)
852	877	return self._buffer.write(obj)
853	878	if check(obj, float):
854	879	if self._use_float:
855		return self._buffer.write(struct.pack(">Bf", 0xca, obj))
856		return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
857		if check(obj, ExtType):
858		code = obj.code
859		data = obj.data
	880	return self._buffer.write(struct.pack(">Bf", 0xCA, obj))
	881	return self._buffer.write(struct.pack(">Bd", 0xCB, obj))
	882	if check(obj, (ExtType, Timestamp)):
	883	if check(obj, Timestamp):
	884	code = -1
	885	data = obj.to_bytes()
	886	else:
	887	code = obj.code
	888	data = obj.data
860	889	assert isinstance(code, int)
861	890	assert isinstance(data, bytes)
862	891	L = len(data)
863	892	if L == 1:
864		self._buffer.write(b'\xd4')
	893	self._buffer.write(b"\xd4")
865	894	elif L == 2:
866		self._buffer.write(b'\xd5')
	895	self._buffer.write(b"\xd5")
867	896	elif L == 4:
868		self._buffer.write(b'\xd6')
	897	self._buffer.write(b"\xd6")
869	898	elif L == 8:
870		self._buffer.write(b'\xd7')
	899	self._buffer.write(b"\xd7")
871	900	elif L == 16:
872		self._buffer.write(b'\xd8')
873		elif L <= 0xff:
874		self._buffer.write(struct.pack(">BB", 0xc7, L))
875		elif L <= 0xffff:
876		self._buffer.write(struct.pack(">BH", 0xc8, L))
	901	self._buffer.write(b"\xd8")
	902	elif L <= 0xFF:
	903	self._buffer.write(struct.pack(">BB", 0xC7, L))
	904	elif L <= 0xFFFF:
	905	self._buffer.write(struct.pack(">BH", 0xC8, L))
877	906	else:
878		self._buffer.write(struct.pack(">BI", 0xc9, L))
	907	self._buffer.write(struct.pack(">BI", 0xC9, L))
879	908	self._buffer.write(struct.pack("b", code))
880	909	self._buffer.write(data)
881	910	return

886	915	self._pack(obj[i], nest_limit - 1)
887	916	return
888	917	if check(obj, dict):
889		return self._pack_map_pairs(len(obj), dict_iteritems(obj),
890		nest_limit - 1)
	918	return self._pack_map_pairs(
	919	len(obj), dict_iteritems(obj), nest_limit - 1
	920	)
	921
	922	if self._datetime and check(obj, _DateTime):
	923	obj = Timestamp.from_datetime(obj)
	924	default_used = 1
	925	continue
	926
891	927	if not default_used and self._default is not None:
892	928	obj = self._default(obj)
893	929	default_used = 1
894	930	continue
895		raise TypeError("Cannot serialize %r" % (obj, ))
	931	raise TypeError("Cannot serialize %r" % (obj,))
896	932
897	933	def pack(self, obj):
898	934	try:

913	949	return ret
914	950
915	951	def pack_array_header(self, n):
916		if n >= 2**32:
	952	if n >= 2 ** 32:
917	953	raise ValueError
918	954	self._pack_array_header(n)
919	955	if self._autoreset:

922	958	return ret
923	959
924	960	def pack_map_header(self, n):
925		if n >= 2**32:
	961	if n >= 2 ** 32:
926	962	raise ValueError
927	963	self._pack_map_header(n)
928	964	if self._autoreset:

938	974	if not isinstance(data, bytes):
939	975	raise TypeError("data must have bytes type")
940	976	L = len(data)
941		if L > 0xffffffff:
	977	if L > 0xFFFFFFFF:
942	978	raise ValueError("Too large data")
943	979	if L == 1:
944		self._buffer.write(b'\xd4')
	980	self._buffer.write(b"\xd4")
945	981	elif L == 2:
946		self._buffer.write(b'\xd5')
	982	self._buffer.write(b"\xd5")
947	983	elif L == 4:
948		self._buffer.write(b'\xd6')
	984	self._buffer.write(b"\xd6")
949	985	elif L == 8:
950		self._buffer.write(b'\xd7')
	986	self._buffer.write(b"\xd7")
951	987	elif L == 16:
952		self._buffer.write(b'\xd8')
953		elif L <= 0xff:
954		self._buffer.write(b'\xc7' + struct.pack('B', L))
955		elif L <= 0xffff:
956		self._buffer.write(b'\xc8' + struct.pack('>H', L))
	988	self._buffer.write(b"\xd8")
	989	elif L <= 0xFF:
	990	self._buffer.write(b"\xc7" + struct.pack("B", L))
	991	elif L <= 0xFFFF:
	992	self._buffer.write(b"\xc8" + struct.pack(">H", L))
957	993	else:
958		self._buffer.write(b'\xc9' + struct.pack('>I', L))
959		self._buffer.write(struct.pack('B', typecode))
	994	self._buffer.write(b"\xc9" + struct.pack(">I", L))
	995	self._buffer.write(struct.pack("B", typecode))
960	996	self._buffer.write(data)
961	997
962	998	def _pack_array_header(self, n):
963		if n <= 0x0f:
964		return self._buffer.write(struct.pack('B', 0x90 + n))
965		if n <= 0xffff:
966		return self._buffer.write(struct.pack(">BH", 0xdc, n))
967		if n <= 0xffffffff:
968		return self._buffer.write(struct.pack(">BI", 0xdd, n))
	999	if n <= 0x0F:
	1000	return self._buffer.write(struct.pack("B", 0x90 + n))
	1001	if n <= 0xFFFF:
	1002	return self._buffer.write(struct.pack(">BH", 0xDC, n))
	1003	if n <= 0xFFFFFFFF:
	1004	return self._buffer.write(struct.pack(">BI", 0xDD, n))
969	1005	raise ValueError("Array is too large")
970	1006
971	1007	def _pack_map_header(self, n):
972		if n <= 0x0f:
973		return self._buffer.write(struct.pack('B', 0x80 + n))
974		if n <= 0xffff:
975		return self._buffer.write(struct.pack(">BH", 0xde, n))
976		if n <= 0xffffffff:
977		return self._buffer.write(struct.pack(">BI", 0xdf, n))
	1008	if n <= 0x0F:
	1009	return self._buffer.write(struct.pack("B", 0x80 + n))
	1010	if n <= 0xFFFF:
	1011	return self._buffer.write(struct.pack(">BH", 0xDE, n))
	1012	if n <= 0xFFFFFFFF:
	1013	return self._buffer.write(struct.pack(">BI", 0xDF, n))
978	1014	raise ValueError("Dict is too large")
979	1015
980	1016	def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):

984	1020	self._pack(v, nest_limit - 1)
985	1021
986	1022	def _pack_raw_header(self, n):
987		if n <= 0x1f:
988		self._buffer.write(struct.pack('B', 0xa0 + n))
989		elif self._use_bin_type and n <= 0xff:
990		self._buffer.write(struct.pack('>BB', 0xd9, n))
991		elif n <= 0xffff:
992		self._buffer.write(struct.pack(">BH", 0xda, n))
993		elif n <= 0xffffffff:
994		self._buffer.write(struct.pack(">BI", 0xdb, n))
	1023	if n <= 0x1F:
	1024	self._buffer.write(struct.pack("B", 0xA0 + n))
	1025	elif self._use_bin_type and n <= 0xFF:
	1026	self._buffer.write(struct.pack(">BB", 0xD9, n))
	1027	elif n <= 0xFFFF:
	1028	self._buffer.write(struct.pack(">BH", 0xDA, n))
	1029	elif n <= 0xFFFFFFFF:
	1030	self._buffer.write(struct.pack(">BI", 0xDB, n))
995	1031	else:
996		raise ValueError('Raw is too large')
	1032	raise ValueError("Raw is too large")
997	1033
998	1034	def _pack_bin_header(self, n):
999	1035	if not self._use_bin_type:
1000	1036	return self._pack_raw_header(n)
1001		elif n <= 0xff:
1002		return self._buffer.write(struct.pack('>BB', 0xc4, n))
1003		elif n <= 0xffff:
1004		return self._buffer.write(struct.pack(">BH", 0xc5, n))
1005		elif n <= 0xffffffff:
1006		return self._buffer.write(struct.pack(">BI", 0xc6, n))
	1037	elif n <= 0xFF:
	1038	return self._buffer.write(struct.pack(">BB", 0xC4, n))
	1039	elif n <= 0xFFFF:
	1040	return self._buffer.write(struct.pack(">BH", 0xC5, n))
	1041	elif n <= 0xFFFFFFFF:
	1042	return self._buffer.write(struct.pack(">BI", 0xC6, n))
1007	1043	else:
1008		raise ValueError('Bin is too large')
	1044	raise ValueError("Bin is too large")
1009	1045
1010	1046	def bytes(self):
1011	1047	"""Return internal buffer contents as bytes object"""

1014	1050	def reset(self):
1015	1051	"""Reset internal buffer.
1016	1052
1017		This method is usaful only when autoreset=False.
	1053	This method is useful only when autoreset=False.
1018	1054	"""
1019	1055	self._buffer = StringIO()
1020	1056

+33

-0

msgpack/pack_template.h less more

758	758
759	759	}
760	760
	761	/*
	762	* Pack Timestamp extension type. Follows msgpack-c pack_template.h.
	763	*/
	764	static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uint32_t nanoseconds)
	765	{
	766	if ((seconds >> 34) == 0) {
	767	/* seconds is unsigned and fits in 34 bits */
	768	uint64_t data64 = ((uint64_t)nanoseconds << 34) \| (uint64_t)seconds;
	769	if ((data64 & 0xffffffff00000000L) == 0) {
	770	/* no nanoseconds and seconds is 32bits or smaller. timestamp32. */
	771	unsigned char buf[4];
	772	uint32_t data32 = (uint32_t)data64;
	773	msgpack_pack_ext(x, -1, 4);
	774	_msgpack_store32(buf, data32);
	775	msgpack_pack_raw_body(x, buf, 4);
	776	} else {
	777	/* timestamp64 */
	778	unsigned char buf[8];
	779	msgpack_pack_ext(x, -1, 8);
	780	_msgpack_store64(buf, data64);
	781	msgpack_pack_raw_body(x, buf, 8);
	782
	783	}
	784	} else {
	785	/* seconds is signed or >34bits */
	786	unsigned char buf[12];
	787	_msgpack_store32(&buf[0], nanoseconds);
	788	_msgpack_store64(&buf[4], seconds);
	789	msgpack_pack_ext(x, -1, 12);
	790	msgpack_pack_raw_body(x, buf, 12);
	791	}
	792	return 0;
	793	}
761	794
762	795
763	796	#undef msgpack_pack_append_buffer

+109

-13

msgpack/unpack.h less more

23	23	bool raw;
24	24	bool has_pairs_hook;
25	25	bool strict_map_key;
	26	int timestamp;
26	27	PyObject *object_hook;
27	28	PyObject *list_hook;
28	29	PyObject *ext_hook;
29		const char *encoding;
	30	PyObject *timestamp_t;
	31	PyObject *giga;
	32	PyObject *utc;
30	33	const char *unicode_errors;
31	34	Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len;
32	35	} unpack_user;

191	194	if (u->strict_map_key && !PyUnicode_CheckExact(k) && !PyBytes_CheckExact(k)) {
192	195	PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name);
193	196	return -1;
	197	}
	198	if (PyUnicode_CheckExact(k)) {
	199	PyUnicode_InternInPlace(&k);
194	200	}
195	201	if (u->has_pairs_hook) {
196	202	msgpack_unpack_object item = PyTuple_Pack(2, k, v);

231	237
232	238	PyObject *py;
233	239
234		if (u->encoding) {
235		py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
236		} else if (u->raw) {
	240	if (u->raw) {
237	241	py = PyBytes_FromStringAndSize(p, l);
238	242	} else {
239	243	py = PyUnicode_DecodeUTF8(p, l, u->unicode_errors);

258	262	return 0;
259	263	}
260	264
261		static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
262		unsigned int length, msgpack_unpack_object* o)
263		{
264		PyObject *py;
	265	typedef struct msgpack_timestamp {
	266	int64_t tv_sec;
	267	uint32_t tv_nsec;
	268	} msgpack_timestamp;
	269
	270	/*
	271	* Unpack ext buffer to a timestamp. Pulled from msgpack-c timestamp.h.
	272	*/
	273	static int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timestamp* ts) {
	274	switch (buflen) {
	275	case 4:
	276	ts->tv_nsec = 0;
	277	{
	278	uint32_t v = _msgpack_load32(uint32_t, buf);
	279	ts->tv_sec = (int64_t)v;
	280	}
	281	return 0;
	282	case 8: {
	283	uint64_t value =_msgpack_load64(uint64_t, buf);
	284	ts->tv_nsec = (uint32_t)(value >> 34);
	285	ts->tv_sec = value & 0x00000003ffffffffLL;
	286	return 0;
	287	}
	288	case 12:
	289	ts->tv_nsec = _msgpack_load32(uint32_t, buf);
	290	ts->tv_sec = _msgpack_load64(int64_t, buf + 4);
	291	return 0;
	292	default:
	293	return -1;
	294	}
	295	}
	296
	297	#include "datetime.h"
	298
	299	static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
	300	unsigned int length, msgpack_unpack_object* o)
	301	{
265	302	int8_t typecode = (int8_t)*pos++;
266	303	if (!u->ext_hook) {
267	304	PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL");

271	308	PyErr_Format(PyExc_ValueError, "%u exceeds max_ext_len(%zd)", length, u->max_ext_len);
272	309	return -1;
273	310	}
	311
	312	PyObject *py = NULL;
274	313	// length also includes the typecode, so the actual data is length-1
275		#if PY_MAJOR_VERSION == 2
276		py = PyObject_CallFunction(u->ext_hook, "(is#)", (int)typecode, pos, (Py_ssize_t)length-1);
277		#else
278		py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1);
279		#endif
	314	if (typecode == -1) {
	315	msgpack_timestamp ts;
	316	if (unpack_timestamp(pos, length-1, &ts) < 0) {
	317	return -1;
	318	}
	319
	320	if (u->timestamp == 2) { // int
	321	PyObject *a = PyLong_FromLongLong(ts.tv_sec);
	322	if (a == NULL) return -1;
	323
	324	PyObject *c = PyNumber_Multiply(a, u->giga);
	325	Py_DECREF(a);
	326	if (c == NULL) {
	327	return -1;
	328	}
	329
	330	PyObject *b = PyLong_FromUnsignedLong(ts.tv_nsec);
	331	if (b == NULL) {
	332	Py_DECREF(c);
	333	return -1;
	334	}
	335
	336	py = PyNumber_Add(c, b);
	337	Py_DECREF(c);
	338	Py_DECREF(b);
	339	}
	340	else if (u->timestamp == 0) { // Timestamp
	341	py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec);
	342	}
	343	else { // float or datetime
	344	PyObject *a = PyFloat_FromDouble((double)ts.tv_nsec);
	345	if (a == NULL) return -1;
	346
	347	PyObject *b = PyNumber_TrueDivide(a, u->giga);
	348	Py_DECREF(a);
	349	if (b == NULL) return -1;
	350
	351	PyObject *c = PyLong_FromLongLong(ts.tv_sec);
	352	if (c == NULL) {
	353	Py_DECREF(b);
	354	return -1;
	355	}
	356
	357	a = PyNumber_Add(b, c);
	358	Py_DECREF(b);
	359	Py_DECREF(c);
	360
	361	if (u->timestamp == 3) { // datetime
	362	PyObject *t = PyTuple_Pack(2, a, u->utc);
	363	Py_DECREF(a);
	364	if (t == NULL) {
	365	return -1;
	366	}
	367	py = PyDateTime_FromTimestamp(t);
	368	Py_DECREF(t);
	369	} else { // float
	370	py = a;
	371	}
	372	}
	373	} else {
	374	py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1);
	375	}
280	376	if (!py)
281	377	return -1;
282	378	*o = py;

+70

-66

setup.py less more

8	8
9	9	from distutils.command.build_ext import build_ext
10	10
11		# for building transitional package.
12		TRANSITIONAL = False
	11
	12	PYPY = hasattr(sys, "pypy_version_info")
	13	PY2 = sys.version_info[0] == 2
	14
13	15
14	16	class NoCython(Exception):
15	17	pass
16	18
	19
17	20	try:
18	21	import Cython.Compiler.Main as cython_compiler
	22
19	23	have_cython = True
20	24	except ImportError:
21	25	have_cython = False

25	29	sys.stderr.write("cythonize: %r\n" % (src,))
26	30	cython_compiler.compile([src], cplus=True)
27	31
	32
28	33	def ensure_source(src):
29		pyx = os.path.splitext(src)[0] + '.pyx'
	34	pyx = os.path.splitext(src)[0] + ".pyx"
30	35
31	36	if not os.path.exists(src):
32	37	if not have_cython:
33	38	raise NoCython
34	39	cythonize(pyx)
35		elif (os.path.exists(pyx) and
36		os.stat(src).st_mtime < os.stat(pyx).st_mtime and
37		have_cython):
	40	elif (
	41	os.path.exists(pyx)
	42	and os.stat(src).st_mtime < os.stat(pyx).st_mtime
	43	and have_cython
	44	):
38	45	cythonize(pyx)
39	46	return src
40	47

57	64	print(e)
58	65
59	66
60		exec(open('msgpack/_version.py').read())
	67	exec(open("msgpack/_version.py").read())
61	68
62		version_str = '.'.join(str(x) for x in version[:3])
63		if len(version) > 3 and version[3] != 'final':
	69	version_str = ".".join(str(x) for x in version[:3])
	70	if len(version) > 3 and version[3] != "final":
64	71	version_str += version[3]
65	72
66		# take care of extension modules.
67		if have_cython:
68		class Sdist(sdist):
69		def __init__(self, args, *kwargs):
70		cythonize('msgpack/_cmsgpack.pyx')
71		sdist.__init__(self, args, *kwargs)
72		else:
73		Sdist = sdist
	73	# Cython is required for sdist
	74	class Sdist(sdist):
	75	def __init__(self, args, *kwargs):
	76	cythonize("msgpack/_cmsgpack.pyx")
	77	sdist.__init__(self, args, *kwargs)
	78
74	79
75	80	libraries = []
76		if sys.platform == 'win32':
77		libraries.append('ws2_32')
	81	if sys.platform == "win32":
	82	libraries.append("ws2_32")
78	83
79		if sys.byteorder == 'big':
80		macros = [('__BIG_ENDIAN__', '1')]
	84	if sys.byteorder == "big":
	85	macros = [("__BIG_ENDIAN__", "1")]
81	86	else:
82		macros = [('__LITTLE_ENDIAN__', '1')]
	87	macros = [("__LITTLE_ENDIAN__", "1")]
83	88
84	89	ext_modules = []
85		if not hasattr(sys, 'pypy_version_info'):
86		ext_modules.append(Extension('msgpack._cmsgpack',
87		sources=['msgpack/_cmsgpack.cpp'],
88		libraries=libraries,
89		include_dirs=['.'],
90		define_macros=macros,
91		))
	90	if not PYPY and not PY2:
	91	ext_modules.append(
	92	Extension(
	93	"msgpack._cmsgpack",
	94	sources=["msgpack/_cmsgpack.cpp"],
	95	libraries=libraries,
	96	include_dirs=["."],
	97	define_macros=macros,
	98	)
	99	)
92	100	del libraries, macros
93	101
94	102
95		desc = 'MessagePack (de)serializer.'
96		with io.open('README.rst', encoding='utf-8') as f:
	103	desc = "MessagePack (de)serializer."
	104	with io.open("README.md", encoding="utf-8") as f:
97	105	long_desc = f.read()
98	106	del f
99	107
100		name = 'msgpack'
101
102		if TRANSITIONAL:
103		name = 'msgpack-python'
104		long_desc = "This package is deprecated. Install msgpack instead."
105
106		setup(name=name,
107		author='INADA Naoki',
108		author_email='songofacandy@gmail.com',
109		version=version_str,
110		cmdclass={'build_ext': BuildExt, 'sdist': Sdist},
111		ext_modules=ext_modules,
112		packages=['msgpack'],
113		description=desc,
114		long_description=long_desc,
115		long_description_content_type="text/x-rst",
116		url='https://msgpack.org/',
117		project_urls = {
118		'Documentation': 'https://msgpack-python.readthedocs.io/',
119		'Source': 'https://github.com/msgpack/msgpack-python',
120		'Tracker': 'https://github.com/msgpack/msgpack-python/issues',
121		},
122		license='Apache 2.0',
123		classifiers=[
124		'Programming Language :: Python :: 2',
125		'Programming Language :: Python :: 2.7',
126		'Programming Language :: Python :: 3',
127		'Programming Language :: Python :: 3.5',
128		'Programming Language :: Python :: 3.6',
129		'Programming Language :: Python :: 3.7',
130		'Programming Language :: Python :: Implementation :: CPython',
131		'Programming Language :: Python :: Implementation :: PyPy',
132		'Intended Audience :: Developers',
133		'License :: OSI Approved :: Apache Software License',
134		],
	108	setup(
	109	name="msgpack",
	110	author="Inada Naoki",
	111	author_email="songofacandy@gmail.com",
	112	version=version_str,
	113	cmdclass={"build_ext": BuildExt, "sdist": Sdist},
	114	ext_modules=ext_modules,
	115	packages=["msgpack"],
	116	description=desc,
	117	long_description=long_desc,
	118	long_description_content_type="text/markdown",
	119	url="https://msgpack.org/",
	120	project_urls={
	121	"Documentation": "https://msgpack-python.readthedocs.io/",
	122	"Source": "https://github.com/msgpack/msgpack-python",
	123	"Tracker": "https://github.com/msgpack/msgpack-python/issues",
	124	},
	125	license="Apache 2.0",
	126	classifiers=[
	127	"Programming Language :: Python :: 2",
	128	"Programming Language :: Python :: 2.7",
	129	"Programming Language :: Python :: 3",
	130	"Programming Language :: Python :: 3.5",
	131	"Programming Language :: Python :: 3.6",
	132	"Programming Language :: Python :: 3.7",
	133	"Programming Language :: Python :: 3.8",
	134	"Programming Language :: Python :: Implementation :: CPython",
	135	"Programming Language :: Python :: Implementation :: PyPy",
	136	"Intended Audience :: Developers",
	137	"License :: OSI Approved :: Apache Software License",
	138	],
135	139	)

+11

-10

test/test_buffer.py less more

0	0	#!/usr/bin/env python
1	1	# coding: utf-8
2	2
	3	import sys
	4	import pytest
3	5	from msgpack import packb, unpackb
4	6
5	7
	8	@pytest.mark.skipif(sys.version_info[0] == 2, reason="Python 2 is not supported")
6	9	def test_unpack_buffer():
7	10	from array import array
8		buf = array('b')
9		try:
10		buf.frombytes(packb((b'foo', b'bar')))
11		except AttributeError: # PY2
12		buf.fromstring(packb((b'foo', b'bar')))
	11
	12	buf = array("b")
	13	buf.frombytes(packb((b"foo", b"bar")))
13	14	obj = unpackb(buf, use_list=1)
14		assert [b'foo', b'bar'] == obj
	15	assert [b"foo", b"bar"] == obj
15	16
16	17
17	18	def test_unpack_bytearray():
18		buf = bytearray(packb(('foo', 'bar')))
	19	buf = bytearray(packb((b"foo", b"bar")))
19	20	obj = unpackb(buf, use_list=1)
20		assert [b'foo', b'bar'] == obj
	21	assert [b"foo", b"bar"] == obj
21	22	expected_type = bytes
22	23	assert all(type(s) == expected_type for s in obj)
23	24
24	25
25	26	def test_unpack_memoryview():
26		buf = bytearray(packb(('foo', 'bar')))
	27	buf = bytearray(packb((b"foo", b"bar")))
27	28	view = memoryview(buf)
28	29	obj = unpackb(view, use_list=1)
29		assert [b'foo', b'bar'] == obj
	30	assert [b"foo", b"bar"] == obj
30	31	expected_type = bytes
31	32	assert all(type(s) == expected_type for s in obj)

+81

-46

test/test_case.py less more

0	0	#!/usr/bin/env python
1	1	# coding: utf-8
2
3	2	from msgpack import packb, unpackb
4	3
5	4
6		def check(length, obj):
7		v = packb(obj)
8		assert len(v) == length, \
9		"%r length should be %r but get %r" % (obj, length, len(v))
10		assert unpackb(v, use_list=0) == obj
	5	def check(length, obj, use_bin_type=True):
	6	v = packb(obj, use_bin_type=use_bin_type)
	7	assert len(v) == length, "%r length should be %r but get %r" % (obj, length, len(v))
	8	assert unpackb(v, use_list=0, raw=not use_bin_type) == obj
	9
11	10
12	11	def test_1():
13		for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
14		-((1<<5)-1), -(1<<5)]:
	12	for o in [
	13	None,
	14	True,
	15	False,
	16	0,
	17	1,
	18	(1 << 6),
	19	(1 << 7) - 1,
	20	-1,
	21	-((1 << 5) - 1),
	22	-(1 << 5),
	23	]:
15	24	check(1, o)
16	25
	26
17	27	def test_2():
18		for o in [1 << 7, (1 << 8) - 1,
19		-((1<<5)+1), -(1<<7)
20		]:
	28	for o in [1 << 7, (1 << 8) - 1, -((1 << 5) + 1), -(1 << 7)]:
21	29	check(2, o)
22	30
	31
23	32	def test_3():
24		for o in [1 << 8, (1 << 16) - 1,
25		-((1<<7)+1), -(1<<15)]:
	33	for o in [1 << 8, (1 << 16) - 1, -((1 << 7) + 1), -(1 << 15)]:
26	34	check(3, o)
27	35
	36
28	37	def test_5():
29		for o in [1 << 16, (1 << 32) - 1,
30		-((1<<15)+1), -(1<<31)]:
	38	for o in [1 << 16, (1 << 32) - 1, -((1 << 15) + 1), -(1 << 31)]:
31	39	check(5, o)
32	40
	41
33	42	def test_9():
34		for o in [1 << 32, (1 << 64) - 1,
35		-((1<<31)+1), -(1<<63),
36		1.0, 0.1, -0.1, -1.0]:
	43	for o in [
	44	1 << 32,
	45	(1 << 64) - 1,
	46	-((1 << 31) + 1),
	47	-(1 << 63),
	48	1.0,
	49	0.1,
	50	-0.1,
	51	-1.0,
	52	]:
37	53	check(9, o)
38	54
39	55
40	56	def check_raw(overhead, num):
41		check(num + overhead, b" " * num)
	57	check(num + overhead, b" " * num, use_bin_type=False)
	58
42	59
43	60	def test_fixraw():
44	61	check_raw(1, 0)
45		check_raw(1, (1<<5) - 1)
	62	check_raw(1, (1 << 5) - 1)
	63
46	64
47	65	def test_raw16():
48		check_raw(3, 1<<5)
49		check_raw(3, (1<<16) - 1)
	66	check_raw(3, 1 << 5)
	67	check_raw(3, (1 << 16) - 1)
	68
50	69
51	70	def test_raw32():
52		check_raw(5, 1<<16)
	71	check_raw(5, 1 << 16)
53	72
54	73
55	74	def check_array(overhead, num):
56	75	check(num + overhead, (None,) * num)
57	76
	77
58	78	def test_fixarray():
59	79	check_array(1, 0)
60	80	check_array(1, (1 << 4) - 1)
61	81
	82
62	83	def test_array16():
63	84	check_array(3, 1 << 4)
64		check_array(3, (1<<16)-1)
	85	check_array(3, (1 << 16) - 1)
	86
65	87
66	88	def test_array32():
67		check_array(5, (1<<16))
	89	check_array(5, (1 << 16))
68	90
69	91
70	92	def match(obj, buf):
71	93	assert packb(obj) == buf
72		assert unpackb(buf, use_list=0) == obj
	94	assert unpackb(buf, use_list=0, strict_map_key=False) == obj
	95
73	96
74	97	def test_match():
75	98	cases = [
76		(None, b'\xc0'),
77		(False, b'\xc2'),
78		(True, b'\xc3'),
79		(0, b'\x00'),
80		(127, b'\x7f'),
81		(128, b'\xcc\x80'),
82		(256, b'\xcd\x01\x00'),
83		(-1, b'\xff'),
84		(-33, b'\xd0\xdf'),
85		(-129, b'\xd1\xff\x7f'),
86		({1:1}, b'\x81\x01\x01'),
	99	(None, b"\xc0"),
	100	(False, b"\xc2"),
	101	(True, b"\xc3"),
	102	(0, b"\x00"),
	103	(127, b"\x7f"),
	104	(128, b"\xcc\x80"),
	105	(256, b"\xcd\x01\x00"),
	106	(-1, b"\xff"),
	107	(-33, b"\xd0\xdf"),
	108	(-129, b"\xd1\xff\x7f"),
	109	({1: 1}, b"\x81\x01\x01"),
87	110	(1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
88		((), b'\x90'),
89		(tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"),
90		(tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"),
91		({}, b'\x80'),
92		(dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'),
93		(dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'),
94		]
	111	((), b"\x90"),
	112	(
	113	tuple(range(15)),
	114	b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e",
	115	),
	116	(
	117	tuple(range(16)),
	118	b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
	119	),
	120	({}, b"\x80"),
	121	(
	122	dict([(x, x) for x in range(15)]),
	123	b"\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e",
	124	),
	125	(
	126	dict([(x, x) for x in range(16)]),
	127	b"\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f",
	128	),
	129	]
95	130
96	131	for v, p in cases:
97	132	match(v, p)
98	133
	134
99	135	def test_unicode():
100		assert unpackb(packb('foobar'), use_list=1) == b'foobar'
101
	136	assert unpackb(packb(u"foobar"), use_list=1) == u"foobar"

+31

-24

test/test_extension.py less more

8	8	packer = msgpack.Packer()
9	9	packer.pack_ext_type(0x42, s)
10	10	return packer.bytes()
11		assert p(b'A') == b'\xd4\x42A' # fixext 1
12		assert p(b'AB') == b'\xd5\x42AB' # fixext 2
13		assert p(b'ABCD') == b'\xd6\x42ABCD' # fixext 4
14		assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH' # fixext 8
15		assert p(b'A'16) == b'\xd8\x42' + b'A'16 # fixext 16
16		assert p(b'ABC') == b'\xc7\x03\x42ABC' # ext 8
17		assert p(b'A'0x0123) == b'\xc8\x01\x23\x42' + b'A'0x0123 # ext 16
18		assert p(b'A'0x00012345) == b'\xc9\x00\x01\x23\x45\x42' + b'A'0x00012345 # ext 32
	11
	12	assert p(b"A") == b"\xd4\x42A" # fixext 1
	13	assert p(b"AB") == b"\xd5\x42AB" # fixext 2
	14	assert p(b"ABCD") == b"\xd6\x42ABCD" # fixext 4
	15	assert p(b"ABCDEFGH") == b"\xd7\x42ABCDEFGH" # fixext 8
	16	assert p(b"A" * 16) == b"\xd8\x42" + b"A" * 16 # fixext 16
	17	assert p(b"ABC") == b"\xc7\x03\x42ABC" # ext 8
	18	assert p(b"A" * 0x0123) == b"\xc8\x01\x23\x42" + b"A" * 0x0123 # ext 16
	19	assert (
	20	p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345
	21	) # ext 32
19	22
20	23
21	24	def test_unpack_ext_type():
22	25	def check(b, expected):
23	26	assert msgpack.unpackb(b) == expected
24	27
25		check(b'\xd4\x42A', ExtType(0x42, b'A')) # fixext 1
26		check(b'\xd5\x42AB', ExtType(0x42, b'AB')) # fixext 2
27		check(b'\xd6\x42ABCD', ExtType(0x42, b'ABCD')) # fixext 4
28		check(b'\xd7\x42ABCDEFGH', ExtType(0x42, b'ABCDEFGH')) # fixext 8
29		check(b'\xd8\x42' + b'A'16, ExtType(0x42, b'A'16)) # fixext 16
30		check(b'\xc7\x03\x42ABC', ExtType(0x42, b'ABC')) # ext 8
31		check(b'\xc8\x01\x23\x42' + b'A'*0x0123,
32		ExtType(0x42, b'A'*0x0123)) # ext 16
33		check(b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345,
34		ExtType(0x42, b'A'*0x00012345)) # ext 32
	28	check(b"\xd4\x42A", ExtType(0x42, b"A")) # fixext 1
	29	check(b"\xd5\x42AB", ExtType(0x42, b"AB")) # fixext 2
	30	check(b"\xd6\x42ABCD", ExtType(0x42, b"ABCD")) # fixext 4
	31	check(b"\xd7\x42ABCDEFGH", ExtType(0x42, b"ABCDEFGH")) # fixext 8
	32	check(b"\xd8\x42" + b"A" * 16, ExtType(0x42, b"A" * 16)) # fixext 16
	33	check(b"\xc7\x03\x42ABC", ExtType(0x42, b"ABC")) # ext 8
	34	check(b"\xc8\x01\x23\x42" + b"A" * 0x0123, ExtType(0x42, b"A" * 0x0123)) # ext 16
	35	check(
	36	b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345,
	37	ExtType(0x42, b"A" * 0x00012345),
	38	) # ext 32
35	39
36	40
37	41	def test_extension_type():
38	42	def default(obj):
39		print('default called', obj)
	43	print("default called", obj)
40	44	if isinstance(obj, array.array):
41		typecode = 123 # application specific typecode
	45	typecode = 123 # application specific typecode
42	46	try:
43	47	data = obj.tobytes()
44	48	except AttributeError:

47	51	raise TypeError("Unknown type object %r" % (obj,))
48	52
49	53	def ext_hook(code, data):
50		print('ext_hook called', code, data)
	54	print("ext_hook called", code, data)
51	55	assert code == 123
52		obj = array.array('d')
	56	obj = array.array("d")
53	57	try:
54	58	obj.frombytes(data)
55	59	except AttributeError: # PY2
56	60	obj.fromstring(data)
57	61	return obj
58	62
59		obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
	63	obj = [42, b"hello", array.array("d", [1.1, 2.2, 3.3])]
60	64	s = msgpack.packb(obj, default=default)
61	65	obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
62	66	assert obj == obj2
63	67
	68
64	69	import sys
65		if sys.version > '3':
	70
	71	if sys.version > "3":
66	72	long = int
	73
67	74
68	75	def test_overriding_hooks():
69	76	def default(obj):

+59

-38

test/test_format.py less more

2	2
3	3	from msgpack import unpackb
4	4
5		def check(src, should, use_list=0):
6		assert unpackb(src, use_list=use_list) == should
	5
	6	def check(src, should, use_list=0, raw=True):
	7	assert unpackb(src, use_list=use_list, raw=raw, strict_map_key=False) == should
	8
7	9
8	10	def testSimpleValue():
9		check(b"\x93\xc0\xc2\xc3",
10		(None, False, True,))
	11	check(b"\x93\xc0\xc2\xc3", (None, False, True))
	12
11	13
12	14	def testFixnum():
13		check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff",
14		((0,64,127,), (-32,-16,-1,),)
15		)
	15	check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0, 64, 127), (-32, -16, -1)))
	16
16	17
17	18	def testFixArray():
18		check(b"\x92\x90\x91\x91\xc0",
19		((),((None,),),),
20		)
	19	check(b"\x92\x90\x91\x91\xc0", ((), ((None,),)))
	20
21	21
22	22	def testFixRaw():
23		check(b"\x94\xa0\xa1a\xa2bc\xa3def",
24		(b"", b"a", b"bc", b"def",),
25		)
	23	check(b"\x94\xa0\xa1a\xa2bc\xa3def", (b"", b"a", b"bc", b"def"))
	24
26	25
27	26	def testFixMap():
28	27	check(
29		b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
30		{False: {None: None}, True:{None:{}}},
31		)
	28	b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}}
	29	)
	30
32	31
33	32	def testUnsignedInt():
34	33	check(
35		b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
36		b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
37		b"\xce\xff\xff\xff\xff",
38		(0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,),
39		)
	34	b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
	35	b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
	36	b"\xce\xff\xff\xff\xff",
	37	(0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295),
	38	)
	39
40	40
41	41	def testSignedInt():
42		check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
43		b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
44		b"\xd2\xff\xff\xff\xff",
45		(0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,))
	42	check(
	43	b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
	44	b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
	45	b"\xd2\xff\xff\xff\xff",
	46	(0, -128, -1, 0, -32768, -1, 0, -2147483648, -1),
	47	)
	48
46	49
47	50	def testRaw():
48		check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
	51	check(
	52	b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
49	53	b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
50		(b"", b"a", b"ab", b"", b"a", b"ab"))
	54	(b"", b"a", b"ab", b"", b"a", b"ab"),
	55	)
	56	check(
	57	b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
	58	b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
	59	("", "a", "ab", "", "a", "ab"),
	60	raw=False,
	61	)
	62
51	63
52	64	def testArray():
53		check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
	65	check(
	66	b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
54	67	b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
55	68	b"\xc2\xc3",
56		((), (None,), (False,True), (), (None,), (False,True))
57		)
	69	((), (None,), (False, True), (), (None,), (False, True)),
	70	)
	71
58	72
59	73	def testMap():
60	74	check(
61	75	b"\x96"
62		b"\xde\x00\x00"
63		b"\xde\x00\x01\xc0\xc2"
64		b"\xde\x00\x02\xc0\xc2\xc3\xc2"
65		b"\xdf\x00\x00\x00\x00"
66		b"\xdf\x00\x00\x00\x01\xc0\xc2"
67		b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
68		({}, {None: False}, {True: False, None: False}, {},
69		{None: False}, {True: False, None: False}))
	76	b"\xde\x00\x00"
	77	b"\xde\x00\x01\xc0\xc2"
	78	b"\xde\x00\x02\xc0\xc2\xc3\xc2"
	79	b"\xdf\x00\x00\x00\x00"
	80	b"\xdf\x00\x00\x00\x01\xc0\xc2"
	81	b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
	82	(
	83	{},
	84	{None: False},
	85	{True: False, None: False},
	86	{},
	87	{None: False},
	88	{True: False, None: False},
	89	),
	90	)

+29

-21

test/test_limits.py less more

3	3	import pytest
4	4
5	5	from msgpack import (
6		packb, unpackb, Packer, Unpacker, ExtType,
7		PackOverflowError, PackValueError, UnpackValueError,
	6	packb,
	7	unpackb,
	8	Packer,
	9	Unpacker,
	10	ExtType,
	11	PackOverflowError,
	12	PackValueError,
	13	UnpackValueError,
8	14	)
9	15
10	16

12	18	x = -(2 ** 63)
13	19	assert unpackb(packb(x)) == x
14	20	with pytest.raises(PackOverflowError):
15		packb(x-1)
	21	packb(x - 1)
16	22
17	23	x = 2 ** 64 - 1
18	24	assert unpackb(packb(x)) == x
19	25	with pytest.raises(PackOverflowError):
20		packb(x+1)
	26	packb(x + 1)
21	27
22	28
23	29	def test_array_header():
24	30	packer = Packer()
25		packer.pack_array_header(2**32-1)
	31	packer.pack_array_header(2 ** 32 - 1)
26	32	with pytest.raises(PackValueError):
27		packer.pack_array_header(2**32)
	33	packer.pack_array_header(2 ** 32)
28	34
29	35
30	36	def test_map_header():
31	37	packer = Packer()
32		packer.pack_map_header(2**32-1)
	38	packer.pack_map_header(2 ** 32 - 1)
33	39	with pytest.raises(PackValueError):
34		packer.pack_array_header(2**32)
	40	packer.pack_array_header(2 ** 32)
35	41
36	42
37	43	def test_max_str_len():
38		d = 'x' * 3
	44	d = "x" * 3
39	45	packed = packb(d)
40	46
41	47	unpacker = Unpacker(max_str_len=3, raw=False)

49	55
50	56
51	57	def test_max_bin_len():
52		d = b'x' * 3
	58	d = b"x" * 3
53	59	packed = packb(d, use_bin_type=True)
54	60
55	61	unpacker = Unpacker(max_bin_len=3)

63	69
64	70
65	71	def test_max_array_len():
66		d = [1,2,3]
	72	d = [1, 2, 3]
67	73	packed = packb(d)
68	74
69	75	unpacker = Unpacker(max_array_len=3)

80	86	d = {1: 2, 3: 4, 5: 6}
81	87	packed = packb(d)
82	88
83		unpacker = Unpacker(max_map_len=3)
	89	unpacker = Unpacker(max_map_len=3, strict_map_key=False)
84	90	unpacker.feed(packed)
85	91	assert unpacker.unpack() == d
86	92
87		unpacker = Unpacker(max_map_len=2)
	93	unpacker = Unpacker(max_map_len=2, strict_map_key=False)
88	94	with pytest.raises(UnpackValueError):
89	95	unpacker.feed(packed)
90	96	unpacker.unpack()

106	112
107	113	# PyPy fails following tests because of constant folding?
108	114	# https://bugs.pypy.org/issue1721
109		#@pytest.mark.skipif(True, reason="Requires very large memory.")
110		#def test_binary():
	115	# @pytest.mark.skipif(True, reason="Requires very large memory.")
	116	# def test_binary():
111	117	# x = b'x' * (2**32 - 1)
112	118	# assert unpackb(packb(x)) == x
113	119	# del x

116	122	# packb(x)
117	123	#
118	124	#
119		#@pytest.mark.skipif(True, reason="Requires very large memory.")
120		#def test_string():
	125	# @pytest.mark.skipif(True, reason="Requires very large memory.")
	126	# def test_string():
121	127	# x = 'x' * (2**32 - 1)
122	128	# assert unpackb(packb(x)) == x
123	129	# x += 'y'

125	131	# packb(x)
126	132	#
127	133	#
128		#@pytest.mark.skipif(True, reason="Requires very large memory.")
129		#def test_array():
	134	# @pytest.mark.skipif(True, reason="Requires very large memory.")
	135	# def test_array():
130	136	# x = [0] * (2**32 - 1)
131	137	# assert unpackb(packb(x)) == x
132	138	# x.append(0)

136	142
137	143	# auto max len
138	144
	145
139	146	def test_auto_max_array_len():
140		packed = b'\xde\x00\x06zz'
	147	packed = b"\xde\x00\x06zz"
141	148	with pytest.raises(UnpackValueError):
142	149	unpackb(packed, raw=False)
143	150

146	153	with pytest.raises(UnpackValueError):
147	154	unpacker.unpack()
148	155
	156
149	157	def test_auto_max_map_len():
150	158	# len(packed) == 6 -> max_map_len == 3
151		packed = b'\xde\x00\x04zzz'
	159	packed = b"\xde\x00\x04zzz"
152	160	with pytest.raises(UnpackValueError):
153	161	unpackb(packed, raw=False)
154	162

+33

-47

test/test_memoryview.py less more

0	0	#!/usr/bin/env python
1	1	# coding: utf-8
2	2
	3	import pytest
3	4	from array import array
4	5	from msgpack import packb, unpackb
5	6	import sys
6	7
7	8
8		# For Python < 3:
9		# - array type only supports old buffer interface
10		# - array.frombytes is not available, must use deprecated array.fromstring
11		if sys.version_info[0] < 3:
12		def make_memoryview(obj):
13		return memoryview(buffer(obj))
	9	pytestmark = pytest.mark.skipif(
	10	sys.version_info[0] < 3, reason="Only Python 3 supports buffer protocol"
	11	)
14	12
15		def make_array(f, data):
16		a = array(f)
17		a.fromstring(data)
18		return a
19	13
20		def get_data(a):
21		return a.tostring()
22		else:
23		make_memoryview = memoryview
24
25		def make_array(f, data):
26		a = array(f)
27		a.frombytes(data)
28		return a
29
30		def get_data(a):
31		return a.tobytes()
	14	def make_array(f, data):
	15	a = array(f)
	16	a.frombytes(data)
	17	return a
32	18
33	19
34	20	def _runtest(format, nbytes, expected_header, expected_prefix, use_bin_type):
35	21	# create a new array
36	22	original_array = array(format)
37	23	original_array.fromlist([255] * (nbytes // original_array.itemsize))
38		original_data = get_data(original_array)
39		view = make_memoryview(original_array)
	24	original_data = original_array.tobytes()
	25	view = memoryview(original_array)
40	26
41	27	# pack, unpack, and reconstruct array
42	28	packed = packb(view, use_bin_type=use_bin_type)
43		unpacked = unpackb(packed)
	29	unpacked = unpackb(packed, raw=(not use_bin_type))
44	30	reconstructed_array = make_array(format, unpacked)
45	31
46	32	# check that we got the right amount of data

48	34	# check packed header
49	35	assert packed[:1] == expected_header
50	36	# check packed length prefix, if any
51		assert packed[1:1+len(expected_prefix)] == expected_prefix
	37	assert packed[1 : 1 + len(expected_prefix)] == expected_prefix
52	38	# check packed data
53		assert packed[1+len(expected_prefix):] == original_data
	39	assert packed[1 + len(expected_prefix) :] == original_data
54	40	# check array unpacked correctly
55	41	assert original_array == reconstructed_array
56	42
57	43
58	44	def test_fixstr_from_byte():
59		_runtest('B', 1, b'\xa1', b'', False)
60		_runtest('B', 31, b'\xbf', b'', False)
	45	_runtest("B", 1, b"\xa1", b"", False)
	46	_runtest("B", 31, b"\xbf", b"", False)
61	47
62	48
63	49	def test_fixstr_from_float():
64		_runtest('f', 4, b'\xa4', b'', False)
65		_runtest('f', 28, b'\xbc', b'', False)
	50	_runtest("f", 4, b"\xa4", b"", False)
	51	_runtest("f", 28, b"\xbc", b"", False)
66	52
67	53
68	54	def test_str16_from_byte():
69		_runtest('B', 2**8, b'\xda', b'\x01\x00', False)
70		_runtest('B', 2**16-1, b'\xda', b'\xff\xff', False)
	55	_runtest("B", 2 ** 8, b"\xda", b"\x01\x00", False)
	56	_runtest("B", 2 ** 16 - 1, b"\xda", b"\xff\xff", False)
71	57
72	58
73	59	def test_str16_from_float():
74		_runtest('f', 2**8, b'\xda', b'\x01\x00', False)
75		_runtest('f', 2**16-4, b'\xda', b'\xff\xfc', False)
	60	_runtest("f", 2 ** 8, b"\xda", b"\x01\x00", False)
	61	_runtest("f", 2 ** 16 - 4, b"\xda", b"\xff\xfc", False)
76	62
77	63
78	64	def test_str32_from_byte():
79		_runtest('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
	65	_runtest("B", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False)
80	66
81	67
82	68	def test_str32_from_float():
83		_runtest('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
	69	_runtest("f", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False)
84	70
85	71
86	72	def test_bin8_from_byte():
87		_runtest('B', 1, b'\xc4', b'\x01', True)
88		_runtest('B', 2**8-1, b'\xc4', b'\xff', True)
	73	_runtest("B", 1, b"\xc4", b"\x01", True)
	74	_runtest("B", 2 ** 8 - 1, b"\xc4", b"\xff", True)
89	75
90	76
91	77	def test_bin8_from_float():
92		_runtest('f', 4, b'\xc4', b'\x04', True)
93		_runtest('f', 2**8-4, b'\xc4', b'\xfc', True)
	78	_runtest("f", 4, b"\xc4", b"\x04", True)
	79	_runtest("f", 2 ** 8 - 4, b"\xc4", b"\xfc", True)
94	80
95	81
96	82	def test_bin16_from_byte():
97		_runtest('B', 2**8, b'\xc5', b'\x01\x00', True)
98		_runtest('B', 2**16-1, b'\xc5', b'\xff\xff', True)
	83	_runtest("B", 2 ** 8, b"\xc5", b"\x01\x00", True)
	84	_runtest("B", 2 ** 16 - 1, b"\xc5", b"\xff\xff", True)
99	85
100	86
101	87	def test_bin16_from_float():
102		_runtest('f', 2**8, b'\xc5', b'\x01\x00', True)
103		_runtest('f', 2**16-4, b'\xc5', b'\xff\xfc', True)
	88	_runtest("f", 2 ** 8, b"\xc5", b"\x01\x00", True)
	89	_runtest("f", 2 ** 16 - 4, b"\xc5", b"\xff\xfc", True)
104	90
105	91
106	92	def test_bin32_from_byte():
107		_runtest('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)
	93	_runtest("B", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True)
108	94
109	95
110	96	def test_bin32_from_float():
111		_runtest('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)
	97	_runtest("f", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True)

+45

-41

test/test_newspec.py less more

3	3
4	4
5	5	def test_str8():
6		header = b'\xd9'
7		data = b'x' * 32
	6	header = b"\xd9"
	7	data = b"x" * 32
8	8	b = packb(data.decode(), use_bin_type=True)
9	9	assert len(b) == len(data) + 2
10		assert b[0:2] == header + b'\x20'
	10	assert b[0:2] == header + b"\x20"
	11	assert b[2:] == data
	12	assert unpackb(b, raw=True) == data
	13	assert unpackb(b, raw=False) == data.decode()
	14
	15	data = b"x" * 255
	16	b = packb(data.decode(), use_bin_type=True)
	17	assert len(b) == len(data) + 2
	18	assert b[0:2] == header + b"\xff"
	19	assert b[2:] == data
	20	assert unpackb(b, raw=True) == data
	21	assert unpackb(b, raw=False) == data.decode()
	22
	23
	24	def test_bin8():
	25	header = b"\xc4"
	26	data = b""
	27	b = packb(data, use_bin_type=True)
	28	assert len(b) == len(data) + 2
	29	assert b[0:2] == header + b"\x00"
11	30	assert b[2:] == data
12	31	assert unpackb(b) == data
13	32
14		data = b'x' * 255
15		b = packb(data.decode(), use_bin_type=True)
16		assert len(b) == len(data) + 2
17		assert b[0:2] == header + b'\xff'
18		assert b[2:] == data
19		assert unpackb(b) == data
20
21
22		def test_bin8():
23		header = b'\xc4'
24		data = b''
	33	data = b"x" * 255
25	34	b = packb(data, use_bin_type=True)
26	35	assert len(b) == len(data) + 2
27		assert b[0:2] == header + b'\x00'
28		assert b[2:] == data
29		assert unpackb(b) == data
30
31		data = b'x' * 255
32		b = packb(data, use_bin_type=True)
33		assert len(b) == len(data) + 2
34		assert b[0:2] == header + b'\xff'
	36	assert b[0:2] == header + b"\xff"
35	37	assert b[2:] == data
36	38	assert unpackb(b) == data
37	39
38	40
39	41	def test_bin16():
40		header = b'\xc5'
41		data = b'x' * 256
	42	header = b"\xc5"
	43	data = b"x" * 256
42	44	b = packb(data, use_bin_type=True)
43	45	assert len(b) == len(data) + 3
44	46	assert b[0:1] == header
45		assert b[1:3] == b'\x01\x00'
	47	assert b[1:3] == b"\x01\x00"
46	48	assert b[3:] == data
47	49	assert unpackb(b) == data
48	50
49		data = b'x' * 65535
	51	data = b"x" * 65535
50	52	b = packb(data, use_bin_type=True)
51	53	assert len(b) == len(data) + 3
52	54	assert b[0:1] == header
53		assert b[1:3] == b'\xff\xff'
	55	assert b[1:3] == b"\xff\xff"
54	56	assert b[3:] == data
55	57	assert unpackb(b) == data
56	58
57	59
58	60	def test_bin32():
59		header = b'\xc6'
60		data = b'x' * 65536
	61	header = b"\xc6"
	62	data = b"x" * 65536
61	63	b = packb(data, use_bin_type=True)
62	64	assert len(b) == len(data) + 5
63	65	assert b[0:1] == header
64		assert b[1:5] == b'\x00\x01\x00\x00'
	66	assert b[1:5] == b"\x00\x01\x00\x00"
65	67	assert b[5:] == data
66	68	assert unpackb(b) == data
	69
67	70
68	71	def test_ext():
69	72	def check(ext, packed):
70	73	assert packb(ext) == packed
71	74	assert unpackb(packed) == ext
72		check(ExtType(0x42, b'Z'), b'\xd4\x42Z') # fixext 1
73		check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ') # fixext 2
74		check(ExtType(0x42, b'Z'4), b'\xd6\x42' + b'Z'4) # fixext 4
75		check(ExtType(0x42, b'Z'8), b'\xd7\x42' + b'Z'8) # fixext 8
76		check(ExtType(0x42, b'Z'16), b'\xd8\x42' + b'Z'16) # fixext 16
	75
	76	check(ExtType(0x42, b"Z"), b"\xd4\x42Z") # fixext 1
	77	check(ExtType(0x42, b"ZZ"), b"\xd5\x42ZZ") # fixext 2
	78	check(ExtType(0x42, b"Z" * 4), b"\xd6\x42" + b"Z" * 4) # fixext 4
	79	check(ExtType(0x42, b"Z" * 8), b"\xd7\x42" + b"Z" * 8) # fixext 8
	80	check(ExtType(0x42, b"Z" * 16), b"\xd8\x42" + b"Z" * 16) # fixext 16
77	81	# ext 8
78		check(ExtType(0x42, b''), b'\xc7\x00\x42')
79		check(ExtType(0x42, b'Z'255), b'\xc7\xff\x42' + b'Z'255)
	82	check(ExtType(0x42, b""), b"\xc7\x00\x42")
	83	check(ExtType(0x42, b"Z" * 255), b"\xc7\xff\x42" + b"Z" * 255)
80	84	# ext 16
81		check(ExtType(0x42, b'Z'256), b'\xc8\x01\x00\x42' + b'Z'256)
82		check(ExtType(0x42, b'Z'0xffff), b'\xc8\xff\xff\x42' + b'Z'0xffff)
	85	check(ExtType(0x42, b"Z" * 256), b"\xc8\x01\x00\x42" + b"Z" * 256)
	86	check(ExtType(0x42, b"Z" * 0xFFFF), b"\xc8\xff\xff\x42" + b"Z" * 0xFFFF)
83	87	# ext 32
84		check(ExtType(0x42, b'Z'0x10000), b'\xc9\x00\x01\x00\x00\x42' + b'Z'0x10000)
	88	check(ExtType(0x42, b"Z" * 0x10000), b"\xc9\x00\x01\x00\x00\x42" + b"Z" * 0x10000)
85	89	# needs large memory
86		#check(ExtType(0x42, b'Z'*0xffffffff),
	90	# check(ExtType(0x42, b'Z'*0xffffffff),
87	91	# b'\xc9\xff\xff\xff\xff\x42' + b'Z'*0xffffffff)

+32

-17

test/test_obj.py less more

3	3	from pytest import raises
4	4	from msgpack import packb, unpackb
5	5
	6
6	7	def _decode_complex(obj):
7		if b'__complex__' in obj:
8		return complex(obj[b'real'], obj[b'imag'])
	8	if b"__complex__" in obj:
	9	return complex(obj[b"real"], obj[b"imag"])
9	10	return obj
	11
10	12
11	13	def _encode_complex(obj):
12	14	if isinstance(obj, complex):
13		return {b'__complex__': True, b'real': 1, b'imag': 2}
	15	return {b"__complex__": True, b"real": 1, b"imag": 2}
14	16	return obj
15	17
	18
16	19	def test_encode_hook():
17		packed = packb([3, 1+2j], default=_encode_complex)
	20	packed = packb([3, 1 + 2j], default=_encode_complex)
18	21	unpacked = unpackb(packed, use_list=1)
19		assert unpacked[1] == {b'__complex__': True, b'real': 1, b'imag': 2}
	22	assert unpacked[1] == {b"__complex__": True, b"real": 1, b"imag": 2}
	23
20	24
21	25	def test_decode_hook():
22		packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
	26	packed = packb([3, {b"__complex__": True, b"real": 1, b"imag": 2}])
23	27	unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1)
24		assert unpacked[1] == 1+2j
	28	assert unpacked[1] == 1 + 2j
	29
25	30
26	31	def test_decode_pairs_hook():
27	32	packed = packb([3, {1: 2, 3: 4}])
28	33	prod_sum = 1 * 2 + 3 * 4
29		unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1)
	34	unpacked = unpackb(
	35	packed,
	36	object_pairs_hook=lambda l: sum(k * v for k, v in l),
	37	use_list=1,
	38	strict_map_key=False,
	39	)
30	40	assert unpacked[1] == prod_sum
	41
31	42
32	43	def test_only_one_obj_hook():
33	44	with raises(TypeError):
34		unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
	45	unpackb(b"", object_hook=lambda x: x, object_pairs_hook=lambda x: x)
	46
35	47
36	48	def test_bad_hook():
37	49	with raises(TypeError):
38		packed = packb([3, 1+2j], default=lambda o: o)
	50	packed = packb([3, 1 + 2j], default=lambda o: o)
39	51	unpacked = unpackb(packed, use_list=1)
40	52
	53
41	54	def _arr_to_str(arr):
42		return ''.join(str(c) for c in arr)
	55	return "".join(str(c) for c in arr)
	56
43	57
44	58	def test_array_hook():
45		packed = packb([1,2,3])
	59	packed = packb([1, 2, 3])
46	60	unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1)
47		assert unpacked == '123'
	61	assert unpacked == "123"
48	62
49	63
50	64	class DecodeError(Exception):
51	65	pass
	66
52	67
53	68	def bad_complex_decoder(o):
54	69	raise DecodeError("Ooops!")

56	71
57	72	def test_an_exception_in_objecthook1():
58	73	with raises(DecodeError):
59		packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}})
60		unpackb(packed, object_hook=bad_complex_decoder)
	74	packed = packb({1: {"__complex__": True, "real": 1, "imag": 2}})
	75	unpackb(packed, object_hook=bad_complex_decoder, strict_map_key=False)
61	76
62	77
63	78	def test_an_exception_in_objecthook2():
64	79	with raises(DecodeError):
65		packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]})
66		unpackb(packed, list_hook=bad_complex_decoder, use_list=1)
	80	packed = packb({1: [{"__complex__": True, "real": 1, "imag": 2}]})
	81	unpackb(packed, list_hook=bad_complex_decoder, use_list=1, strict_map_key=False)

+76

-50

test/test_pack.py less more

4	4	from collections import OrderedDict
5	5	from io import BytesIO
6	6	import struct
	7	import sys
7	8
8	9	import pytest
9	10	from pytest import raises, xfail

12	13
13	14
14	15	def check(data, use_list=False):
15		re = unpackb(packb(data), use_list=use_list)
	16	re = unpackb(packb(data), use_list=use_list, strict_map_key=False)
16	17	assert re == data
	18
17	19
18	20	def testPack():
19	21	test_data = [
20		0, 1, 127, 128, 255, 256, 65535, 65536, 4294967295, 4294967296,
21		-1, -32, -33, -128, -129, -32768, -32769, -4294967296, -4294967297,
22		1.0,
23		b"", b"a", b"a"31, b"a"32,
24		None, True, False,
25		(), ((),), ((), None,),
	22	0,
	23	1,
	24	127,
	25	128,
	26	255,
	27	256,
	28	65535,
	29	65536,
	30	4294967295,
	31	4294967296,
	32	-1,
	33	-32,
	34	-33,
	35	-128,
	36	-129,
	37	-32768,
	38	-32769,
	39	-4294967296,
	40	-4294967297,
	41	1.0,
	42	b"",
	43	b"a",
	44	b"a" * 31,
	45	b"a" * 32,
	46	None,
	47	True,
	48	False,
	49	(),
	50	((),),
	51	((), None),
26	52	{None: 0},
27		(1<<23),
28		]
	53	(1 << 23),
	54	]
29	55	for td in test_data:
30	56	check(td)
	57
31	58
32	59	def testPackUnicode():
33	60	test_data = ["", "abcd", ["defgh"], "Русский текст"]

39	66	re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack()
40	67	assert re == td
41	68
42		def testPackUTF32(): # deprecated
43		try:
44		test_data = [
45		"",
46		"abcd",
47		["defgh"],
48		"Русский текст",
49		]
50		for td in test_data:
51		with pytest.deprecated_call():
52		re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
53		assert re == td
54		except LookupError as e:
55		xfail(e)
56	69
57	70	def testPackBytes():
58		test_data = [
59		b"", b"abcd", (b"defgh",),
60		]
	71	test_data = [b"", b"abcd", (b"defgh",)]
61	72	for td in test_data:
62	73	check(td)
63	74
	75
64	76	def testPackByteArrays():
65		test_data = [
66		bytearray(b""), bytearray(b"abcd"), (bytearray(b"defgh"),),
67		]
	77	test_data = [bytearray(b""), bytearray(b"abcd"), (bytearray(b"defgh"),)]
68	78	for td in test_data:
69	79	check(td)
70	80
71		def testIgnoreUnicodeErrors(): # deprecated
72		with pytest.deprecated_call():
73		re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
	81
	82	@pytest.mark.skipif(
	83	sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates"
	84	)
	85	def testIgnoreUnicodeErrors():
	86	re = unpackb(
	87	packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore"
	88	)
74	89	assert re == "abcdef"
75	90
	91
76	92	def testStrictUnicodeUnpack():
77		packed = packb(b'abc\xeddef')
	93	packed = packb(b"abc\xeddef", use_bin_type=False)
78	94	with pytest.raises(UnicodeDecodeError):
79	95	unpackb(packed, raw=False, use_list=1)
80	96
81		def testStrictUnicodePack(): # deprecated
82		with raises(UnicodeEncodeError):
83		with pytest.deprecated_call():
84		packb("abc\xeddef", encoding='ascii', unicode_errors='strict')
85	97
86		def testIgnoreErrorsPack(): # deprecated
87		with pytest.deprecated_call():
88		re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1)
	98	@pytest.mark.skipif(
	99	sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates"
	100	)
	101	def testIgnoreErrorsPack():
	102	re = unpackb(
	103	packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"),
	104	raw=False,
	105	use_list=1,
	106	)
89	107	assert re == "abcdef"
90	108
	109
91	110	def testDecodeBinary():
92		re = unpackb(packb(b"abc"), encoding=None, use_list=1)
	111	re = unpackb(packb(b"abc"), use_list=1)
93	112	assert re == b"abc"
94	113
	114
95	115	def testPackFloat():
96		assert packb(1.0, use_single_float=True) == b'\xca' + struct.pack(str('>f'), 1.0)
97		assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack(str('>d'), 1.0)
	116	assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(str(">f"), 1.0)
	117	assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(str(">d"), 1.0)
	118
98	119
99	120	def testArraySize(sizes=[0, 5, 50, 1000]):
100	121	bio = BytesIO()

109	130	for size in sizes:
110	131	assert unpacker.unpack() == list(range(size))
111	132
	133
112	134	def test_manualreset(sizes=[0, 5, 50, 1000]):
113	135	packer = Packer(autoreset=False)
114	136	for size in sizes:

122	144	assert unpacker.unpack() == list(range(size))
123	145
124	146	packer.reset()
125		assert packer.bytes() == b''
	147	assert packer.bytes() == b""
	148
126	149
127	150	def testMapSize(sizes=[0, 5, 50, 1000]):
128	151	bio = BytesIO()

130	153	for size in sizes:
131	154	bio.write(packer.pack_map_header(size))
132	155	for i in range(size):
133		bio.write(packer.pack(i)) # key
134		bio.write(packer.pack(i * 2)) # value
	156	bio.write(packer.pack(i)) # key
	157	bio.write(packer.pack(i * 2)) # value
135	158
136	159	bio.seek(0)
137		unpacker = Unpacker(bio)
	160	unpacker = Unpacker(bio, strict_map_key=False)
138	161	for size in sizes:
139	162	assert unpacker.unpack() == dict((i, i * 2) for i in range(size))
140	163
141	164
142	165	def test_odict():
143		seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
	166	seq = [(b"one", 1), (b"two", 2), (b"three", 3), (b"four", 4)]
144	167	od = OrderedDict(seq)
145	168	assert unpackb(packb(od), use_list=1) == dict(seq)
	169
146	170	def pair_hook(seq):
147	171	return list(seq)
	172
148	173	assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
149	174
150	175
151	176	def test_pairlist():
152		pairlist = [(b'a', 1), (2, b'b'), (b'foo', b'bar')]
	177	pairlist = [(b"a", 1), (2, b"b"), (b"foo", b"bar")]
153	178	packer = Packer()
154	179	packed = packer.pack_map_pairs(pairlist)
155		unpacked = unpackb(packed, object_pairs_hook=list)
	180	unpacked = unpackb(packed, object_pairs_hook=list, strict_map_key=False)
156	181	assert pairlist == unpacked
	182
157	183
158	184	def test_get_buffer():
159	185	packer = Packer(autoreset=0, use_bin_type=True)

+27

-22

test/test_read_size.py less more

0	0	"""Test Unpacker's read_array_header and read_map_header methods"""
1	1	from msgpack import packb, Unpacker, OutOfData
	2
2	3	UnexpectedTypeException = ValueError
	4
3	5
4	6	def test_read_array_header():
5	7	unpacker = Unpacker()
6		unpacker.feed(packb(['a', 'b', 'c']))
	8	unpacker.feed(packb(["a", "b", "c"]))
7	9	assert unpacker.read_array_header() == 3
8		assert unpacker.unpack() == b'a'
9		assert unpacker.unpack() == b'b'
10		assert unpacker.unpack() == b'c'
	10	assert unpacker.unpack() == "a"
	11	assert unpacker.unpack() == "b"
	12	assert unpacker.unpack() == "c"
11	13	try:
12	14	unpacker.unpack()
13		assert 0, 'should raise exception'
	15	assert 0, "should raise exception"
14	16	except OutOfData:
15		assert 1, 'okay'
	17	assert 1, "okay"
16	18
17	19
18	20	def test_read_map_header():
19	21	unpacker = Unpacker()
20		unpacker.feed(packb({'a': 'A'}))
	22	unpacker.feed(packb({"a": "A"}))
21	23	assert unpacker.read_map_header() == 1
22		assert unpacker.unpack() == B'a'
23		assert unpacker.unpack() == B'A'
	24	assert unpacker.unpack() == "a"
	25	assert unpacker.unpack() == "A"
24	26	try:
25	27	unpacker.unpack()
26		assert 0, 'should raise exception'
	28	assert 0, "should raise exception"
27	29	except OutOfData:
28		assert 1, 'okay'
	30	assert 1, "okay"
	31
29	32
30	33	def test_incorrect_type_array():
31	34	unpacker = Unpacker()
32	35	unpacker.feed(packb(1))
33	36	try:
34	37	unpacker.read_array_header()
35		assert 0, 'should raise exception'
	38	assert 0, "should raise exception"
36	39	except UnexpectedTypeException:
37		assert 1, 'okay'
	40	assert 1, "okay"
	41
38	42
39	43	def test_incorrect_type_map():
40	44	unpacker = Unpacker()
41	45	unpacker.feed(packb(1))
42	46	try:
43	47	unpacker.read_map_header()
44		assert 0, 'should raise exception'
	48	assert 0, "should raise exception"
45	49	except UnexpectedTypeException:
46		assert 1, 'okay'
	50	assert 1, "okay"
	51
47	52
48	53	def test_correct_type_nested_array():
49	54	unpacker = Unpacker()
50		unpacker.feed(packb({'a': ['b', 'c', 'd']}))
	55	unpacker.feed(packb({"a": ["b", "c", "d"]}))
51	56	try:
52	57	unpacker.read_array_header()
53		assert 0, 'should raise exception'
	58	assert 0, "should raise exception"
54	59	except UnexpectedTypeException:
55		assert 1, 'okay'
	60	assert 1, "okay"
	61
56	62
57	63	def test_incorrect_type_nested_map():
58	64	unpacker = Unpacker()
59		unpacker.feed(packb([{'a': 'b'}]))
	65	unpacker.feed(packb([{"a": "b"}]))
60	66	try:
61	67	unpacker.read_map_header()
62		assert 0, 'should raise exception'
	68	assert 0, "should raise exception"
63	69	except UnexpectedTypeException:
64		assert 1, 'okay'
65
	70	assert 1, "okay"

-5

test/test_seq.py less more

6	6
7	7	binarydata = bytes(bytearray(range(256)))
8	8
	9
9	10	def gen_binary_data(idx):
10		return binarydata[:idx % 300]
	11	return binarydata[: idx % 300]
11	12
12	13
13	14	def test_exceeding_unpacker_read_size():

17	18
18	19	NUMBER_OF_STRINGS = 6
19	20	read_size = 16
20		# 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop):
21		# 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev)
22		# 40 ok for read_size=1024, while 50 introduces errors
23		# 7000 ok for read_size=10241024, while 8000 leads to glibc detected ** python: double free or corruption (!prev):
	21	# 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop):
	22	# 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev)
	23	# 40 ok for read_size=1024, while 50 introduces errors
	24	# 7000 ok for read_size=10241024, while 8000 leads to glibc detected ** python: double free or corruption (!prev):
24	25
25	26	for idx in range(NUMBER_OF_STRINGS):
26	27	data = gen_binary_data(idx)

+77

-56

test/test_sequnpack.py less more

0	0	#!/usr/bin/env python
1	1	# coding: utf-8
2
3	2	import io
4	3	from msgpack import Unpacker, BufferFull
5	4	from msgpack import pack

9	8
10	9	def test_partialdata():
11	10	unpacker = Unpacker()
12		unpacker.feed(b'\xa5')
13		with raises(StopIteration): next(iter(unpacker))
14		unpacker.feed(b'h')
15		with raises(StopIteration): next(iter(unpacker))
16		unpacker.feed(b'a')
17		with raises(StopIteration): next(iter(unpacker))
18		unpacker.feed(b'l')
19		with raises(StopIteration): next(iter(unpacker))
20		unpacker.feed(b'l')
21		with raises(StopIteration): next(iter(unpacker))
22		unpacker.feed(b'o')
23		assert next(iter(unpacker)) == b'hallo'
	11	unpacker.feed(b"\xa5")
	12	with raises(StopIteration):
	13	next(iter(unpacker))
	14	unpacker.feed(b"h")
	15	with raises(StopIteration):
	16	next(iter(unpacker))
	17	unpacker.feed(b"a")
	18	with raises(StopIteration):
	19	next(iter(unpacker))
	20	unpacker.feed(b"l")
	21	with raises(StopIteration):
	22	next(iter(unpacker))
	23	unpacker.feed(b"l")
	24	with raises(StopIteration):
	25	next(iter(unpacker))
	26	unpacker.feed(b"o")
	27	assert next(iter(unpacker)) == "hallo"
	28
24	29
25	30	def test_foobar():
26	31	unpacker = Unpacker(read_size=3, use_list=1)
27		unpacker.feed(b'foobar')
28		assert unpacker.unpack() == ord(b'f')
29		assert unpacker.unpack() == ord(b'o')
30		assert unpacker.unpack() == ord(b'o')
31		assert unpacker.unpack() == ord(b'b')
32		assert unpacker.unpack() == ord(b'a')
33		assert unpacker.unpack() == ord(b'r')
	32	unpacker.feed(b"foobar")
	33	assert unpacker.unpack() == ord(b"f")
	34	assert unpacker.unpack() == ord(b"o")
	35	assert unpacker.unpack() == ord(b"o")
	36	assert unpacker.unpack() == ord(b"b")
	37	assert unpacker.unpack() == ord(b"a")
	38	assert unpacker.unpack() == ord(b"r")
34	39	with raises(OutOfData):
35	40	unpacker.unpack()
36	41
37		unpacker.feed(b'foo')
38		unpacker.feed(b'bar')
	42	unpacker.feed(b"foo")
	43	unpacker.feed(b"bar")
39	44
40	45	k = 0
41		for o, e in zip(unpacker, 'foobarbaz'):
	46	for o, e in zip(unpacker, "foobarbaz"):
42	47	assert o == ord(e)
43	48	k += 1
44		assert k == len(b'foobar')
	49	assert k == len(b"foobar")
	50
45	51
46	52	def test_foobar_skip():
47	53	unpacker = Unpacker(read_size=3, use_list=1)
48		unpacker.feed(b'foobar')
49		assert unpacker.unpack() == ord(b'f')
	54	unpacker.feed(b"foobar")
	55	assert unpacker.unpack() == ord(b"f")
50	56	unpacker.skip()
51		assert unpacker.unpack() == ord(b'o')
	57	assert unpacker.unpack() == ord(b"o")
52	58	unpacker.skip()
53		assert unpacker.unpack() == ord(b'a')
	59	assert unpacker.unpack() == ord(b"a")
54	60	unpacker.skip()
55	61	with raises(OutOfData):
56	62	unpacker.unpack()
	63
57	64
58	65	def test_maxbuffersize():
59	66	with raises(ValueError):
60	67	Unpacker(read_size=5, max_buffer_size=3)
61	68	unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
62		unpacker.feed(b'fo')
	69	unpacker.feed(b"fo")
63	70	with raises(BufferFull):
64		unpacker.feed(b'ob')
65		unpacker.feed(b'o')
66		assert ord('f') == next(unpacker)
67		unpacker.feed(b'b')
68		assert ord('o') == next(unpacker)
69		assert ord('o') == next(unpacker)
70		assert ord('b') == next(unpacker)
	71	unpacker.feed(b"ob")
	72	unpacker.feed(b"o")
	73	assert ord("f") == next(unpacker)
	74	unpacker.feed(b"b")
	75	assert ord("o") == next(unpacker)
	76	assert ord("o") == next(unpacker)
	77	assert ord("b") == next(unpacker)
71	78
72	79
73	80	def test_readbytes():
74	81	unpacker = Unpacker(read_size=3)
75		unpacker.feed(b'foobar')
76		assert unpacker.unpack() == ord(b'f')
77		assert unpacker.read_bytes(3) == b'oob'
78		assert unpacker.unpack() == ord(b'a')
79		assert unpacker.unpack() == ord(b'r')
	82	unpacker.feed(b"foobar")
	83	assert unpacker.unpack() == ord(b"f")
	84	assert unpacker.read_bytes(3) == b"oob"
	85	assert unpacker.unpack() == ord(b"a")
	86	assert unpacker.unpack() == ord(b"r")
80	87
81	88	# Test buffer refill
82		unpacker = Unpacker(io.BytesIO(b'foobar'), read_size=3)
83		assert unpacker.unpack() == ord(b'f')
84		assert unpacker.read_bytes(3) == b'oob'
85		assert unpacker.unpack() == ord(b'a')
86		assert unpacker.unpack() == ord(b'r')
	89	unpacker = Unpacker(io.BytesIO(b"foobar"), read_size=3)
	90	assert unpacker.unpack() == ord(b"f")
	91	assert unpacker.read_bytes(3) == b"oob"
	92	assert unpacker.unpack() == ord(b"a")
	93	assert unpacker.unpack() == ord(b"r")
	94
	95	# Issue 352
	96	u = Unpacker()
	97	u.feed(b"x")
	98	assert bytes(u.read_bytes(1)) == b"x"
	99	with raises(StopIteration):
	100	next(u)
	101	u.feed(b"\1")
	102	assert next(u) == 1
	103
87	104
88	105	def test_issue124():
89	106	unpacker = Unpacker()
90		unpacker.feed(b'\xa1?\xa1!')
91		assert tuple(unpacker) == (b'?', b'!')
	107	unpacker.feed(b"\xa1?\xa1!")
	108	assert tuple(unpacker) == ("?", "!")
92	109	assert tuple(unpacker) == ()
93	110	unpacker.feed(b"\xa1?\xa1")
94		assert tuple(unpacker) == (b'?',)
	111	assert tuple(unpacker) == ("?",)
95	112	assert tuple(unpacker) == ()
96	113	unpacker.feed(b"!")
97		assert tuple(unpacker) == (b'!',)
	114	assert tuple(unpacker) == ("!",)
98	115	assert tuple(unpacker) == ()
99	116
100	117
101	118	def test_unpack_tell():
102	119	stream = io.BytesIO()
103		messages = [2**i-1 for i in range(65)]
104		messages += [-(2**i) for i in range(1, 64)]
105		messages += [b'hello', b'hello'*1000, list(range(20)),
106		{i: bytes(i)*i for i in range(10)},
107		{i: bytes(i)*i for i in range(32)}]
	120	messages = [2 ** i - 1 for i in range(65)]
	121	messages += [-(2 ** i) for i in range(1, 64)]
	122	messages += [
	123	b"hello",
	124	b"hello" * 1000,
	125	list(range(20)),
	126	{i: bytes(i) * i for i in range(10)},
	127	{i: bytes(i) * i for i in range(32)},
	128	]
108	129	offsets = []
109	130	for m in messages:
110	131	pack(m, stream)
111	132	offsets.append(stream.tell())
112	133	stream.seek(0)
113		unpacker = Unpacker(stream)
	134	unpacker = Unpacker(stream, strict_map_key=False)
114	135	for m, o in zip(messages, offsets):
115	136	m2 = next(unpacker)
116	137	assert m == m2

+14

-14

test/test_stricttype.py less more

4	4
5	5
6	6	def test_namedtuple():
7		T = namedtuple('T', "foo bar")
	7	T = namedtuple("T", "foo bar")
	8
8	9	def default(o):
9	10	if isinstance(o, T):
10	11	return dict(o._asdict())
11		raise TypeError('Unsupported type %s' % (type(o),))
	12	raise TypeError("Unsupported type %s" % (type(o),))
	13
12	14	packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default)
13	15	unpacked = unpackb(packed, raw=False)
14		assert unpacked == {'foo': 1, 'bar': 42}
	16	assert unpacked == {"foo": 1, "bar": 42}
15	17
16	18
17	19	def test_tuple():
18		t = ('one', 2, b'three', (4, ))
	20	t = ("one", 2, b"three", (4,))
19	21
20	22	def default(o):
21	23	if isinstance(o, tuple):
22		return {
23		'__type__': 'tuple',
24		'value': list(o),
25		}
26		raise TypeError('Unsupported type %s' % (type(o),))
	24	return {"__type__": "tuple", "value": list(o)}
	25	raise TypeError("Unsupported type %s" % (type(o),))
27	26
28	27	def convert(o):
29		if o.get('__type__') == 'tuple':
30		return tuple(o['value'])
	28	if o.get("__type__") == "tuple":
	29	return tuple(o["value"])
31	30	return o
32	31
33	32	data = packb(t, strict_types=True, use_bin_type=True, default=default)

37	36
38	37
39	38	def test_tuple_ext():
40		t = ('one', 2, b'three', (4, ))
	39	t = ("one", 2, b"three", (4,))
41	40
42	41	MSGPACK_EXT_TYPE_TUPLE = 0
43	42

45	44	if isinstance(o, tuple):
46	45	# Convert to list and pack
47	46	payload = packb(
48		list(o), strict_types=True, use_bin_type=True, default=default)
	47	list(o), strict_types=True, use_bin_type=True, default=default
	48	)
49	49	return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload)
50	50	raise TypeError(repr(o))
51	51

53	53	if code == MSGPACK_EXT_TYPE_TUPLE:
54	54	# Unpack and convert to tuple
55	55	return tuple(unpackb(payload, raw=False, ext_hook=convert))
56		raise ValueError('Unknown Ext code {}'.format(code))
	56	raise ValueError("Unknown Ext code {}".format(code))
57	57
58	58	data = packb(t, strict_types=True, use_bin_type=True, default=default)
59	59	expected = unpackb(data, raw=False, ext_hook=convert)

-1

test/test_subtype.py less more

3	3	from msgpack import packb, unpackb
4	4	from collections import namedtuple
5	5
	6
6	7	class MyList(list):
7	8	pass
	9
8	10
9	11	class MyDict(dict):
10	12	pass
11	13
	14
12	15	class MyTuple(tuple):
13	16	pass
14	17
15		MyNamedTuple = namedtuple('MyNamedTuple', 'x y')
	18
	19	MyNamedTuple = namedtuple("MyNamedTuple", "x y")
	20
16	21
17	22	def test_types():
18	23	assert packb(MyDict()) == packb(dict())

+123

-0

test/test_timestamp.py less more

	0	import pytest
	1	import sys
	2	import datetime
	3	import msgpack
	4	from msgpack.ext import Timestamp
	5
	6	if sys.version_info[0] > 2:
	7	from msgpack.ext import _utc
	8
	9
	10	def test_timestamp():
	11	# timestamp32
	12	ts = Timestamp(2 ** 32 - 1)
	13	assert ts.to_bytes() == b"\xff\xff\xff\xff"
	14	packed = msgpack.packb(ts)
	15	assert packed == b"\xd6\xff" + ts.to_bytes()
	16	unpacked = msgpack.unpackb(packed)
	17	assert ts == unpacked
	18	assert ts.seconds == 2 ** 32 - 1 and ts.nanoseconds == 0
	19
	20	# timestamp64
	21	ts = Timestamp(2 ** 34 - 1, 999999999)
	22	assert ts.to_bytes() == b"\xee\x6b\x27\xff\xff\xff\xff\xff"
	23	packed = msgpack.packb(ts)
	24	assert packed == b"\xd7\xff" + ts.to_bytes()
	25	unpacked = msgpack.unpackb(packed)
	26	assert ts == unpacked
	27	assert ts.seconds == 2 ** 34 - 1 and ts.nanoseconds == 999999999
	28
	29	# timestamp96
	30	ts = Timestamp(2 ** 63 - 1, 999999999)
	31	assert ts.to_bytes() == b"\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff"
	32	packed = msgpack.packb(ts)
	33	assert packed == b"\xc7\x0c\xff" + ts.to_bytes()
	34	unpacked = msgpack.unpackb(packed)
	35	assert ts == unpacked
	36	assert ts.seconds == 2 ** 63 - 1 and ts.nanoseconds == 999999999
	37
	38	# negative fractional
	39	ts = Timestamp.from_unix(-2.3) # s: -3, ns: 700000000
	40	assert ts.seconds == -3 and ts.nanoseconds == 700000000
	41	assert ts.to_bytes() == b"\x29\xb9\x27\x00\xff\xff\xff\xff\xff\xff\xff\xfd"
	42	packed = msgpack.packb(ts)
	43	assert packed == b"\xc7\x0c\xff" + ts.to_bytes()
	44	unpacked = msgpack.unpackb(packed)
	45	assert ts == unpacked
	46
	47
	48	def test_unpack_timestamp():
	49	# timestamp 32
	50	assert msgpack.unpackb(b"\xd6\xff\x00\x00\x00\x00") == Timestamp(0)
	51
	52	# timestamp 64
	53	assert msgpack.unpackb(b"\xd7\xff" + b"\x00" * 8) == Timestamp(0)
	54	with pytest.raises(ValueError):
	55	msgpack.unpackb(b"\xd7\xff" + b"\xff" * 8)
	56
	57	# timestamp 96
	58	assert msgpack.unpackb(b"\xc7\x0c\xff" + b"\x00" * 12) == Timestamp(0)
	59	with pytest.raises(ValueError):
	60	msgpack.unpackb(b"\xc7\x0c\xff" + b"\xff" * 12) == Timestamp(0)
	61
	62	# Undefined
	63	with pytest.raises(ValueError):
	64	msgpack.unpackb(b"\xd4\xff\x00") # fixext 1
	65	with pytest.raises(ValueError):
	66	msgpack.unpackb(b"\xd5\xff\x00\x00") # fixext 2
	67	with pytest.raises(ValueError):
	68	msgpack.unpackb(b"\xc7\x00\xff") # ext8 (len=0)
	69	with pytest.raises(ValueError):
	70	msgpack.unpackb(b"\xc7\x03\xff\0\0\0") # ext8 (len=3)
	71	with pytest.raises(ValueError):
	72	msgpack.unpackb(b"\xc7\x05\xff\0\0\0\0\0") # ext8 (len=5)
	73
	74
	75	def test_timestamp_from():
	76	t = Timestamp(42, 14000)
	77	assert Timestamp.from_unix(42.000014) == t
	78	assert Timestamp.from_unix_nano(42000014000) == t
	79
	80
	81	def test_timestamp_to():
	82	t = Timestamp(42, 14000)
	83	assert t.to_unix() == 42.000014
	84	assert t.to_unix_nano() == 42000014000
	85
	86
	87	@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
	88	def test_timestamp_datetime():
	89	t = Timestamp(42, 14)
	90	assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc)
	91
	92
	93	@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
	94	def test_unpack_datetime():
	95	t = Timestamp(42, 14)
	96	packed = msgpack.packb(t)
	97	unpacked = msgpack.unpackb(packed, timestamp=3)
	98	assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc)
	99
	100
	101	@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
	102	def test_pack_datetime():
	103	t = Timestamp(42, 14000)
	104	dt = t.to_datetime()
	105	assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc)
	106
	107	packed = msgpack.packb(dt, datetime=True)
	108	packed2 = msgpack.packb(t)
	109	assert packed == packed2
	110
	111	unpacked = msgpack.unpackb(packed)
	112	print(packed, unpacked)
	113	assert unpacked == t
	114
	115	unpacked = msgpack.unpackb(packed, timestamp=3)
	116	assert unpacked == dt
	117
	118	x = []
	119	packed = msgpack.packb(dt, datetime=False, default=x.append)
	120	assert x
	121	assert x[0] == dt
	122	assert msgpack.unpackb(packed) is None

+13

-14

test/test_unpack.py less more

4	4
5	5
6	6	def test_unpack_array_header_from_file():
7		f = BytesIO(packb([1,2,3,4]))
	7	f = BytesIO(packb([1, 2, 3, 4]))
8	8	unpacker = Unpacker(f)
9	9	assert unpacker.read_array_header() == 4
10	10	assert unpacker.unpack() == 1

15	15	unpacker.unpack()
16	16
17	17
18		@mark.skipif("not hasattr(sys, 'getrefcount') == True",
19		reason='sys.getrefcount() is needed to pass this test')
	18	@mark.skipif(
	19	"not hasattr(sys, 'getrefcount') == True",
	20	reason="sys.getrefcount() is needed to pass this test",
	21	)
20	22	def test_unpacker_hook_refcnt():
21	23	result = []
22	24

42	44
43	45
44	46	def test_unpacker_ext_hook():
45
46	47	class MyUnpacker(Unpacker):
47
48	48	def __init__(self):
49		super(MyUnpacker, self).__init__(
50		ext_hook=self._hook, raw=False)
	49	super(MyUnpacker, self).__init__(ext_hook=self._hook, raw=False)
51	50
52	51	def _hook(self, code, data):
53	52	if code == 1:

56	55	return ExtType(code, data)
57	56
58	57	unpacker = MyUnpacker()
59		unpacker.feed(packb({'a': 1}))
60		assert unpacker.unpack() == {'a': 1}
61		unpacker.feed(packb({'a': ExtType(1, b'123')}))
62		assert unpacker.unpack() == {'a': 123}
63		unpacker.feed(packb({'a': ExtType(2, b'321')}))
64		assert unpacker.unpack() == {'a': ExtType(2, b'321')}
	58	unpacker.feed(packb({"a": 1}))
	59	assert unpacker.unpack() == {"a": 1}
	60	unpacker.feed(packb({"a": ExtType(1, b"123")}))
	61	assert unpacker.unpack() == {"a": 123}
	62	unpacker.feed(packb({"a": ExtType(2, b"321")}))
	63	assert unpacker.unpack() == {"a": ExtType(2, b"321")}
65	64
66	65
67		if __name__ == '__main__':
	66	if __name__ == "__main__":
68	67	test_unpack_array_header_from_file()
69	68	test_unpacker_hook_refcnt()
70	69	test_unpacker_ext_hook()

-1

tox.ini less more

0	0	[tox]
1		envlist = {py27,py35,py36,py37}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
	1	envlist =
	2	py27-pure,
	3	{py35,py36,py37,py38}-{c,pure},
	4	{pypy,pypy3}-pure,
	5	py27-x86,
	6	py34-x86,
2	7
3	8	[variants:pure]
4	9	setenv=