diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index eda8d07..c545953 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -4,6 +4,10 @@ jobs: black: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest steps: - name: Setup Python diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f9707b1..95c672b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -6,6 +6,10 @@ jobs: build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest steps: - name: Checkout @@ -47,6 +51,17 @@ pytest -v test + - name: Set up Python 3.10 + uses: actions/setup-python@v1 + with: + python-version: "3.10" + + - name: Run test (3.10) + run: | + pip install pytest + pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse + pytest -v test + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 78d944c..85844e7 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,7 +1,6 @@ name: Build macOS Wheels on: push: - pull_request: create: jobs: @@ -12,10 +11,11 @@ - name: Checkout uses: actions/checkout@v1 - - name: Set up Python 3.8 + # Python 3.9 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: "3.8" + python-version: "3.9" - name: Cythonize run: | @@ -23,66 +23,53 @@ pip install -r requirements.txt make cython - - name: Build wheel + - name: Build wheels + uses: pypa/cibuildwheel@v2.2.2 + env: + CIBW_ARCHS_MACOS: x86_64 universal2 + CIBW_SKIP: pp* + + - name: Run test run: | - pip install setuptools wheel - python setup.py bdist_wheel + ls wheelhouse/ + pip install pytest + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index + pytest -v test + + # Python 3.10 + - name: Set up Python 3.10 + uses: actions/setup-python@v1 + with: + python-version: "3.10" - name: Run test run: | pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test - - - name: Set up Python 3.9 + # Python 3.8 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: - python-version: "3.9" - - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel + python-version: "3.8" - name: Run test run: | pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test - + # Python 3.7 - name: Set up Python 3.7 uses: actions/setup-python@v1 with: python-version: "3.7" - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel - - name: Run test run: | pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index - pytest -v test - - - - name: Set up Python 3.6 - uses: actions/setup-python@v1 - with: - python-version: "3.6" - - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test @@ -90,4 +77,4 @@ uses: actions/upload-artifact@v1 with: name: macos-wheels - path: ./dist/ + path: ./wheelhouse/ diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index 139a5a6..0ce50f5 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -1,14 +1,15 @@ name: Build and test windows wheels on: push: - branches: - - master - - test pull_request: create: jobs: build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: windows-latest steps: - name: Checkout @@ -77,6 +78,20 @@ run: | ci/runtests.sh + - name: Python 3.10 (amd64) + env: + PYTHON: "py -3.10-64" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.10 (x86) + env: + PYTHON: "py -3.10-32" + shell: bash + run: | + ci/runtests.sh + - name: Upload Wheels uses: actions/upload-artifact@v1 with: diff --git a/ChangeLog.rst b/ChangeLog.rst index 230cc30..fc6df68 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +1.0.3 +===== + +Release Date: 2021-11-24 JST + +* Fix Docstring (#459) +* Fix error formatting (#463) +* Improve error message about strict_map_key (#485) + 1.0.2 ===== diff --git a/Makefile b/Makefile index 05cca55..b50fa80 100644 --- a/Makefile +++ b/Makefile @@ -31,13 +31,13 @@ .PHONY: update-docker update-docker: - docker pull quay.io/pypa/manylinux2010_i686 - docker pull quay.io/pypa/manylinux2010_x86_64 + docker pull quay.io/pypa/manylinux1_i686 + docker pull quay.io/pypa/manylinux1_x86_64 .PHONY: linux-wheel linux-wheel: - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_i686 bash docker/buildwheel.sh - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_i686 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_x86_64 bash docker/buildwheel.sh .PHONY: linux-arm64-wheel linux-arm64-wheel: diff --git a/README.md b/README.md index d8ce9ba..cb81648 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,6 @@ But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy and Python 2. -Since the [pip](https://pip.pypa.io/) uses the pure Python implementation, -Python 2 support will not be dropped in the foreseeable future. ### Windows diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh index 89a2570..ff34139 100644 --- a/docker/buildwheel.sh +++ b/docker/buildwheel.sh @@ -7,10 +7,12 @@ ARCH=`uname -p` echo "arch=$ARCH" +ls /opt/python + for V in "${PYTHON_VERSIONS[@]}"; do PYBIN=/opt/python/$V/bin rm -rf build/ # Avoid lib build by narrow Python is used by wide python - $PYBIN/python setup.py bdist_wheel + $PYBIN/python -m build -w done cd dist diff --git a/docker/shared.env b/docker/shared.env index 3601a07..80274ac 100644 --- a/docker/shared.env +++ b/docker/shared.env @@ -1,7 +1,7 @@ PYTHON_VERSIONS=( + cp310-cp310 cp39-cp39 cp38-cp38 cp37-cp37m cp36-cp36m - cp35-cp35m ) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index e6cd2c7..396da0c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -285,6 +285,8 @@ o = self._default(o) default_used = 1 continue + elif self.datetime and PyDateTime_CheckExact(o): + PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name) else: PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) return ret diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e4f3f1e..27facc0 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -3,13 +3,12 @@ from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject - cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1 object PyMemoryView_GetContiguous(object obj, int buffertype, char order) from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * -ctypedef unsigned long long uint64_t +from libc.stdint cimport uint64_t from .exceptions import ( BufferFull, @@ -212,49 +211,76 @@ cdef class Unpacker(object): - """ - MessagePack Packer - - Usage:: - - packer = Packer() - astream.write(packer.pack(a)) - astream.write(packer.pack(b)) - - Packer's constructor has some keyword arguments: - - :param callable default: - Convert user type to builtin type that Packer supports. - See also simplejson's document. - - :param bool use_single_float: - Use single precision float type for float. (default: False) - - :param bool autoreset: - Reset buffer after each pack and return its content as `bytes`. (default: True). - If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. - - :param bool use_bin_type: - Use bin type introduced in msgpack spec 2.0 for bytes. - It also enables str8 type for unicode. (default: True) - - :param bool strict_types: - If set to true, types will be checked to be exact. Derived classes - from serializable types will not be serialized and will be - treated as unsupported type and forwarded to default. - Additionally tuples will not be serialized as lists. - This is useful when trying to implement accurate serialization - for python types. - - :param bool datetime: - If set to true, datetime with tzinfo is packed into Timestamp type. - Note that the tzinfo is stripped in the timestamp. - You can get UTC datetime with `timestamp=3` option of the Unpacker. - (Python 2 is not supported). + """Streaming unpacker. + + Arguments: + + :param file_like: + File-like object having `.read(n)` method. + If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. + + :param int read_size: + Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) + + :param bool use_list: + If true, unpack msgpack array to Python list. + Otherwise, unpack to Python tuple. (default: True) + + :param bool raw: + If true, unpack msgpack raw to Python bytes. + Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). + + :param int timestamp: + Control how timestamp type is unpacked: + + 0 - Timestamp + 1 - float (Seconds from the EPOCH) + 2 - int (Nanoseconds from the EPOCH) + 3 - datetime.datetime (UTC). Python 2 is not supported. + + :param bool strict_map_key: + If true (default), only str or bytes are accepted for map (dict) keys. + + :param callable object_hook: + When specified, it should be callable. + Unpacker calls it with a dict argument after unpacking msgpack map. + (See also simplejson) + + :param callable object_pairs_hook: + When specified, it should be callable. + Unpacker calls it with a list of key-value pairs after unpacking msgpack map. + (See also simplejson) :param str unicode_errors: - The error handler for encoding unicode. (default: 'strict') - DO NOT USE THIS!! This option is kept for very specific usage. + The error handler for decoding unicode. (default: 'strict') + This option should be used only when you have msgpack data which + contains invalid UTF-8 string. + + :param int max_buffer_size: + Limits size of data waiting unpacked. 0 means 2**32-1. + The default value is 100*1024*1024 (100MiB). + Raises `BufferFull` exception when it is insufficient. + You should set this parameter when unpacking data from untrusted source. + + :param int max_str_len: + Deprecated, use *max_buffer_size* instead. + Limits max length of str. (default: max_buffer_size) + + :param int max_bin_len: + Deprecated, use *max_buffer_size* instead. + Limits max length of bin. (default: max_buffer_size) + + :param int max_array_len: + Limits max length of array. + (default: max_buffer_size) + + :param int max_map_len: + Limits max length of map. + (default: max_buffer_size//2) + + :param int max_ext_len: + Deprecated, use *max_buffer_size* instead. + Limits max size of ext type. (default: max_buffer_size) Example of streaming deserialize from file-like object:: diff --git a/msgpack/_version.py b/msgpack/_version.py index 1c83c8e..fb878b3 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 2) +version = (1, 0, 3) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0bfa94e..b27acb2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,5 +1,4 @@ """Fallback pure Python implementation of msgpack""" - from datetime import datetime as _DateTime import sys import struct @@ -148,6 +147,38 @@ else: _unpack_from = struct.unpack_from +_NO_FORMAT_USED = "" +_MSGPACK_HEADERS = { + 0xC4: (1, _NO_FORMAT_USED, TYPE_BIN), + 0xC5: (2, ">H", TYPE_BIN), + 0xC6: (4, ">I", TYPE_BIN), + 0xC7: (2, "Bb", TYPE_EXT), + 0xC8: (3, ">Hb", TYPE_EXT), + 0xC9: (5, ">Ib", TYPE_EXT), + 0xCA: (4, ">f"), + 0xCB: (8, ">d"), + 0xCC: (1, _NO_FORMAT_USED), + 0xCD: (2, ">H"), + 0xCE: (4, ">I"), + 0xCF: (8, ">Q"), + 0xD0: (1, "b"), + 0xD1: (2, ">h"), + 0xD2: (4, ">i"), + 0xD3: (8, ">q"), + 0xD4: (1, "b1s", TYPE_EXT), + 0xD5: (2, "b2s", TYPE_EXT), + 0xD6: (4, "b4s", TYPE_EXT), + 0xD7: (8, "b8s", TYPE_EXT), + 0xD8: (16, "b16s", TYPE_EXT), + 0xD9: (1, _NO_FORMAT_USED, TYPE_RAW), + 0xDA: (2, ">H", TYPE_RAW), + 0xDB: (4, ">I", TYPE_RAW), + 0xDC: (2, ">H", TYPE_ARRAY), + 0xDD: (4, ">I", TYPE_ARRAY), + 0xDE: (2, ">H", TYPE_MAP), + 0xDF: (4, ">I", TYPE_MAP), +} + class Unpacker(object): """Streaming unpacker. @@ -229,7 +260,7 @@ Example of streaming deserialize from socket:: - unpacker = Unpacker(max_buffer_size) + unpacker = Unpacker() while True: buf = sock.recv(1024**2) if not buf: @@ -354,7 +385,7 @@ self._buffer.extend(view) def _consume(self): - """ Gets rid of the used parts of the buffer. """ + """Gets rid of the used parts of the buffer.""" self._stream_offset += self._buff_i - self._buf_checkpoint self._buf_checkpoint = self._buff_i @@ -409,7 +440,7 @@ self._buff_i = 0 # rollback raise OutOfData - def _read_header(self, execute=EX_CONSTRUCT): + def _read_header(self): typ = TYPE_IMMEDIATE n = 0 obj = None @@ -424,205 +455,95 @@ n = b & 0b00011111 typ = TYPE_RAW if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) obj = self._read(n) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError( + "%s exceeds max_array_len(%s)" % (n, self._max_array_len) + ) elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) elif b == 0xC0: obj = None elif b == 0xC2: obj = False elif b == 0xC3: obj = True - elif b == 0xC4: - typ = TYPE_BIN - self._reserve(1) - n = self._buffer[self._buff_i] - self._buff_i += 1 + elif 0xC4 <= b <= 0xC6: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + if len(fmt) > 0: + n = _unpack_from(fmt, self._buffer, self._buff_i)[0] + else: + n = self._buffer[self._buff_i] + self._buff_i += size if n > self._max_bin_len: raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) - elif b == 0xC5: - typ = TYPE_BIN - self._reserve(2) - n = _unpack_from(">H", self._buffer, self._buff_i)[0] - self._buff_i += 2 - if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n) - elif b == 0xC6: - typ = TYPE_BIN - self._reserve(4) - n = _unpack_from(">I", self._buffer, self._buff_i)[0] - self._buff_i += 4 - if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n) - elif b == 0xC7: # ext 8 - typ = TYPE_EXT - self._reserve(2) - L, n = _unpack_from("Bb", self._buffer, self._buff_i) - self._buff_i += 2 + elif 0xC7 <= b <= 0xC9: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + L, n = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size if L > self._max_ext_len: raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) - elif b == 0xC8: # ext 16 - typ = TYPE_EXT - self._reserve(3) - L, n = _unpack_from(">Hb", self._buffer, self._buff_i) - self._buff_i += 3 - if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L) - elif b == 0xC9: # ext 32 - typ = TYPE_EXT - self._reserve(5) - L, n = _unpack_from(">Ib", self._buffer, self._buff_i) - self._buff_i += 5 - if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L) - elif b == 0xCA: - self._reserve(4) - obj = _unpack_from(">f", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xCB: - self._reserve(8) - obj = _unpack_from(">d", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xCC: - self._reserve(1) - obj = self._buffer[self._buff_i] - self._buff_i += 1 - elif b == 0xCD: - self._reserve(2) - obj = _unpack_from(">H", self._buffer, self._buff_i)[0] - self._buff_i += 2 - elif b == 0xCE: - self._reserve(4) - obj = _unpack_from(">I", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xCF: - self._reserve(8) - obj = _unpack_from(">Q", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xD0: - self._reserve(1) - obj = _unpack_from("b", self._buffer, self._buff_i)[0] - self._buff_i += 1 - elif b == 0xD1: - self._reserve(2) - obj = _unpack_from(">h", self._buffer, self._buff_i)[0] - self._buff_i += 2 - elif b == 0xD2: - self._reserve(4) - obj = _unpack_from(">i", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xD3: - self._reserve(8) - obj = _unpack_from(">q", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xD4: # fixext 1 - typ = TYPE_EXT - if self._max_ext_len < 1: - raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) - self._reserve(2) - n, obj = _unpack_from("b1s", self._buffer, self._buff_i) - self._buff_i += 2 - elif b == 0xD5: # fixext 2 - typ = TYPE_EXT - if self._max_ext_len < 2: - raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) - self._reserve(3) - n, obj = _unpack_from("b2s", self._buffer, self._buff_i) - self._buff_i += 3 - elif b == 0xD6: # fixext 4 - typ = TYPE_EXT - if self._max_ext_len < 4: - raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) - self._reserve(5) - n, obj = _unpack_from("b4s", self._buffer, self._buff_i) - self._buff_i += 5 - elif b == 0xD7: # fixext 8 - typ = TYPE_EXT - if self._max_ext_len < 8: - raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) - self._reserve(9) - n, obj = _unpack_from("b8s", self._buffer, self._buff_i) - self._buff_i += 9 - elif b == 0xD8: # fixext 16 - typ = TYPE_EXT - if self._max_ext_len < 16: - raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) - self._reserve(17) - n, obj = _unpack_from("b16s", self._buffer, self._buff_i) - self._buff_i += 17 - elif b == 0xD9: - typ = TYPE_RAW - self._reserve(1) - n = self._buffer[self._buff_i] - self._buff_i += 1 + elif 0xCA <= b <= 0xD3: + size, fmt = _MSGPACK_HEADERS[b] + self._reserve(size) + if len(fmt) > 0: + obj = _unpack_from(fmt, self._buffer, self._buff_i)[0] + else: + obj = self._buffer[self._buff_i] + self._buff_i += size + elif 0xD4 <= b <= 0xD8: + size, fmt, typ = _MSGPACK_HEADERS[b] + if self._max_ext_len < size: + raise ValueError( + "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) + ) + self._reserve(size + 1) + n, obj = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size + 1 + elif 0xD9 <= b <= 0xDB: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + if len(fmt) > 0: + (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + else: + n = self._buffer[self._buff_i] + self._buff_i += size if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) obj = self._read(n) - elif b == 0xDA: - typ = TYPE_RAW - self._reserve(2) - (n,) = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b == 0xDB: - typ = TYPE_RAW - self._reserve(4) - (n,) = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b == 0xDC: - typ = TYPE_ARRAY - self._reserve(2) - (n,) = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 + elif 0xDC <= b <= 0xDD: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xDD: - typ = TYPE_ARRAY - self._reserve(4) - (n,) = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xDE: - self._reserve(2) - (n,) = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 + raise ValueError( + "%s exceeds max_array_len(%s)" % (n, self._max_array_len) + ) + elif 0xDE <= b <= 0xDF: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - typ = TYPE_MAP - elif b == 0xDF: - self._reserve(4) - (n,) = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - typ = TYPE_MAP + raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) else: raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj def _unpack(self, execute=EX_CONSTRUCT): - typ, n, obj = self._read_header(execute) + typ, n, obj = self._read_header() if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: @@ -953,6 +874,10 @@ obj = self._default(obj) default_used = 1 continue + + if self._datetime and check(obj, _DateTime): + raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,)) + raise TypeError("Cannot serialize %r" % (obj,)) def pack(self, obj): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 34212bc..23aa622 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -193,7 +193,7 @@ static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { if (u->strict_map_key && !PyUnicode_CheckExact(k) && !PyBytes_CheckExact(k)) { - PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name); + PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key when strict_map_key=True", Py_TYPE(k)->tp_name); return -1; } if (PyUnicode_CheckExact(k)) { diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a9eb8aa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + # Also declared in requirements.txt, if updating here please also update + # there + "Cython~=0.29.13", + "setuptools >= 35.0.2", +] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index a2cce25..180fe85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ +# Also declared in pyproject.toml, if updating here please also update there Cython~=0.29.13 diff --git a/setup.py b/setup.py index 8e88750..01f125f 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] -if not PYPY and not PY2: +if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", @@ -125,14 +125,12 @@ }, license="Apache 2.0", classifiers=[ - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 6a29be7..4e26489 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -140,3 +140,19 @@ unpacked = msgpack.unpackb(packed, timestamp=3) assert dt == unpacked + + +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_pack_datetime_without_tzinfo(): + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14) + with pytest.raises(ValueError, match="where tzinfo=None"): + packed = msgpack.packb(dt, datetime=True) + + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14) + packed = msgpack.packb(dt, datetime=True, default=lambda x: None) + assert packed == msgpack.packb(None) + + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + packed = msgpack.packb(dt, datetime=True) + unpacked = msgpack.unpackb(packed, timestamp=3) + assert unpacked == dt diff --git a/test/test_unpack.py b/test/test_unpack.py index 057b7bf..aa4c01f 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -90,10 +90,3 @@ assert obj == unp assert pos == unpacker.tell() assert unpacker.read_bytes(n) == raw - - -if __name__ == "__main__": - test_unpack_array_header_from_file() - test_unpacker_hook_refcnt() - test_unpacker_ext_hook() - test_unpacker_tell() diff --git a/tox.ini b/tox.ini index 607b182..29c256d 100644 --- a/tox.ini +++ b/tox.ini @@ -5,10 +5,7 @@ {pypy,pypy3}-pure, py27-x86, py34-x86, - -[variants:pure] -setenv= - MSGPACK_PUREPYTHON=x +isolated_build = true [testenv] deps= @@ -19,6 +16,8 @@ c,x86: python -c 'from msgpack import _cmsgpack' c,x86: py.test pure: py.test +setenv= + pure: MSGPACK_PUREPYTHON=x [testenv:py27-x86] basepython=python2.7-x86