New Upstream Release - python-parsel
Ready changes
Summary
Merged new upstream version: 1.8.1+dfsg (was: 1.7.0+dfsg).
Diff
diff --git a/.bandit.yml b/.bandit.yml
index f4d993c..bb9aab2 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -1,3 +1,4 @@
skips:
+- B101
- B320
- B410
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index a0e930d..5ed0848 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 1.7.0
+current_version = 1.8.1
commit = True
tag = True
tag_name = v{new_version}
diff --git a/.coveragerc b/.coveragerc
index a0e59ef..ba07b2f 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,6 +1,5 @@
[run]
branch = true
-include = parsel/*
[report]
exclude_lines =
diff --git a/.flake8 b/.flake8
index fe2937e..d086822 100644
--- a/.flake8
+++ b/.flake8
@@ -1,5 +1,5 @@
[flake8]
-ignore = E203
+ignore = E203,W503
per-file-ignores =
docs/conftest.py:E501
parsel/csstranslator.py:E501
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index ef4487e..de11698 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -26,6 +26,9 @@ jobs:
- python-version: "3.11"
env:
TOXENV: black
+ - python-version: "3.11"
+ env:
+ TOXENV: twinecheck
steps:
- uses: actions/checkout@v2
diff --git a/.gitignore b/.gitignore
index d95b13b..9a1e3c0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@ pip-log.txt
# Unit test / coverage reports
.coverage
+/coverage.xml
.tox
nosetests.xml
htmlcov
diff --git a/NEWS b/NEWS
index 2b6f41b..a80a74b 100644
--- a/NEWS
+++ b/NEWS
@@ -3,6 +3,33 @@
History
-------
+1.8.1 (2023-04-18)
+~~~~~~~~~~~~~~~~~~
+
+* Remove a Sphinx reference from NEWS to fix the PyPI description
+* Add a ``twine check`` CI check to detect such problems
+
+1.8.0 (2023-04-18)
+~~~~~~~~~~~~~~~~~~
+
+* Add support for JMESPath: you can now create a selector for a JSON document
+ and call ``Selector.jmespath()``. See `the documentation`_ for more
+ information and examples.
+* Selectors can now be constructed from ``bytes`` (using the ``body`` and
+ ``encoding`` arguments) instead of ``str`` (using the ``text`` argument), so
+ that there is no internal conversion from ``str`` to ``bytes`` and the memory
+ usage is lower.
+* Typing improvements
+* The ``pkg_resources`` module (which was absent from the requirements) is no
+ longer used
+* Documentation build fixes
+* New requirements:
+
+ * ``jmespath``
+ * ``typing_extensions`` (on Python 3.7)
+
+ .. _the documentation: https://parsel.readthedocs.io/en/latest/usage.html
+
1.7.0 (2022-11-01)
~~~~~~~~~~~~~~~~~~
diff --git a/README.rst b/README.rst
index d5309ab..7fdc75e 100644
--- a/README.rst
+++ b/README.rst
@@ -19,9 +19,16 @@ Parsel
:alt: Coverage report
-Parsel is a BSD-licensed Python_ library to extract and remove data from HTML_
-and XML_ using XPath_ and CSS_ selectors, optionally combined with
-`regular expressions`_.
+Parsel is a BSD-licensed Python_ library to extract data from HTML_, JSON_, and
+XML_ documents.
+
+It supports:
+
+- CSS_ and XPath_ expressions for HTML and XML documents
+
+- JMESPath_ expressions for JSON documents
+
+- `Regular expressions`_
Find the Parsel online documentation at https://parsel.readthedocs.org.
@@ -30,15 +37,18 @@ Example (`open online demo`_):
.. code-block:: python
>>> from parsel import Selector
- >>> selector = Selector(text="""<html>
- <body>
- <h1>Hello, Parsel!</h1>
- <ul>
- <li><a href="http://example.com">Link 1</a></li>
- <li><a href="http://scrapy.org">Link 2</a></li>
- </ul>
- </body>
- </html>""")
+ >>> text = """
+ <html>
+ <body>
+ <h1>Hello, Parsel!</h1>
+ <ul>
+ <li><a href="http://example.com">Link 1</a></li>
+ <li><a href="http://scrapy.org">Link 2</a></li>
+ </ul>
+ <script type="application/json">{"a": ["b", "c"]}</script>
+ </body>
+ </html>"""
+ >>> selector = Selector(text=text)
>>> selector.css('h1::text').get()
'Hello, Parsel!'
>>> selector.xpath('//h1/text()').re(r'\w+')
@@ -47,12 +57,18 @@ Example (`open online demo`_):
... print(li.xpath('.//@href').get())
http://example.com
http://scrapy.org
-
+ >>> selector.css('script::text').jmespath("a").get()
+ 'b'
+ >>> selector.css('script::text').jmespath("a").getall()
+ ['b', 'c']
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
.. _HTML: https://en.wikipedia.org/wiki/HTML
+.. _JMESPath: https://jmespath.org/
+.. _JSON: https://en.wikipedia.org/wiki/JSON
.. _open online demo: https://colab.research.google.com/drive/149VFa6Px3wg7S3SEnUqk--TyBrKplxCN#forceEdit=true&sandboxMode=true
.. _Python: https://www.python.org/
.. _regular expressions: https://docs.python.org/library/re.html
.. _XML: https://en.wikipedia.org/wiki/XML
.. _XPath: https://en.wikipedia.org/wiki/XPath
+
diff --git a/debian/changelog b/debian/changelog
index 340acab..a1dab69 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+python-parsel (1.8.1+dfsg-1) UNRELEASED; urgency=low
+
+ * New upstream release.
+
+ -- Debian Janitor <janitor@jelmer.uk> Wed, 03 May 2023 01:32:25 -0000
+
python-parsel (1.7.0+dfsg-1) unstable; urgency=medium
* New upstream version.
diff --git a/debian/patches/0001-Skip-some-doctests-that-don-t-work-in-a-package.patch b/debian/patches/0001-Skip-some-doctests-that-don-t-work-in-a-package.patch
index f784d62..efa5652 100644
--- a/debian/patches/0001-Skip-some-doctests-that-don-t-work-in-a-package.patch
+++ b/debian/patches/0001-Skip-some-doctests-that-don-t-work-in-a-package.patch
@@ -6,11 +6,11 @@ Subject: Skip some doctests that don't work in a package.
docs/usage.rst | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
-diff --git a/docs/usage.rst b/docs/usage.rst
-index dcef13d..c62bb51 100644
---- a/docs/usage.rst
-+++ b/docs/usage.rst
-@@ -858,6 +858,7 @@ Using CSS selectors in multi-root documents
+Index: python-parsel.git/docs/usage.rst
+===================================================================
+--- python-parsel.git.orig/docs/usage.rst
++++ python-parsel.git/docs/usage.rst
+@@ -878,6 +878,7 @@ Using CSS selectors in multi-root docume
Some webpages may have multiple root elements. It can happen, for example, when
a webpage has broken code, such as missing closing tags.
@@ -18,7 +18,7 @@ index dcef13d..c62bb51 100644
.. invisible-code-block: python
selector = load_selector('multiroot.html')
-@@ -877,6 +878,7 @@ you need to precede your CSS query by an XPath query that reaches all root
+@@ -897,6 +898,7 @@ you need to precede your CSS query by an
elements::
selector.xpath('/*').css('<your CSS selector>')
@@ -26,7 +26,7 @@ index dcef13d..c62bb51 100644
Command-Line Interface Tools
-@@ -973,8 +975,6 @@ Let's download the atom feed using :mod:`requests` and create a selector:
+@@ -993,8 +995,6 @@ Let's download the atom feed using :mod:
>>> text = requests.get('https://feeds.feedburner.com/PythonInsider').text
>>> sel = Selector(text=text, type='xml')
@@ -35,8 +35,8 @@ index dcef13d..c62bb51 100644
.. invisible-code-block: python
sel = load_selector('python-insider.xml', type='xml')
-@@ -1015,6 +1015,8 @@ directly by their names::
- <Selector xpath='//link' data='<link rel="next" type="application/at...'>,
+@@ -1035,6 +1035,8 @@ directly by their names::
+ <Selector query='//link' data='<link rel="next" type="application/at...'>,
...]
+.. skip: end
@@ -44,7 +44,7 @@ index dcef13d..c62bb51 100644
If you wonder why the namespace removal procedure isn't called always by default
instead of having to call it manually, this is because of two reasons, which, in order
of relevance, are:
-@@ -1047,8 +1049,6 @@ Let's use the same Python Insider Atom feed:
+@@ -1067,8 +1069,6 @@ Let's use the same Python Insider Atom f
>>> text = requests.get('https://feeds.feedburner.com/PythonInsider').text
>>> sel = Selector(text=text, type='xml')
@@ -53,7 +53,7 @@ index dcef13d..c62bb51 100644
.. invisible-code-block: python
sel = load_selector('python-insider.xml', type='xml')
-@@ -1070,6 +1070,8 @@ You can pass several namespaces (here we're using shorter 1-letter prefixes)::
+@@ -1090,6 +1090,8 @@ You can pass several namespaces (here we
'https://img1.blogblog.com/img/b16-rounded.gif',
...]
diff --git a/docs/conf.py b/docs/conf.py
index 3e877e9..689af48 100755
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -3,8 +3,6 @@
import os
import sys
-import parsel
-
# Get the project root dir, which is the parent dir of this
cwd = os.getcwd()
@@ -15,6 +13,8 @@ project_root = os.path.dirname(cwd)
# version is used.
sys.path.insert(0, project_root)
+import parsel
+
# -- General configuration ---------------------------------------------
@@ -134,6 +134,8 @@ intersphinx_mapping = {
# nitpicky = True # https://github.com/scrapy/cssselect/pull/110
nitpick_ignore = [
+ ("py:class", "ExpressionError"),
+ ("py:class", "SelectorSyntaxError"),
("py:class", "cssselect.xpath.GenericTranslator"),
("py:class", "cssselect.xpath.HTMLTranslator"),
("py:class", "cssselect.xpath.XPathExpr"),
diff --git a/docs/usage.rst b/docs/usage.rst
index dcef13d..e3eb91f 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -4,32 +4,38 @@
Usage
=====
-Create a :class:`~parsel.selector.Selector` object for the HTML or XML text
-that you want to parse::
+Create a :class:`~parsel.selector.Selector` object for your input text.
+
+For HTML or XML, use `CSS`_ or `XPath`_ expressions to select data::
>>> from parsel import Selector
- >>> text = "<html><body><h1>Hello, Parsel!</h1></body></html>"
- >>> selector = Selector(text=text)
+ >>> html_text = "<html><body><h1>Hello, Parsel!</h1></body></html>"
+ >>> html_selector = Selector(text=html_text)
+ >>> html_selector.css('h1')
+ [<Selector query='descendant-or-self::h1' data='<h1>Hello, Parsel!</h1>'>]
+ >>> html_selector.xpath('//h1') # the same, but now with XPath
+ [<Selector query='//h1' data='<h1>Hello, Parsel!</h1>'>]
-Then use `CSS`_ or `XPath`_ expressions to select elements::
+For JSON, use `JMESPath`_ expressions to select data::
- >>> selector.css('h1')
- [<Selector xpath='descendant-or-self::h1' data='<h1>Hello, Parsel!</h1>'>]
- >>> selector.xpath('//h1') # the same, but now with XPath
- [<Selector xpath='//h1' data='<h1>Hello, Parsel!</h1>'>]
+ >>> json_text = '{"title":"Hello, Parsel!"}'
+ >>> json_selector = Selector(text=json_text)
+ >>> json_selector.jmespath('title')
+ [<Selector query='title' data='Hello, Parsel!'>]
And extract data from those elements::
- >>> selector.css('h1::text').get()
+ >>> html_selector.xpath('//h1/text()').get()
'Hello, Parsel!'
- >>> selector.xpath('//h1/text()').getall()
+ >>> json_selector.jmespath('title').getall()
['Hello, Parsel!']
.. _CSS: https://www.w3.org/TR/selectors
.. _XPath: https://www.w3.org/TR/xpath
+.. _JMESPath: https://jmespath.org/
-Learning CSS and XPath
-======================
+Learning expression languages
+=============================
`CSS`_ is a language for applying styles to HTML documents. It defines
selectors to associate those styles with specific HTML elements. Resources to
@@ -39,6 +45,11 @@ learn CSS_ selectors include:
- `XPath/CSS Equivalents in Wikibooks`_
+Parsel support for CSS selectors comes from cssselect, so read about `CSS
+selectors supported by cssselect`_.
+
+.. _CSS selectors supported by cssselect: https://cssselect.readthedocs.io/en/latest/#supported-selectors
+
`XPath`_ is a language for selecting nodes in XML documents, which can also be
used with HTML. Resources to learn XPath_ include:
@@ -46,13 +57,22 @@ used with HTML. Resources to learn XPath_ include:
- `XPath cheatsheet`_
-You can use either CSS_ or XPath_. CSS_ is usually more readable, but some
-things can only be done with XPath_.
+For HTML and XML input, you can use either CSS_ or XPath_. CSS_ is usually
+more readable, but some things can only be done with XPath_.
+
+JMESPath_ allows you to declaratively specify how to extract elements from
+a JSON document. Resources to learn JMESPath_ include:
+
+- `JMESPath Tutorial`_
+
+- `JMESPath Specification`_
.. _CSS selectors in the MDN: https://developer.mozilla.org/en-US/docs/Learn/CSS/Building_blocks/Selectors
.. _XPath cheatsheet: https://devhints.io/xpath
.. _XPath Tutorial in W3Schools: https://www.w3schools.com/xml/xpath_intro.asp
.. _XPath/CSS Equivalents in Wikibooks: https://en.wikibooks.org/wiki/XPath/CSS_Equivalents
+.. _JMESPath Tutorial: https://jmespath.org/tutorial.html
+.. _JMESPath Specification: https://jmespath.org/specification.html
Using selectors
@@ -95,12 +115,12 @@ So, by looking at the :ref:`HTML code <topics-selectors-htmlcode>` of that
page, let's construct an XPath for selecting the text inside the title tag::
>>> selector.xpath('//title/text()')
- [<Selector xpath='//title/text()' data='Example website'>]
+ [<Selector query='//title/text()' data='Example website'>]
You can also ask the same thing using CSS instead::
>>> selector.css('title::text')
- [<Selector xpath='descendant-or-self::title/text()' data='Example website'>]
+ [<Selector query='descendant-or-self::title/text()' data='Example website'>]
To actually extract the textual data, you must call the selector ``.get()``
or ``.getall()`` methods, as follows::
@@ -597,10 +617,10 @@ returns ``True`` for nodes that have all of the specified HTML classes::
... """)
...
>>> sel.xpath('//p[has-class("foo")]')
- [<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
- <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
+ [<Selector query='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
+ <Selector query='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
>>> sel.xpath('//p[has-class("foo", "bar-baz")]')
- [<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
+ [<Selector query='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
>>> sel.xpath('//p[has-class("foo", "bar")]')
[]
@@ -1011,8 +1031,8 @@ directly by their names::
>>> sel.remove_namespaces()
>>> sel.xpath("//link")
- [<Selector xpath='//link' data='<link rel="alternate" type="text/html...'>,
- <Selector xpath='//link' data='<link rel="next" type="application/at...'>,
+ [<Selector query='//link' data='<link rel="alternate" type="text/html...'>,
+ <Selector query='//link' data='<link rel="next" type="application/at...'>,
...]
If you wonder why the namespace removal procedure isn't called always by default
@@ -1057,8 +1077,8 @@ And try to select the links again, now using an "atom:" prefix
for the "link" node test::
>>> sel.xpath("//atom:link", namespaces={"atom": "http://www.w3.org/2005/Atom"})
- [<Selector xpath='//atom:link' data='<link xmlns="http://www.w3.org/2005/A...'>,
- <Selector xpath='//atom:link' data='<link xmlns="http://www.w3.org/2005/A...'>,
+ [<Selector query='//atom:link' data='<link xmlns="http://www.w3.org/2005/A...'>,
+ <Selector query='//atom:link' data='<link xmlns="http://www.w3.org/2005/A...'>,
...]
You can pass several namespaces (here we're using shorter 1-letter prefixes)::
diff --git a/parsel/__init__.py b/parsel/__init__.py
index a0d7f7c..955f3df 100644
--- a/parsel/__init__.py
+++ b/parsel/__init__.py
@@ -5,7 +5,7 @@ or CSS selectors
__author__ = "Scrapy project"
__email__ = "info@scrapy.org"
-__version__ = "1.7.0"
+__version__ = "1.8.1"
__all__ = [
"Selector",
"SelectorList",
diff --git a/parsel/csstranslator.py b/parsel/csstranslator.py
index c240e6a..c4f95e6 100644
--- a/parsel/csstranslator.py
+++ b/parsel/csstranslator.py
@@ -1,19 +1,30 @@
from functools import lru_cache
+from typing import TYPE_CHECKING, Any, Optional
from cssselect import GenericTranslator as OriginalGenericTranslator
from cssselect import HTMLTranslator as OriginalHTMLTranslator
from cssselect.xpath import XPathExpr as OriginalXPathExpr
from cssselect.xpath import ExpressionError
-from cssselect.parser import FunctionalPseudoElement
+from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
+
+
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
class XPathExpr(OriginalXPathExpr):
- textnode = False
- attribute = None
+ textnode: bool = False
+ attribute: Optional[str] = None
@classmethod
- def from_xpath(cls, xpath, textnode=False, attribute=None):
+ def from_xpath(
+ cls,
+ xpath: OriginalXPathExpr,
+ textnode: bool = False,
+ attribute: Optional[str] = None,
+ ) -> "Self":
x = cls(
path=xpath.path, element=xpath.element, condition=xpath.condition
)
@@ -21,7 +32,7 @@ class XPathExpr(OriginalXPathExpr):
x.attribute = attribute
return x
- def __str__(self):
+ def __str__(self) -> str:
path = super().__str__()
if self.textnode:
if path == "*":
@@ -38,38 +49,67 @@ class XPathExpr(OriginalXPathExpr):
return path
- def join(self, combiner, other, *args, **kwargs):
+ def join(
+ self: "Self",
+ combiner: str,
+ other: OriginalXPathExpr,
+ *args: Any,
+ **kwargs: Any,
+ ) -> "Self":
+ if not isinstance(other, XPathExpr):
+ raise ValueError(
+ f"Expressions of type {__name__}.XPathExpr can ony join expressions"
+ f" of the same type (or its descendants), got {type(other)}"
+ )
super().join(combiner, other, *args, **kwargs)
self.textnode = other.textnode
self.attribute = other.attribute
return self
+if TYPE_CHECKING:
+ # requires Python 3.8
+ from typing import Protocol
+
+ # e.g. cssselect.GenericTranslator, cssselect.HTMLTranslator
+ class TranslatorProtocol(Protocol):
+ def xpath_element(self, selector: Element) -> OriginalXPathExpr:
+ pass
+
+ def css_to_xpath(self, css: str, prefix: str = ...) -> str:
+ pass
+
+
class TranslatorMixin:
"""This mixin adds support to CSS pseudo elements via dynamic dispatch.
Currently supported pseudo-elements are ``::text`` and ``::attr(ATTR_NAME)``.
"""
- def xpath_element(self, selector):
- xpath = super().xpath_element(selector)
+ def xpath_element(
+ self: "TranslatorProtocol", selector: Element
+ ) -> XPathExpr:
+ # https://github.com/python/mypy/issues/12344
+ xpath = super().xpath_element(selector) # type: ignore[safe-super]
return XPathExpr.from_xpath(xpath)
- def xpath_pseudo_element(self, xpath, pseudo_element):
+ def xpath_pseudo_element(
+ self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement
+ ) -> OriginalXPathExpr:
"""
Dispatch method that transforms XPath to support pseudo-element
"""
if isinstance(pseudo_element, FunctionalPseudoElement):
- method = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element"
- method = getattr(self, method, None)
+ method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element"
+ method = getattr(self, method_name, None)
if not method:
raise ExpressionError(
f"The functional pseudo-element ::{pseudo_element.name}() is unknown"
)
xpath = method(xpath, pseudo_element)
else:
- method = f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element"
- method = getattr(self, method, None)
+ method_name = f"xpath_{pseudo_element.replace('-', '_')}_simple_pseudo_element"
+ method = getattr(self, method_name, None)
if not method:
raise ExpressionError(
f"The pseudo-element ::{pseudo_element} is unknown"
@@ -77,7 +117,9 @@ class TranslatorMixin:
xpath = method(xpath)
return xpath
- def xpath_attr_functional_pseudo_element(self, xpath, function):
+ def xpath_attr_functional_pseudo_element(
+ self, xpath: OriginalXPathExpr, function: FunctionalPseudoElement
+ ) -> XPathExpr:
"""Support selecting attribute values using ::attr() pseudo-element"""
if function.argument_types() not in (["STRING"], ["IDENT"]):
raise ExpressionError(
@@ -87,26 +129,32 @@ class TranslatorMixin:
xpath, attribute=function.arguments[0].value
)
- def xpath_text_simple_pseudo_element(self, xpath):
+ def xpath_text_simple_pseudo_element(
+ self, xpath: OriginalXPathExpr
+ ) -> XPathExpr:
"""Support selecting text nodes using ::text pseudo-element"""
return XPathExpr.from_xpath(xpath, textnode=True)
class GenericTranslator(TranslatorMixin, OriginalGenericTranslator):
@lru_cache(maxsize=256)
- def css_to_xpath(self, css, prefix="descendant-or-self::"):
+ def css_to_xpath(
+ self, css: str, prefix: str = "descendant-or-self::"
+ ) -> str:
return super().css_to_xpath(css, prefix)
class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
@lru_cache(maxsize=256)
- def css_to_xpath(self, css, prefix="descendant-or-self::"):
+ def css_to_xpath(
+ self, css: str, prefix: str = "descendant-or-self::"
+ ) -> str:
return super().css_to_xpath(css, prefix)
_translator = HTMLTranslator()
-def css2xpath(query):
+def css2xpath(query: str) -> str:
"Return translated XPath version of a given CSS query"
return _translator.css_to_xpath(query)
diff --git a/parsel/selector.py b/parsel/selector.py
index 4e5b27c..8d884a4 100644
--- a/parsel/selector.py
+++ b/parsel/selector.py
@@ -1,9 +1,10 @@
-"""
-XPath selectors based on lxml
-"""
+"""XPath and JMESPath selectors based on the lxml and jmespath Python
+packages."""
+import json
import typing
import warnings
+from io import BytesIO
from typing import (
Any,
Dict,
@@ -11,20 +12,25 @@ from typing import (
Mapping,
Optional,
Pattern,
+ Tuple,
Type,
TypeVar,
Union,
)
from warnings import warn
-from cssselect import GenericTranslator as OriginalGenericTranslator
+try:
+ from typing import TypedDict # pylint: disable=ungrouped-imports
+except ImportError: # Python 3.7
+ from typing_extensions import TypedDict
+
+import jmespath
from lxml import etree, html
-from pkg_resources import parse_version
+from packaging.version import Version
from .csstranslator import GenericTranslator, HTMLTranslator
from .utils import extract_regex, flatten, iflatten, shorten
-
if typing.TYPE_CHECKING:
# both require Python 3.8
from typing import Literal, SupportsIndex
@@ -39,8 +45,8 @@ if typing.TYPE_CHECKING:
_SelectorType = TypeVar("_SelectorType", bound="Selector")
_ParserType = Union[etree.XMLParser, etree.HTMLParser]
-lxml_version = parse_version(etree.__version__)
-lxml_huge_tree_version = parse_version("4.2")
+lxml_version = Version(etree.__version__)
+lxml_huge_tree_version = Version("4.2")
LXML_SUPPORTS_HUGE_TREE = lxml_version >= lxml_huge_tree_version
@@ -56,13 +62,19 @@ class CannotDropElementWithoutParent(CannotRemoveElementWithoutParent):
pass
-class SafeXMLParser(etree.XMLParser):
- def __init__(self, *args, **kwargs) -> None:
+class SafeXMLParser(etree.XMLParser): # type: ignore[type-arg]
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
kwargs.setdefault("resolve_entities", False)
super().__init__(*args, **kwargs)
-_ctgroup = {
+class CTGroupValue(TypedDict):
+ _parser: Union[Type[etree.XMLParser], Type[html.HTMLParser]] # type: ignore[type-arg]
+ _csstranslator: Union[GenericTranslator, HTMLTranslator]
+ _tostring_method: str
+
+
+_ctgroup: Dict[str, CTGroupValue] = {
"html": {
"_parser": html.HTMLParser,
"_csstranslator": HTMLTranslator(),
@@ -76,13 +88,8 @@ _ctgroup = {
}
-def _st(st: Optional[str]) -> str:
- if st is None:
- return "html"
- elif st in _ctgroup:
- return st
- else:
- raise ValueError(f"Invalid type: {st}")
+def _xml_or_html(type: Optional[str]) -> str:
+ return "xml" if type == "xml" else "html"
def create_root_node(
@@ -90,16 +97,21 @@ def create_root_node(
parser_cls: Type[_ParserType],
base_url: Optional[str] = None,
huge_tree: bool = LXML_SUPPORTS_HUGE_TREE,
+ body: bytes = b"",
+ encoding: str = "utf8",
) -> etree._Element:
"""Create root node for text using given parser class."""
- body = text.strip().replace("\x00", "").encode("utf8") or b"<html/>"
+ if not text:
+ body = body.replace(b"\x00", b"").strip()
+ else:
+ body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"
+
if huge_tree and LXML_SUPPORTS_HUGE_TREE:
- parser = parser_cls(recover=True, encoding="utf8", huge_tree=True)
- # the stub wrongly thinks base_url can't be None
- root = etree.fromstring(body, parser=parser, base_url=base_url) # type: ignore[arg-type]
+ parser = parser_cls(recover=True, encoding=encoding, huge_tree=True)
+ root = etree.fromstring(body, parser=parser, base_url=base_url)
else:
- parser = parser_cls(recover=True, encoding="utf8")
- root = etree.fromstring(body, parser=parser, base_url=base_url) # type: ignore[arg-type]
+ parser = parser_cls(recover=True, encoding=encoding)
+ root = etree.fromstring(body, parser=parser, base_url=base_url)
for error in parser.error_log:
if "use XML_PARSE_HUGE option" in error.message:
warnings.warn(
@@ -139,17 +151,35 @@ class SelectorList(List[_SelectorType]):
def __getstate__(self) -> None:
raise TypeError("can't pickle SelectorList objects")
+ def jmespath(
+ self, query: str, **kwargs: Any
+ ) -> "SelectorList[_SelectorType]":
+ """
+ Call the ``.jmespath()`` method for each element in this list and return
+ their results flattened as another :class:`SelectorList`.
+
+ ``query`` is the same argument as the one in :meth:`Selector.jmespath`.
+
+ Any additional named arguments are passed to the underlying
+ ``jmespath.search`` call, e.g.::
+
+ selector.jmespath('author.name', options=jmespath.Options(dict_cls=collections.OrderedDict))
+ """
+ return self.__class__(
+ flatten([x.jmespath(query, **kwargs) for x in self])
+ )
+
def xpath(
self,
xpath: str,
namespaces: Optional[Mapping[str, str]] = None,
- **kwargs,
+ **kwargs: Any,
) -> "SelectorList[_SelectorType]":
"""
Call the ``.xpath()`` method for each element in this list and return
their results flattened as another :class:`SelectorList`.
- ``query`` is the same argument as the one in :meth:`Selector.xpath`
+ ``xpath`` is the same argument as the one in :meth:`Selector.xpath`
``namespaces`` is an optional ``prefix: namespace-uri`` mapping (dict)
for additional prefixes to those registered with ``register_namespace(prefix, uri)``.
@@ -230,7 +260,7 @@ class SelectorList(List[_SelectorType]):
for el in iflatten(
x.re(regex, replace_entities=replace_entities) for x in self
):
- return el
+ return typing.cast(str, el)
return default
def getall(self) -> List[str]:
@@ -250,7 +280,7 @@ class SelectorList(List[_SelectorType]):
def get(self, default: str) -> str:
pass
- def get(self, default: Optional[str] = None) -> Optional[str]:
+ def get(self, default: Optional[str] = None) -> Any:
"""
Return the result of ``.get()`` for the first element in this list.
If the list is empty, return the default value.
@@ -290,15 +320,109 @@ class SelectorList(List[_SelectorType]):
x.drop()
+_NOT_SET = object()
+
+
+def _get_root_from_text(
+ text: str, *, type: str, **lxml_kwargs: Any
+) -> etree._Element:
+ return create_root_node(text, _ctgroup[type]["_parser"], **lxml_kwargs)
+
+
+def _get_root_and_type_from_bytes(
+ body: bytes,
+ encoding: str,
+ *,
+ input_type: Optional[str],
+ **lxml_kwargs: Any,
+) -> Tuple[Any, str]:
+ if input_type == "text":
+ return body.decode(encoding), input_type
+ if encoding == "utf8":
+ try:
+ data = json.load(BytesIO(body))
+ except ValueError:
+ data = _NOT_SET
+ if data is not _NOT_SET:
+ return data, "json"
+ if input_type == "json":
+ return None, "json"
+ assert input_type in ("html", "xml", None) # nosec
+ type = _xml_or_html(input_type)
+ root = create_root_node(
+ text="",
+ body=body,
+ encoding=encoding,
+ parser_cls=_ctgroup[type]["_parser"],
+ **lxml_kwargs,
+ )
+ return root, type
+
+
+def _get_root_and_type_from_text(
+ text: str, *, input_type: Optional[str], **lxml_kwargs: Any
+) -> Tuple[Any, str]:
+ if input_type == "text":
+ return text, input_type
+ try:
+ data = json.loads(text)
+ except ValueError:
+ data = _NOT_SET
+ if data is not _NOT_SET:
+ return data, "json"
+ if input_type == "json":
+ return None, "json"
+ assert input_type in ("html", "xml", None) # nosec
+ type = _xml_or_html(input_type)
+ root = _get_root_from_text(text, type=type, **lxml_kwargs)
+ return root, type
+
+
+def _get_root_type(root: Any, *, input_type: Optional[str]) -> str:
+ if isinstance(root, etree._Element): # pylint: disable=protected-access
+ if input_type in {"json", "text"}:
+ raise ValueError(
+ f"Selector got an lxml.etree._Element object as root, "
+ f"and {input_type!r} as type."
+ )
+ return _xml_or_html(input_type)
+ elif isinstance(root, (dict, list)) or _is_valid_json(root):
+ return "json"
+ return input_type or "json"
+
+
+def _is_valid_json(text: str) -> bool:
+ try:
+ json.loads(text)
+ except (TypeError, ValueError):
+ return False
+ else:
+ return True
+
+
+def _load_json_or_none(text: str) -> Any:
+ if isinstance(text, (str, bytes, bytearray)):
+ try:
+ return json.loads(text)
+ except ValueError:
+ return None
+ return None
+
+
class Selector:
- """
- :class:`Selector` allows you to select parts of an XML or HTML text using CSS
- or XPath expressions and extract data from it.
+ """Wrapper for input data in HTML, JSON, or XML format, that allows
+ selecting parts of it using selection expressions.
+
+ You can write selection expressions in CSS or XPath for HTML and XML
+ inputs, or in JMESPath for JSON inputs.
+
+ ``text`` is an ``str`` object.
- ``text`` is a `str`` object
+ ``body`` is a ``bytes`` object. It can be used together with the
+ ``encoding`` argument instead of the ``text`` argument.
- ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
- If ``type`` is ``None``, the selector defaults to ``"html"``.
+ ``type`` defines the selector type. It can be ``"html"`` (default),
+ ``"json"``, or ``"xml"``.
``base_url`` allows setting a URL for the document. This is needed when looking up external entities with relative paths.
See the documentation for :func:`lxml.etree.fromstring` for more information.
@@ -313,18 +437,16 @@ class Selector:
"""
__slots__ = [
- "text",
"namespaces",
"type",
"_expr",
+ "_huge_tree",
"root",
+ "_text",
+ "body",
"__weakref__",
- "_parser",
- "_csstranslator",
- "_tostring_method",
]
- _default_type: Optional[str] = None
_default_namespaces = {
"re": "http://exslt.org/regular-expressions",
# supported in libxslt:
@@ -342,55 +464,141 @@ class Selector:
self,
text: Optional[str] = None,
type: Optional[str] = None,
+ body: bytes = b"",
+ encoding: str = "utf8",
namespaces: Optional[Mapping[str, str]] = None,
- root: Optional[Any] = None,
+ root: Optional[Any] = _NOT_SET,
base_url: Optional[str] = None,
_expr: Optional[str] = None,
huge_tree: bool = LXML_SUPPORTS_HUGE_TREE,
) -> None:
- self.type = st = _st(type or self._default_type)
- self._parser: Type[_ParserType] = typing.cast(
- Type[_ParserType], _ctgroup[st]["_parser"]
- )
- self._csstranslator: OriginalGenericTranslator = typing.cast(
- OriginalGenericTranslator, _ctgroup[st]["_csstranslator"]
- )
- self._tostring_method: "_TostringMethodType" = typing.cast(
- "_TostringMethodType", _ctgroup[st]["_tostring_method"]
- )
+ self.root: Any
+ if type not in ("html", "json", "text", "xml", None):
+ raise ValueError(f"Invalid type: {type}")
+
+ if text is None and not body and root is _NOT_SET:
+ raise ValueError("Selector needs text, body, or root arguments")
+
+ if text is not None and not isinstance(text, str):
+ msg = f"text argument should be of type str, got {text.__class__}"
+ raise TypeError(msg)
if text is not None:
+ if root is not _NOT_SET:
+ warnings.warn(
+ "Selector got both text and root, root is being ignored.",
+ stacklevel=2,
+ )
if not isinstance(text, str):
msg = f"text argument should be of type str, got {text.__class__}"
raise TypeError(msg)
- root = self._get_root(text, base_url, huge_tree)
- elif root is None:
- raise ValueError("Selector needs either text or root argument")
+
+ root, type = _get_root_and_type_from_text(
+ text,
+ input_type=type,
+ base_url=base_url,
+ huge_tree=huge_tree,
+ )
+ self.root = root
+ self.type = type
+ elif body:
+ if not isinstance(body, bytes):
+ msg = f"body argument should be of type bytes, got {body.__class__}"
+ raise TypeError(msg)
+ root, type = _get_root_and_type_from_bytes(
+ body=body,
+ encoding=encoding,
+ input_type=type,
+ base_url=base_url,
+ huge_tree=huge_tree,
+ )
+ self.root = root
+ self.type = type
+ elif root is _NOT_SET:
+ raise ValueError("Selector needs text, body, or root arguments")
+ else:
+ self.root = root
+ self.type = _get_root_type(root, input_type=type)
self.namespaces = dict(self._default_namespaces)
if namespaces is not None:
self.namespaces.update(namespaces)
- self.root = root
+
self._expr = _expr
+ self._huge_tree = huge_tree
+ self._text = text
def __getstate__(self) -> Any:
raise TypeError("can't pickle Selector objects")
def _get_root(
self,
- text: str,
+ text: str = "",
base_url: Optional[str] = None,
huge_tree: bool = LXML_SUPPORTS_HUGE_TREE,
+ type: Optional[str] = None,
+ body: bytes = b"",
+ encoding: str = "utf8",
) -> etree._Element:
return create_root_node(
- text, self._parser, base_url=base_url, huge_tree=huge_tree
+ text,
+ body=body,
+ encoding=encoding,
+ parser_cls=_ctgroup[type or self.type]["_parser"],
+ base_url=base_url,
+ huge_tree=huge_tree,
+ )
+
+ def jmespath(
+ self: _SelectorType,
+ query: str,
+ **kwargs: Any,
+ ) -> SelectorList[_SelectorType]:
+ """
+ Find objects matching the JMESPath ``query`` and return the result as a
+ :class:`SelectorList` instance with all elements flattened. List
+ elements implement :class:`Selector` interface too.
+
+ ``query`` is a string containing the `JMESPath
+ <https://jmespath.org/>`_ query to apply.
+
+ Any additional named arguments are passed to the underlying
+ ``jmespath.search`` call, e.g.::
+
+ selector.jmespath('author.name', options=jmespath.Options(dict_cls=collections.OrderedDict))
+ """
+ if self.type == "json":
+ if isinstance(self.root, str):
+ # Selector received a JSON string as root.
+ data = _load_json_or_none(self.root)
+ else:
+ data = self.root
+ else:
+ assert self.type in {"html", "xml"} # nosec
+ data = _load_json_or_none(self.root.text)
+
+ result = jmespath.search(query, data, **kwargs)
+ if result is None:
+ result = []
+ elif not isinstance(result, list):
+ result = [result]
+
+ def make_selector(x: Any) -> _SelectorType: # closure function
+ if isinstance(x, str):
+ return self.__class__(text=x, _expr=query, type="text")
+ else:
+ return self.__class__(root=x, _expr=query)
+
+ result = [make_selector(x) for x in result]
+ return typing.cast(
+ SelectorList[_SelectorType], self.selectorlist_cls(result)
)
def xpath(
self: _SelectorType,
query: str,
namespaces: Optional[Mapping[str, str]] = None,
- **kwargs,
+ **kwargs: Any,
) -> SelectorList[_SelectorType]:
"""
Find nodes matching the xpath ``query`` and return the result as a
@@ -409,12 +617,24 @@ class Selector:
selector.xpath('//a[href=$url]', url="http://www.example.com")
"""
- try:
- xpathev = self.root.xpath
- except AttributeError:
- return typing.cast(
- SelectorList[_SelectorType], self.selectorlist_cls([])
+ if self.type not in ("html", "xml", "text"):
+ raise ValueError(
+ f"Cannot use xpath on a Selector of type {self.type!r}"
)
+ if self.type in ("html", "xml"):
+ try:
+ xpathev = self.root.xpath
+ except AttributeError:
+ return typing.cast(
+ SelectorList[_SelectorType], self.selectorlist_cls([])
+ )
+ else:
+ try:
+ xpathev = self._get_root(self._text or "", type="html").xpath
+ except AttributeError:
+ return typing.cast(
+ SelectorList[_SelectorType], self.selectorlist_cls([])
+ )
nsp = dict(self.namespaces)
if namespaces is not None:
@@ -434,7 +654,10 @@ class Selector:
result = [
self.__class__(
- root=x, _expr=query, namespaces=self.namespaces, type=self.type
+ root=x,
+ _expr=query,
+ namespaces=self.namespaces,
+ type=_xml_or_html(self.type),
)
for x in result
]
@@ -453,10 +676,15 @@ class Selector:
.. _cssselect: https://pypi.python.org/pypi/cssselect/
"""
+ if self.type not in ("html", "xml", "text"):
+ raise ValueError(
+ f"Cannot use css on a Selector of type {self.type!r}"
+ )
return self.xpath(self._css2xpath(query))
def _css2xpath(self, query: str) -> str:
- return self._csstranslator.css_to_xpath(query)
+ type = _xml_or_html(self.type)
+ return _ctgroup[type]["_csstranslator"].css_to_xpath(query)
def re(
self, regex: Union[str, Pattern[str]], replace_entities: bool = True
@@ -473,9 +701,8 @@ class Selector:
Passing ``replace_entities`` as ``False`` switches off these
replacements.
"""
- return extract_regex(
- regex, self.get(), replace_entities=replace_entities
- )
+ data = self.get()
+ return extract_regex(regex, data, replace_entities=replace_entities)
@typing.overload
def re_first(
@@ -516,17 +743,22 @@ class Selector:
default,
)
- def get(self) -> str:
+ def get(self) -> Any:
"""
Serialize and return the matched nodes in a single string.
Percent encoded content is unquoted.
"""
+ if self.type in ("text", "json"):
+ return self.root
try:
- return etree.tostring(
- self.root,
- method=self._tostring_method,
- encoding="unicode",
- with_tail=False,
+ return typing.cast(
+ str,
+ etree.tostring(
+ self.root,
+ method=_ctgroup[self.type]["_tostring_method"],
+ encoding="unicode",
+ with_tail=False,
+ ),
)
except (AttributeError, TypeError):
if self.root is True:
@@ -563,10 +795,7 @@ class Selector:
# loop on element attributes also
for an in el.attrib:
if an.startswith("{"):
- # this cast shouldn't be needed as pop never returns None
- el.attrib[an.split("}", 1)[1]] = typing.cast(
- str, el.attrib.pop(an)
- )
+ el.attrib[an.split("}", 1)[1]] = el.attrib.pop(an)
# remove namespace declarations
etree.cleanup_namespaces(self.root)
@@ -591,7 +820,7 @@ class Selector:
)
try:
- parent.remove(self.root) # type: ignore[union-attr]
+ parent.remove(self.root)
except AttributeError:
# 'NoneType' object has no attribute 'remove'
raise CannotRemoveElementWithoutParent(
@@ -599,7 +828,7 @@ class Selector:
"are you trying to remove a root element?"
)
- def drop(self):
+ def drop(self) -> None:
"""
Drop matched nodes from the parent element.
"""
@@ -616,9 +845,11 @@ class Selector:
try:
if self.type == "xml":
+ if parent is None:
+ raise ValueError("This node has no parent")
parent.remove(self.root)
else:
- self.root.drop_tree()
+ typing.cast(html.HtmlElement, self.root).drop_tree()
except (AttributeError, AssertionError):
# 'NoneType' object has no attribute 'drop'
raise CannotDropElementWithoutParent(
@@ -643,6 +874,6 @@ class Selector:
def __str__(self) -> str:
data = repr(shorten(self.get(), width=40))
- return f"<{type(self).__name__} xpath={self._expr!r} data={data}>"
+ return f"<{type(self).__name__} query={self._expr!r} data={data}>"
__repr__ = __str__
diff --git a/parsel/utils.py b/parsel/utils.py
index 5e6d92d..2677f47 100644
--- a/parsel/utils.py
+++ b/parsel/utils.py
@@ -1,9 +1,9 @@
import re
-from typing import Any, List, Pattern, Union, cast, Match
+from typing import Any, Iterable, Iterator, List, Match, Pattern, Union, cast
from w3lib.html import replace_entities as w3lib_replace_entities
-def flatten(x):
+def flatten(x: Iterable[Any]) -> List[Any]:
"""flatten(sequence) -> list
Returns a single, flat list which contains all elements retrieved
from the sequence and all recursively contained sub-sequences
@@ -21,7 +21,7 @@ def flatten(x):
return list(iflatten(x))
-def iflatten(x):
+def iflatten(x: Iterable[Any]) -> Iterator[Any]:
"""iflatten(sequence) -> Iterator
Similar to ``.flatten()``, but returns iterator instead"""
for el in x:
diff --git a/parsel/xpathfuncs.py b/parsel/xpathfuncs.py
index 9e5c0a9..e8cea0a 100644
--- a/parsel/xpathfuncs.py
+++ b/parsel/xpathfuncs.py
@@ -1,13 +1,16 @@
import re
+from typing import Any, Callable, Optional
+
from lxml import etree
from w3lib.html import HTML5_WHITESPACE
+
regex = f"[{HTML5_WHITESPACE}]+"
replace_html5_whitespaces = re.compile(regex).sub
-def set_xpathfunc(fname, func):
+def set_xpathfunc(fname: str, func: Optional[Callable]) -> None: # type: ignore[type-arg]
"""Register a custom extension function to use in XPath expressions.
The function ``func`` registered under ``fname`` identifier will be called
@@ -21,18 +24,18 @@ def set_xpathfunc(fname, func):
.. _`in lxml documentation`: https://lxml.de/extensions.html#xpath-extension-functions
"""
- ns_fns = etree.FunctionNamespace(None)
+ ns_fns = etree.FunctionNamespace(None) # type: ignore[attr-defined]
if func is not None:
ns_fns[fname] = func
else:
del ns_fns[fname]
-def setup():
+def setup() -> None:
set_xpathfunc("has-class", has_class)
-def has_class(context, *classes):
+def has_class(context: Any, *classes: str) -> bool:
"""has-class function.
Return True if all ``classes`` are present in element's class attr.
diff --git a/pylintrc b/pylintrc
index 1892721..b044a4c 100644
--- a/pylintrc
+++ b/pylintrc
@@ -4,7 +4,6 @@ persistent=no
[MESSAGES CONTROL]
disable=c-extension-no-member,
- deprecated-method,
fixme,
import-error,
import-outside-toplevel,
@@ -26,4 +25,4 @@ disable=c-extension-no-member,
unused-argument,
use-a-generator,
wrong-import-order,
- wrong-import-position
+ wrong-import-position,
diff --git a/setup.py b/setup.py
index fbd8a6a..e67bcfb 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@ with open("NEWS", encoding="utf-8") as history_file:
setup(
name="parsel",
- version="1.7.0",
+ version="1.8.1",
description="Parsel is a library to extract data from HTML and XML using XPath and CSS selectors",
long_description=readme + "\n\n" + history,
author="Scrapy project",
@@ -26,8 +26,10 @@ setup(
include_package_data=True,
install_requires=[
"cssselect>=0.9",
+ "jmespath",
"lxml",
"packaging",
+ "typing_extensions; python_version < '3.8'",
"w3lib>=1.19.0",
],
python_requires=">=3.7",
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 6d89065..c96ecf5 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -5,7 +5,7 @@ import unittest
import pickle
import typing
-from typing import Any
+from typing import cast, Any, Optional, Mapping
from lxml import etree
from lxml.html import HtmlElement
@@ -15,6 +15,8 @@ from parsel import Selector, SelectorList
from parsel.selector import (
CannotRemoveElementWithoutRoot,
CannotRemoveElementWithoutParent,
+ LXML_SUPPORTS_HUGE_TREE,
+ _NOT_SET,
)
@@ -65,7 +67,8 @@ class SelectorTestCase(unittest.TestCase):
)
self.assertEqual(
- [x.extract() for x in sel.xpath("//input[@name='a']/@name")], ["a"]
+ [x.extract() for x in sel.xpath("//input[@name='a']/@name")],
+ ["a"],
)
self.assertEqual(
[
@@ -230,7 +233,7 @@ class SelectorTestCase(unittest.TestCase):
sel = self.sscls(text=body)
representation = (
- f"<Selector xpath='//input/@name' data='{37 * 'b'}...'>"
+ f"<Selector query='//input/@name' data='{37 * 'b'}...'>"
)
self.assertEqual(
@@ -241,7 +244,7 @@ class SelectorTestCase(unittest.TestCase):
body = f"<p><input name='{50 * 'b'}' value='\xa9'/></p>"
representation = (
- "<Selector xpath='//input[@value=\"©\"]/@value' data='©'>"
+ "<Selector query='//input[@value=\"©\"]/@value' data='©'>"
)
sel = self.sscls(text=body)
@@ -421,7 +424,7 @@ class SelectorTestCase(unittest.TestCase):
def test_text_or_root_is_required(self) -> None:
self.assertRaisesRegex(
ValueError,
- "Selector needs either text or root argument",
+ "Selector needs text, body, or root arguments",
self.sscls,
)
@@ -560,7 +563,8 @@ class SelectorTestCase(unittest.TestCase):
self.assertEqual(
x.xpath(
- "//somens:a/text()", namespaces={"somens": "http://scrapy.org"}
+ "//somens:a/text()",
+ namespaces={"somens": "http://scrapy.org"},
).extract(),
["take this"],
)
@@ -654,7 +658,8 @@ class SelectorTestCase(unittest.TestCase):
# "xmlns" is still defined
self.assertEqual(
x.xpath(
- "//xmlns:TestTag/@b:att", namespaces={"b": "http://somens.com"}
+ "//xmlns:TestTag/@b:att",
+ namespaces={"b": "http://somens.com"},
).extract()[0],
"value",
)
@@ -936,7 +941,8 @@ class SelectorTestCase(unittest.TestCase):
self.assertEqual(
len(
sel.xpath(
- "//f:link", namespaces={"f": "http://www.w3.org/2005/Atom"}
+ "//f:link",
+ namespaces={"f": "http://www.w3.org/2005/Atom"},
)
),
2,
@@ -1031,7 +1037,7 @@ class SelectorTestCase(unittest.TestCase):
selectorlist_cls = MySelectorList
def extra_method(self) -> str:
- return "extra" + self.get()
+ return "extra" + cast(str, self.get())
sel = MySelector(text="<html><div>foo</div></html>")
self.assertIsInstance(sel.xpath("//div"), MySelectorList)
@@ -1108,7 +1114,7 @@ class SelectorTestCase(unittest.TestCase):
sel.css("body").drop()
self.assertEqual(sel.get(), "<html></html>")
- def test_deep_nesting(self):
+ def test_deep_nesting(self) -> None:
lxml_version = parse_version(etree.__version__)
lxml_huge_tree_version = parse_version("4.2")
@@ -1179,6 +1185,73 @@ class SelectorTestCase(unittest.TestCase):
self.assertEqual(len(sel.css("span")), nest_level)
self.assertEqual(len(sel.css("td")), 1)
+ def test_invalid_type(self) -> None:
+ with self.assertRaises(ValueError):
+ self.sscls("", type="xhtml")
+
+ def test_default_type(self) -> None:
+ text = "foo"
+ selector = self.sscls(text)
+ self.assertEqual(selector.type, "html")
+
+ def test_json_type(self) -> None:
+ obj = 1
+ selector = self.sscls(str(obj), type="json")
+ self.assertEqual(selector.root, obj)
+ self.assertEqual(selector.type, "json")
+
+ def test_html_root(self) -> None:
+ root = etree.fromstring("<html/>")
+ selector = self.sscls(root=root)
+ self.assertEqual(selector.root, root)
+ self.assertEqual(selector.type, "html")
+
+ def test_json_root(self) -> None:
+ obj = 1
+ selector = self.sscls(root=obj)
+ self.assertEqual(selector.root, obj)
+ self.assertEqual(selector.type, "json")
+
+ def test_json_xpath(self) -> None:
+ obj = 1
+ selector = self.sscls(root=obj)
+ with self.assertRaises(ValueError):
+ selector.xpath("//*")
+
+ def test_json_css(self) -> None:
+ obj = 1
+ selector = self.sscls(root=obj)
+ with self.assertRaises(ValueError):
+ selector.css("*")
+
+ def test_invalid_json(self) -> None:
+ text = "<html/>"
+ selector = self.sscls(text, type="json")
+ self.assertEqual(selector.root, None)
+ self.assertEqual(selector.type, "json")
+
+ def test_text_and_root_warning(self) -> None:
+ with warnings.catch_warnings(record=True) as w:
+ Selector(text="a", root="b")
+ self.assertIn("both text and root", str(w[0].message))
+
+ def test_etree_root_invalid_type(self) -> None:
+ selector = Selector("<html></html>")
+ self.assertRaisesRegex(
+ ValueError,
+ "object as root",
+ Selector,
+ root=selector.root,
+ type="text",
+ )
+ self.assertRaisesRegex(
+ ValueError,
+ "object as root",
+ Selector,
+ root=selector.root,
+ type="json",
+ )
+
class ExsltTestCase(unittest.TestCase):
@@ -1336,3 +1409,57 @@ class ExsltTestCase(unittest.TestCase):
assert el.root.getparent() is not None
el.drop()
assert sel.get() == "<a><c/></a>"
+
+
+class SelectorBytesInput(Selector):
+ def __init__(
+ self,
+ text: Optional[str] = None,
+ type: Optional[str] = None,
+ body: bytes = b"",
+ encoding: str = "utf8",
+ namespaces: Optional[Mapping[str, str]] = None,
+ root: Optional[Any] = _NOT_SET,
+ base_url: Optional[str] = None,
+ _expr: Optional[str] = None,
+ huge_tree: bool = LXML_SUPPORTS_HUGE_TREE,
+ ) -> None:
+ if text:
+ body = bytes(text, encoding=encoding)
+ text = None
+ super().__init__(
+ text=text,
+ type=type,
+ body=body,
+ encoding=encoding,
+ namespaces=namespaces,
+ root=root,
+ base_url=base_url,
+ _expr=_expr,
+ huge_tree=huge_tree,
+ )
+
+
+class SelectorTestCaseBytes(SelectorTestCase):
+ sscls = SelectorBytesInput
+
+ def test_representation_slice(self) -> None:
+ pass
+
+ def test_representation_unicode_query(self) -> None:
+ pass
+
+ def test_weakref_slots(self) -> None:
+ pass
+
+ def test_check_text_argument_type(self) -> None:
+ self.assertRaisesRegex(
+ TypeError,
+ "body argument should be of type",
+ self.sscls,
+ body="<html/>",
+ )
+
+
+class ExsltTestCaseBytes(ExsltTestCase):
+ sscls = SelectorBytesInput
diff --git a/tests/test_selector_csstranslator.py b/tests/test_selector_csstranslator.py
index e2934d1..bd52f16 100644
--- a/tests/test_selector_csstranslator.py
+++ b/tests/test_selector_csstranslator.py
@@ -2,6 +2,7 @@
Selector tests for cssselect backend
"""
import unittest
+from typing import TYPE_CHECKING, Any, Callable, List, Type
import cssselect
import pytest
@@ -49,12 +50,39 @@ HTMLBODY = """
"""
+if TYPE_CHECKING:
+ # requires Python 3.8
+ from typing import Protocol
+
+ from parsel.csstranslator import TranslatorProtocol
+
+ class TranslatorTestProtocol(Protocol):
+ tr_cls: Type[TranslatorProtocol]
+ tr: TranslatorProtocol
+
+ def c2x(self, css: str, prefix: str = ...) -> str:
+ pass
+
+ def assertEqual(self, first: Any, second: Any, msg: Any = ...) -> None:
+ pass
+
+ def assertRaises(
+ self,
+ expected_exception: type[BaseException]
+ | tuple[type[BaseException], ...],
+ callable: Callable[..., object],
+ *args: Any,
+ **kwargs: Any,
+ ) -> None:
+ pass
+
+
class TranslatorTestMixin:
- def setUp(self):
+ def setUp(self: "TranslatorTestProtocol") -> None:
self.tr = self.tr_cls()
self.c2x = self.tr.css_to_xpath
- def test_attr_function(self):
+ def test_attr_function(self: "TranslatorTestProtocol") -> None:
cases = [
("::attr(name)", "descendant-or-self::*/@name"),
("a::attr(href)", "descendant-or-self::a/@href"),
@@ -67,7 +95,7 @@ class TranslatorTestMixin:
for css, xpath in cases:
self.assertEqual(self.c2x(css), xpath, css)
- def test_attr_function_exception(self):
+ def test_attr_function_exception(self: "TranslatorTestProtocol") -> None:
cases = [
("::attr(12)", ExpressionError),
("::attr(34test)", ExpressionError),
@@ -76,7 +104,7 @@ class TranslatorTestMixin:
for css, exc in cases:
self.assertRaises(exc, self.c2x, css)
- def test_text_pseudo_element(self):
+ def test_text_pseudo_element(self: "TranslatorTestProtocol") -> None:
cases = [
("::text", "descendant-or-self::text()"),
("p::text", "descendant-or-self::p/text()"),
@@ -105,7 +133,7 @@ class TranslatorTestMixin:
for css, xpath in cases:
self.assertEqual(self.c2x(css), xpath, css)
- def test_pseudo_function_exception(self):
+ def test_pseudo_function_exception(self: "TranslatorTestProtocol") -> None:
cases = [
("::attribute(12)", ExpressionError),
("::text()", ExpressionError),
@@ -114,14 +142,14 @@ class TranslatorTestMixin:
for css, exc in cases:
self.assertRaises(exc, self.c2x, css)
- def test_unknown_pseudo_element(self):
+ def test_unknown_pseudo_element(self: "TranslatorTestProtocol") -> None:
cases = [
("::text-node", ExpressionError),
]
for css, exc in cases:
self.assertRaises(exc, self.c2x, css)
- def test_unknown_pseudo_class(self):
+ def test_unknown_pseudo_class(self: "TranslatorTestProtocol") -> None:
cases = [
(":text", ExpressionError),
(":attribute(name)", ExpressionError),
@@ -139,7 +167,7 @@ class GenericTranslatorTest(TranslatorTestMixin, unittest.TestCase):
class UtilCss2XPathTest(unittest.TestCase):
- def test_css2xpath(self):
+ def test_css2xpath(self) -> None:
from parsel import css2xpath
expected_xpath = (
@@ -153,15 +181,15 @@ class CSSSelectorTest(unittest.TestCase):
sscls = Selector
- def setUp(self):
+ def setUp(self) -> None:
self.sel = self.sscls(text=HTMLBODY)
- def x(self, *a, **kw):
+ def x(self, *a: Any, **kw: Any) -> List[str]:
return [
v.strip() for v in self.sel.css(*a, **kw).extract() if v.strip()
]
- def test_selector_simple(self):
+ def test_selector_simple(self) -> None:
for x in self.sel.css("input"):
self.assertTrue(isinstance(x, self.sel.__class__), x)
self.assertEqual(
@@ -169,7 +197,7 @@ class CSSSelectorTest(unittest.TestCase):
[x.extract() for x in self.sel.css("input")],
)
- def test_text_pseudo_element(self):
+ def test_text_pseudo_element(self) -> None:
self.assertEqual(self.x("#p-b2"), ['<b id="p-b2">guy</b>'])
self.assertEqual(self.x("#p-b2::text"), ["guy"])
self.assertEqual(self.x("#p-b2 ::text"), ["guy"])
@@ -183,7 +211,7 @@ class CSSSelectorTest(unittest.TestCase):
self.x("p ::text"), ["lorem ipsum text", "hi", "there", "guy"]
)
- def test_attribute_function(self):
+ def test_attribute_function(self) -> None:
self.assertEqual(self.x("#p-b2::attr(id)"), ["p-b2"])
self.assertEqual(self.x(".cool-footer::attr(class)"), ["cool-footer"])
self.assertEqual(
@@ -193,7 +221,7 @@ class CSSSelectorTest(unittest.TestCase):
self.x('map[name="dummymap"] ::attr(shape)'), ["circle", "default"]
)
- def test_nested_selector(self):
+ def test_nested_selector(self) -> None:
self.assertEqual(
self.sel.css("p").css("b::text").extract(), ["hi", "guy"]
)
@@ -206,5 +234,10 @@ class CSSSelectorTest(unittest.TestCase):
Version(cssselect.__version__) < Version("1.2.0"),
reason="Support added in cssselect 1.2.0",
)
- def test_pseudoclass_has(self):
+ def test_pseudoclass_has(self) -> None:
self.assertEqual(self.x("p:has(b)::text"), ["lorem ipsum text"])
+
+
+class CSSSelectorTestBytes(CSSSelectorTest):
+ def setUp(self) -> None:
+ self.sel = self.sscls(body=bytes(HTMLBODY, encoding="utf8"))
diff --git a/tests/test_selector_jmespath.py b/tests/test_selector_jmespath.py
new file mode 100644
index 0000000..fa607fa
--- /dev/null
+++ b/tests/test_selector_jmespath.py
@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+from parsel import Selector
+from parsel.selector import _NOT_SET
+
+
+class JMESPathTestCase(unittest.TestCase):
+ def test_json_has_html(self) -> None:
+ """Sometimes the information is returned in a json wrapper"""
+ data = """
+ {
+ "content": [
+ {
+ "name": "A",
+ "value": "a"
+ },
+ {
+ "name": {
+ "age": 18
+ },
+ "value": "b"
+ },
+ {
+ "name": "C",
+ "value": "c"
+ },
+ {
+ "name": "<a>D</a>",
+ "value": "<div>d</div>"
+ }
+ ],
+ "html": "<div><a>a<br>b</a>c</div><div><a>d</a>e<b>f</b></div>"
+ }
+ """
+ sel = Selector(text=data)
+ self.assertEqual(
+ sel.jmespath("html").get(),
+ "<div><a>a<br>b</a>c</div><div><a>d</a>e<b>f</b></div>",
+ )
+ self.assertEqual(
+ sel.jmespath("html").xpath("//div/a/text()").getall(),
+ ["a", "b", "d"],
+ )
+ self.assertEqual(
+ sel.jmespath("html").css("div > b").getall(), ["<b>f</b>"]
+ )
+ self.assertEqual(
+ sel.jmespath("content").jmespath("name.age").get(), 18
+ )
+
+ def test_html_has_json(self) -> None:
+ html_text = """
+ <div>
+ <h1>Information</h1>
+ <content>
+ {
+ "user": [
+ {
+ "name": "A",
+ "age": 18
+ },
+ {
+ "name": "B",
+ "age": 32
+ },
+ {
+ "name": "C",
+ "age": 22
+ },
+ {
+ "name": "D",
+ "age": 25
+ }
+ ],
+ "total": 4,
+ "status": "ok"
+ }
+ </content>
+ </div>
+ """
+ sel = Selector(text=html_text)
+ self.assertEqual(
+ sel.xpath("//div/content/text()")
+ .jmespath("user[*].name")
+ .getall(),
+ ["A", "B", "C", "D"],
+ )
+ self.assertEqual(
+ sel.xpath("//div/content").jmespath("user[*].name").getall(),
+ ["A", "B", "C", "D"],
+ )
+ self.assertEqual(sel.xpath("//div/content").jmespath("total").get(), 4)
+
+ def test_jmestpath_with_re(self) -> None:
+ html_text = """
+ <div>
+ <h1>Information</h1>
+ <content>
+ {
+ "user": [
+ {
+ "name": "A",
+ "age": 18
+ },
+ {
+ "name": "B",
+ "age": 32
+ },
+ {
+ "name": "C",
+ "age": 22
+ },
+ {
+ "name": "D",
+ "age": 25
+ }
+ ],
+ "total": 4,
+ "status": "ok"
+ }
+ </content>
+ </div>
+ """
+ sel = Selector(text=html_text)
+ self.assertEqual(
+ sel.xpath("//div/content/text()")
+ .jmespath("user[*].name")
+ .re(r"(\w+)"),
+ ["A", "B", "C", "D"],
+ )
+ self.assertEqual(
+ sel.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"),
+ ["A", "B", "C", "D"],
+ )
+
+ with self.assertRaises(TypeError):
+ sel.xpath("//div/content").jmespath("user[*].age").re(r"(\d+)")
+
+ self.assertEqual(
+ sel.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), []
+ )
+
+ self.assertEqual(
+ sel.xpath("//div/content")
+ .jmespath("unavailable")
+ .re_first(r"(\d+)"),
+ None,
+ )
+
+ self.assertEqual(
+ sel.xpath("//div/content")
+ .jmespath("user[*].age.to_string(@)")
+ .re(r"(\d+)"),
+ ["18", "32", "22", "25"],
+ )
+
+ def test_json_types(self) -> None:
+ for text, root in (
+ ("{}", {}),
+ ('{"a": "b"}', {"a": "b"}),
+ ("[]", []),
+ ('["a"]', ["a"]),
+ ('""', ""),
+ ("0", 0),
+ ("1", 1),
+ ("true", True),
+ ("false", False),
+ ("null", None),
+ ):
+ selector = Selector(text=text, root=_NOT_SET)
+ self.assertEqual(selector.type, "json")
+ self.assertEqual(
+ selector._text, text # pylint: disable=protected-access
+ )
+ self.assertEqual(selector.root, root)
+
+ selector = Selector(text=None, root=root)
+ self.assertEqual(selector.type, "json")
+ self.assertEqual(
+ selector._text, None # pylint: disable=protected-access
+ )
+ self.assertEqual(selector.root, root)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index e2bca55..b82540e 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,3 +1,5 @@
+from typing import Pattern, List, Type, Union
+
from parsel.utils import shorten, extract_regex
from pytest import mark, raises
@@ -17,7 +19,7 @@ from pytest import mark, raises
(7, "foobar"),
),
)
-def test_shorten(width, expected):
+def test_shorten(width: int, expected: Union[str, Type[Exception]]) -> None:
if isinstance(expected, str):
assert shorten("foobar", width) == expected
else:
@@ -66,5 +68,10 @@ def test_shorten(width, expected):
],
),
)
-def test_extract_regex(regex, text, replace_entities, expected):
+def test_extract_regex(
+ regex: Union[str, Pattern[str]],
+ text: str,
+ replace_entities: bool,
+ expected: List[str],
+) -> None:
assert extract_regex(regex, text, replace_entities) == expected
diff --git a/tests/test_xml_attacks.py b/tests/test_xml_attacks.py
index 45b0243..e38a983 100644
--- a/tests/test_xml_attacks.py
+++ b/tests/test_xml_attacks.py
@@ -11,7 +11,7 @@ from parsel import Selector
MiB_1 = 1024**2
-def _load(attack):
+def _load(attack: str) -> str:
folder_path = path.dirname(__file__)
file_path = path.join(folder_path, "xml_attacks", f"{attack}.xml")
with open(file_path, "rb") as attack_file:
@@ -21,7 +21,7 @@ def _load(attack):
# List of known attacks:
# https://github.com/tiran/defusedxml#python-xml-libraries
class XMLAttackTestCase(TestCase):
- def test_billion_laughs(self):
+ def test_billion_laughs(self) -> None:
process = Process()
memory_usage_before = process.memory_info().rss
selector = Selector(text=_load("billion_laughs"))
diff --git a/tests/test_xpathfuncs.py b/tests/test_xpathfuncs.py
index 744472a..3adad0d 100644
--- a/tests/test_xpathfuncs.py
+++ b/tests/test_xpathfuncs.py
@@ -1,10 +1,12 @@
+from typing import Any
+import unittest
+
from parsel import Selector
from parsel.xpathfuncs import set_xpathfunc
-import unittest
class XPathFuncsTestCase(unittest.TestCase):
- def test_has_class_simple(self):
+ def test_has_class_simple(self) -> None:
body = """
<p class="foo bar-baz">First</p>
<p class="foo">Second</p>
@@ -35,7 +37,7 @@ class XPathFuncsTestCase(unittest.TestCase):
["First"],
)
- def test_has_class_error_no_args(self):
+ def test_has_class_error_no_args(self) -> None:
body = """
<p CLASS="foo">First</p>
"""
@@ -47,7 +49,7 @@ class XPathFuncsTestCase(unittest.TestCase):
"has-class()",
)
- def test_has_class_error_invalid_arg_type(self):
+ def test_has_class_error_invalid_arg_type(self) -> None:
body = """
<p CLASS="foo">First</p>
"""
@@ -59,7 +61,7 @@ class XPathFuncsTestCase(unittest.TestCase):
"has-class(.)",
)
- def test_has_class_error_invalid_unicode(self):
+ def test_has_class_error_invalid_unicode(self) -> None:
body = """
<p CLASS="foo">First</p>
"""
@@ -71,7 +73,7 @@ class XPathFuncsTestCase(unittest.TestCase):
'has-class("héllö")'.encode(),
)
- def test_has_class_unicode(self):
+ def test_has_class_unicode(self) -> None:
body = """
<p CLASS="fóó">First</p>
"""
@@ -81,7 +83,7 @@ class XPathFuncsTestCase(unittest.TestCase):
["First"],
)
- def test_has_class_uppercase(self):
+ def test_has_class_uppercase(self) -> None:
body = """
<p CLASS="foo">First</p>
"""
@@ -91,7 +93,7 @@ class XPathFuncsTestCase(unittest.TestCase):
["First"],
)
- def test_has_class_newline(self):
+ def test_has_class_newline(self) -> None:
body = """
<p CLASS="foo
bar">First</p>
@@ -102,7 +104,7 @@ class XPathFuncsTestCase(unittest.TestCase):
["First"],
)
- def test_has_class_tab(self):
+ def test_has_class_tab(self) -> None:
body = """
<p CLASS="foo\tbar">First</p>
"""
@@ -112,11 +114,11 @@ class XPathFuncsTestCase(unittest.TestCase):
["First"],
)
- def test_set_xpathfunc(self):
- def myfunc(ctx):
- myfunc.call_count += 1
+ def test_set_xpathfunc(self) -> None:
+ def myfunc(ctx: Any) -> None:
+ myfunc.call_count += 1 # type: ignore[attr-defined]
- myfunc.call_count = 0
+ myfunc.call_count = 0 # type: ignore[attr-defined]
body = """
<p CLASS="foo">First</p>
@@ -131,7 +133,7 @@ class XPathFuncsTestCase(unittest.TestCase):
set_xpathfunc("myfunc", myfunc)
sel.xpath("myfunc()")
- self.assertEqual(myfunc.call_count, 1)
+ self.assertEqual(myfunc.call_count, 1) # type: ignore[attr-defined]
set_xpathfunc("myfunc", None)
self.assertRaisesRegex(
diff --git a/tox.ini b/tox.ini
index 53b3d87..c983947 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = security,flake8,typing,pylint,black,docs,py37,py38,py39,py310,pypy3.9
+envlist = security,flake8,typing,pylint,black,docs,twinecheck,py37,py38,py39,py310,pypy3.9
[testenv]
usedevelop = True
@@ -23,12 +23,15 @@ commands =
[testenv:typing]
deps =
{[testenv]deps}
- types-lxml==2022.4.10
- types-psutil==5.9.5.4
- types-setuptools==65.5.0.1
- mypy==0.982
+ types-jmespath==1.0.2.6
+ types-lxml==2022.11.8
+ types-psutil==5.9.5.6
+ types-setuptools==67.2.0.1
+ py==1.11.0
+ mypy==1.0.0
+ typing-extensions==4.4.0
commands =
- mypy {posargs:parsel tests} --warn-unused-ignores
+ mypy {posargs:parsel tests} --strict
[testenv:pylint]
deps =
@@ -41,7 +44,7 @@ commands =
deps =
black==22.10.0
commands =
- black --line-length=79 --check {posargs:parsel tests setup.py}
+ black --line-length=79 {posargs:--check parsel tests setup.py}
[docs]
changedir = docs
@@ -55,4 +58,13 @@ deps = {[docs]deps}
commands =
sphinx-build -W -b html . {envtmpdir}/html
sphinx-build -b latex . {envtmpdir}/latex
- sphinx-build -W -b epub . {envtmpdir}/epub
+ sphinx-build -b epub . {envtmpdir}/epub
+
+[testenv:twinecheck]
+basepython = python3
+deps =
+ twine==4.0.2
+ build==0.10.0
+commands =
+ python -m build --sdist
+ twine check dist/*