Commit b986cca714f7a9ba9ea4b1a66a2c7a9664cf112b - fabric

Update upstream source from tag 'upstream/2.6.0' Update to upstream version '2.6.0' with Debian dir a40f0e131c4a5377a4535f95c51ff2c7c492eec9 Luca Boccassi 2 years ago

18 changed file(s) with 494 addition(s) and 264 deletion(s). Raw diff Collapse all Expand all

-2

.travis.yml less more

73	73	- pip uninstall -y fabric
74	74	- "PACKAGE_AS_FABRIC2=yes inv travis.test-packaging --package=fabric2 --sanity=\"fab2 --version\""
75	75	- inv sanity-test-from-v1
76		after_success:
	76	#after_success:
77	77	# Upload coverage data to codecov
78		- codecov
	78	#- codecov
79	79	notifications:
80	80	irc:
81	81	channels: "irc.freenode.org#fabric"

-1

LICENSE less more

0		Copyright (c) 2019 Jeff Forcier.
	0	Copyright (c) 2020 Jeff Forcier.
1	1	All rights reserved.
2	2
3	3	Redistribution and use in source and binary forms, with or without

-2

dev-requirements.txt less more

13	13	# Linting!
14	14	flake8==3.6.0
15	15	# Coverage!
16		coverage==3.7.1
17		codecov==1.6.3
	16	coverage==5.3.1
	17	codecov==2.1.11
18	18	# Documentation tools
19	19	sphinx>=1.4,<1.7
20	20	alabaster==0.7.12

-1

fabric/_version.py less more

0		__version_info__ = (2, 5, 0)
	0	__version_info__ = (2, 6, 0)
1	1	__version__ = ".".join(map(str, __version_info__))

-1

fabric/connection.py less more

299	299
300	300	Default: ``config.timeouts.connect``.
301	301
302		.. _connect_kwargs-arg:
303	302
304	303	:param dict connect_kwargs:
	304
	305	.. _connect_kwargs-arg:
	306
305	307	Keyword arguments handed verbatim to
306	308	`SSHClient.connect <paramiko.client.SSHClient.connect>` (when
307	309	`.open` is called).

+81

-35

fabric/group.py less more

17	17	concrete subclasses (such as `.SerialGroup` or `.ThreadingGroup`) or
18	18	you'll get ``NotImplementedError`` on most of the methods.
19	19
20		Most methods in this class mirror those of `.Connection`, taking the same
21		arguments; however their return values and exception-raising behavior
22		differs:
	20	Most methods in this class wrap those of `.Connection` and will accept the
	21	same arguments; however their return values and exception-raising behavior
	22	differ:
23	23
24	24	- Return values are dict-like objects (`.GroupResult`) mapping
25	25	`.Connection` objects to the return value for the respective connections:

98	98	group.extend(connections)
99	99	return group
100	100
	101	def _do(self, method, args, *kwargs):
	102	# TODO: rename this something public & commit to an API for user
	103	# subclasses
	104	raise NotImplementedError
	105
101	106	def run(self, args, *kwargs):
102	107	"""
103	108	Executes `.Connection.run` on all member `Connections <.Connection>`.

106	111
107	112	.. versionadded:: 2.0
108	113	"""
109		# TODO: probably best to suck it up & match actual run() sig?
110	114	# TODO: how to change method of execution across contents? subclass,
111	115	# kwargs, additional methods, inject an executor? Doing subclass for
112	116	# now, but not 100% sure it's the best route.
113	117	# TODO: also need way to deal with duplicate connections (see THOUGHTS)
114		# TODO: and errors - probably FailureSet? How to handle other,
115		# regular, non Failure, exceptions though? Still need an aggregate
116		# exception type either way, whether it is FailureSet or what...
117		# TODO: OTOH, users may well want to be able to operate on the hosts
118		# that did not fail (esp if failure % is low) so we really _do_ want
119		# something like a result object mixing success and failure, or maybe a
120		# golang style two-tuple of successes and failures?
121		# TODO: or keep going w/ a "return or except", but the object is
122		# largely similar (if not identical) in both situations, with the
123		# exception just being the signal that Shit Broke?
124		raise NotImplementedError
125
126		# TODO: how to handle sudo? Probably just an inner worker method that takes
127		# the method name to actually call (run, sudo, etc)?
	118	return self._do("run", args, *kwargs)
	119
	120	def sudo(self, args, *kwargs):
	121	"""
	122	Executes `.Connection.sudo` on all member `Connections <.Connection>`.
	123
	124	:returns: a `.GroupResult`.
	125
	126	.. versionadded:: 2.6
	127	"""
	128	# TODO: see run() TODOs
	129	return self._do("sudo", args, *kwargs)
128	130
129	131	# TODO: this all needs to mesh well with similar strategies applied to
130	132	# entire tasks - so that may still end up factored out into Executors or

132	134
133	135	# TODO: local? Invoke wants ability to do that on its own though, which
134	136	# would be distinct from Group. (May want to switch Group to use that,
135		# though, whatever it ends up being?)
	137	# though, whatever it ends up being? Eg many cases where you do want to do
	138	# some local thing either N times identically, or parameterized by remote
	139	# cxn values)
	140
	141	def put(self, args, *kwargs):
	142	"""
	143	Executes `.Connection.put` on all member `Connections <.Connection>`.
	144
	145	This is a straightforward application: aside from whatever the concrete
	146	group subclass does for concurrency or lack thereof, the effective
	147	result is like running a loop over the connections and calling their
	148	``put`` method.
	149
	150	:returns:
	151	a `.GroupResult` whose values are `.transfer.Result` instances.
	152
	153	.. versionadded:: 2.6
	154	"""
	155	return self._do("put", args, *kwargs)
136	156
137	157	def get(self, args, *kwargs):
138	158	"""
139	159	Executes `.Connection.get` on all member `Connections <.Connection>`.
140	160
141		:returns: a `.GroupResult`.
142
143		.. versionadded:: 2.0
144		"""
145		# TODO: probably best to suck it up & match actual get() sig?
146		# TODO: actually implement on subclasses
147		raise NotImplementedError
	161	.. note::
	162	This method changes some behaviors over e.g. directly calling
	163	`.Connection.get` on a ``for`` loop of connections; the biggest is
	164	that the implied default value for the ``local`` parameter is
	165	``"{host}/"``, which triggers use of local path parameterization
	166	based on each connection's target hostname.
	167
	168	Thus, unless you override ``local`` yourself, a copy of the
	169	downloaded file will be stored in (relative) directories named
	170	after each host in the group.
	171
	172	.. warning::
	173	Using file-like objects as the ``local`` argument is not currently
	174	supported, as it would be equivalent to supplying that same object
	175	to a series of individual ``get()`` calls.
	176
	177	:returns:
	178	a `.GroupResult` whose values are `.transfer.Result` instances.
	179
	180	.. versionadded:: 2.6
	181	"""
	182	# TODO: consider a backwards incompat change after we drop Py2 that
	183	# just makes a lot of these kwarg-only methods? then below could become
	184	# kwargs.setdefault() if desired.
	185	# TODO: do we care enough to handle explicitly given, yet falsey,
	186	# values? it's a lot more complexity for a corner case.
	187	if len(args) < 2 and "local" not in kwargs:
	188	kwargs["local"] = "{host}/"
	189	return self._do("get", args, *kwargs)
148	190
149	191	def close(self):
150	192	"""

169	211	.. versionadded:: 2.0
170	212	"""
171	213
172		def run(self, args, *kwargs):
	214	def _do(self, method, args, *kwargs):
173	215	results = GroupResult()
174	216	excepted = False
175	217	for cxn in self:
176	218	try:
177		results[cxn] = cxn.run(args, *kwargs)
	219	results[cxn] = getattr(cxn, method)(args, *kwargs)
178	220	except Exception as e:
179	221	results[cxn] = e
180	222	excepted = True

183	225	return results
184	226
185	227
186		def thread_worker(cxn, queue, args, kwargs):
187		result = cxn.run(args, *kwargs)
	228	def thread_worker(cxn, queue, method, args, kwargs):
	229	result = getattr(cxn, method)(args, *kwargs)
188	230	# TODO: namedtuple or attrs object?
189	231	queue.put((cxn, result))
190	232

196	238	.. versionadded:: 2.0
197	239	"""
198	240
199		def run(self, args, *kwargs):
	241	def _do(self, method, args, *kwargs):
200	242	results = GroupResult()
201	243	queue = Queue()
202	244	threads = []
203	245	for cxn in self:
204		my_kwargs = dict(cxn=cxn, queue=queue, args=args, kwargs=kwargs)
205	246	thread = ExceptionHandlingThread(
206		target=thread_worker, kwargs=my_kwargs
	247	target=thread_worker,
	248	kwargs=dict(
	249	cxn=cxn,
	250	queue=queue,
	251	method=method,
	252	args=args,
	253	kwargs=kwargs,
	254	),
207	255	)
208	256	threads.append(thread)
209	257	for thread in threads:
210	258	thread.start()
211	259	for thread in threads:
212	260	# TODO: configurable join timeout
213		# TODO: (in sudo's version) configurability around interactive
214		# prompting resulting in an exception instead, as in v1
215	261	thread.join()
216	262	# Get non-exception results from queue
217	263	while not queue.empty():

+10

-3

fabric/testing/base.py less more

375	375	# Set up mocks
376	376	self.os_patcher = patch("fabric.transfer.os")
377	377	self.client_patcher = patch("fabric.connection.SSHClient")
	378	self.path_patcher = patch("fabric.transfer.Path")
378	379	mock_os = self.os_patcher.start()
379	380	Client = self.client_patcher.start()
	381	self.path_patcher.start()
380	382	sftp = Client.return_value.open_sftp.return_value
381	383
382	384	# Handle common filepath massage actions; tests will assume these.
383	385	def fake_abspath(path):
384		return "/local/{}".format(path)
	386	# Run normpath to avoid tests not seeing abspath wrinkles (like
	387	# trailing slash chomping)
	388	return "/local/{}".format(os.path.normpath(path))
385	389
386	390	mock_os.path.abspath.side_effect = fake_abspath
387	391	sftp.getcwd.return_value = "/remote"

391	395	sftp.stat.return_value.st_mode = fake_mode
392	396	mock_os.stat.return_value.st_mode = fake_mode
393	397	# Not super clear to me why the 'wraps' functionality in mock isn't
394		# working for this :(
395		mock_os.path.basename.side_effect = os.path.basename
	398	# working for this :( reinstate a bunch of os(.path) so it still works
	399	mock_os.sep = os.sep
	400	for name in ("basename", "split", "join", "normpath"):
	401	getattr(mock_os.path, name).side_effect = getattr(os.path, name)
396	402	# Return the sftp and OS mocks for use by decorator use case.
397	403	return sftp, mock_os
398	404
399	405	def stop(self):
400	406	self.os_patcher.stop()
401	407	self.client_patcher.stop()
	408	self.path_patcher.stop()

-0

fabric/testing/fixtures.py less more

103	103	"""
104	104	mock = MockSFTP(autostart=False)
105	105	client, mock_os = mock.start()
	106	# Regular ol transfer to save some time
106	107	transfer = Transfer(Connection("host"))
107	108	yield transfer, client, mock_os
108	109	# TODO: old mock_sftp() lacked any 'stop'...why? feels bad man

+57

-14

fabric/transfer.py less more

4	4	import os
5	5	import posixpath
6	6	import stat
	7
	8	try:
	9	from pathlib import Path
	10	except ImportError:
	11	from pathlib2 import Path
7	12
8	13	from .util import debug # TODO: actual logging! LOL
9	14

39	44
40	45	def get(self, remote, local=None, preserve_mode=True):
41	46	"""
42		Download a file from the current connection to the local filesystem.
	47	Copy a file from wrapped connection's host to the local filesystem.
43	48
44	49	:param str remote:
45	50	Remote file to download.

59	64
60	65	If None or another 'falsey'/empty value is given (the default),
61	66	the remote file is downloaded to the current working directory (as
62		seen by `os.getcwd`) using its remote filename.
	67	seen by `os.getcwd`) using its remote filename. (This is equivalent
	68	to giving ``"{basename}"``; see the below subsection on
	69	interpolation.)
63	70
64	71	If a string is given, it should be a path to a local directory
65	72	or file and is subject to similar behavior as that seen by common

70	77	'/tmp/')`` would result in creation or overwriting of
71	78	``/tmp/file.txt``).
72	79
73		.. note::
74		When dealing with nonexistent file paths, normal Python file
75		handling concerns come into play - for example, a ``local``
76		path containing non-leaf directories which do not exist, will
77		typically result in an `OSError`.
	80	This path will be interpolated with some useful parameters,
	81	using `str.format`:
	82
	83	- The `.Connection` object's ``host``, ``user`` and ``port``
	84	attributes.
	85	- The ``basename`` and ``dirname`` of the ``remote`` path, as
	86	derived by `os.path` (specifically, its ``posixpath`` flavor, so
	87	that the resulting values are useful on remote POSIX-compatible
	88	SFTP servers even if the local client is Windows).
	89	- Thus, for example, ``"/some/path/{user}@{host}/{basename}"`` will
	90	yield different local paths depending on the properties of both
	91	the connection and the remote path.
	92
	93	.. note::
	94	If nonexistent directories are present in this path (including
	95	the final path component, if it ends in `os.sep`) they will be
	96	created automatically using `os.makedirs`.
78	97
79	98	If a file-like object is given, the contents of the remote file
80	99	are simply written into it.

86	105	:returns: A `.Result` object.
87	106
88	107	.. versionadded:: 2.0
	108	.. versionchanged:: 2.6
	109	Added ``local`` path interpolation of connection & remote file
	110	attributes.
	111	.. versionchanged:: 2.6
	112	Create missing ``local`` directories automatically.
89	113	"""
90	114	# TODO: how does this API change if we want to implement
91	115	# remote-to-remote file transfer? (Is that even realistic?)
92		# TODO: handle v1's string interpolation bits, especially the default
93		# one, or at least think about how that would work re: split between
94		# single and multiple server targets.
95	116	# TODO: callback support
96	117	# TODO: how best to allow changing the behavior/semantics of
97	118	# remote/local (e.g. users might want 'safer' behavior that complains

106	127	self.sftp.getcwd() or self.sftp.normalize("."), remote
107	128	)
108	129
109		# Massage local path:
110		# - handle file-ness
111		# - if path, fill with remote name if empty, & make absolute
	130	# Massage local path
112	131	orig_local = local
113	132	is_file_like = hasattr(local, "write") and callable(local.write)
	133	remote_filename = posixpath.basename(remote)
114	134	if not local:
115		local = posixpath.basename(remote)
	135	local = remote_filename
	136	# Path-driven local downloads need interpolation, abspath'ing &
	137	# directory creation
116	138	if not is_file_like:
	139	local = local.format(
	140	host=self.connection.host,
	141	user=self.connection.user,
	142	port=self.connection.port,
	143	dirname=posixpath.dirname(remote),
	144	basename=remote_filename,
	145	)
	146	# Must treat dir vs file paths differently, lest we erroneously
	147	# mkdir what was intended as a filename, and so that non-empty
	148	# dir-like paths still get remote filename tacked on.
	149	if local.endswith(os.sep):
	150	dir_path = local
	151	local = os.path.join(local, remote_filename)
	152	else:
	153	dir_path, _ = os.path.split(local)
117	154	local = os.path.abspath(local)
	155	Path(dir_path).mkdir(parents=True, exist_ok=True)
	156	# TODO: reimplement mkdir (or otherwise write a testing function)
	157	# allowing us to track what was created so we can revert if
	158	# transfer fails.
	159	# TODO: Alternately, transfer to temp location and then move, but
	160	# that's basically inverse of v1's sudo-put which gets messy
118	161
119	162	# Run Paramiko-level .get() (side-effects only. womp.)
120	163	# TODO: push some of the path handling into Paramiko; it should be

-1

setup.py less more

66	66	author="Jeff Forcier",
67	67	author_email="jeff@bitprophet.org",
68	68	url="http://fabfile.org",
69		install_requires=["invoke>=1.3,<2.0", "paramiko>=2.4"],
	69	install_requires=["invoke>=1.3,<2.0", "paramiko>=2.4", "pathlib2"],
70	70	extras_require={
71	71	"testing": testing_deps,
72	72	"pytest": testing_deps + pytest_deps,

-2

sites/docs/getting-started.rst less more

13	13	on top; user code will most often import from the ``fabric`` package, but
14	14	you'll sometimes import directly from ``invoke`` or ``paramiko`` too:
15	15
16		- `Invoke <https://pyinvoke.org>`_ implements CLI parsing, task organization,
	16	- `Invoke <https://www.pyinvoke.org>`_ implements CLI parsing, task organization,
17	17	and shell command execution (a generic framework plus specific implementation
18	18	for local commands.)
19	19

23	23	- Fabric users will frequently import Invoke objects, in cases where Fabric
24	24	itself has no need to subclass or otherwise modify what Invoke provides.
25	25
26		- `Paramiko <https://paramiko.org>`_ implements low/mid level SSH
	26	- `Paramiko <https://www.paramiko.org>`_ implements low/mid level SSH
27	27	functionality - SSH and SFTP sessions, key management, etc.
28	28
29	29	- Fabric mostly uses this under the hood; users will only rarely import

+25

-0

sites/www/changelog.rst less more

4	4	.. note::
5	5	Looking for the Fabric 1.x changelog? See :doc:`/changelog-v1`.
6	6
	7	- :release:`2.6.0 <2021-01-18>`
	8	- :bug:`- major` Fix a handful of issues in the handling and
	9	mocking of SFTP local paths and ``os.path`` members within
	10	:ref:`fabric.testing <testing-subpackage>`; this should remove some
	11	occasional "useless Mocks" as well as hewing closer to the real behavior of
	12	things like ``os.path.abspath`` re: path normalization.
	13	- :feature:`-` When the ``local`` path argument to
	14	`Transfer.get <fabric.transfer.Transfer.get>` contains nonexistent
	15	directories, they are now created instead of raising an error.
	16
	17	.. warning::
	18	This change introduces a new runtime dependency: ``pathlib2``.
	19
	20	- :feature:`1868` Ported a feature from v1: interpolating the local path
	21	argument in `Transfer.get <fabric.transfer.Transfer.get>` with connection
	22	and remote filepath attributes.
	23
	24	For example, ``cxn.get(remote="/var/log/foo.log", local="{host}/")`` is now
	25	feasible for storing a file in per-host-named directories or files, and in
	26	fact `Group.get <fabric.group.Group.get>` does this by default.
	27	- :feature:`1810` Add `put <fabric.group.Group.put>`/`get
	28	<fabric.group.Group.get>` support to `~fabric.group.Group`.
	29	- :feature:`1999` Add `sudo <fabric.group.Group.sudo>` support to
	30	`~fabric.group.Group`. Thanks to Bonnie Hardin for the report and to Winston
	31	Nolan for an early patchset.
7	32	- :release:`2.5.0 <2019-08-06>`
8	33	- :support:`-` Update minimum Invoke version requirement to ``>=1.3``.
9	34	- :feature:`1985` Add support for explicitly closing remote subprocess' stdin

-4

sites/www/upgrading.rst less more

841	841
842	842	* - ``shell`` / ``env.use_shell`` designating whether or not to wrap
843	843	commands within an explicit call to e.g. ``/bin/sh -c "real command"``
844		- `Pending <https://github.com/pyinvoke/invoke/issues/344>`__/Removed
	844	- `Pending <https://github.com/pyinvoke/invoke/issues/459>`__/Removed
845	845	- See the note above under ``run`` for details on shell wrapping
846	846	as a general strategy; unfortunately for ``sudo``, some sort of manual
847	847	wrapping is still necessary for nontrivial commands (i.e. anything

1126	1126	own, so it's gone.
1127	1127	* - Naming downloaded files after some aspect of the remote destination, to
1128	1128	avoid overwriting during multi-server actions
1129		- `Pending <https://github.com/fabric/fabric/issues/1868>`__
1130		- This falls under the `~fabric.group.Group` family, which still needs
1131		some work in this regard.
	1129	- Ported
	1130	- Added back (to `fabric.transfer.Transfer.get`) in Fabric 2.6.
1132	1131
1133	1132
1134	1133	.. _upgrading-configuration:

-23

tests/_util.py less more

5	5	from invoke.vendor.lexicon import Lexicon
6	6	from pytest_relaxed import trap
7	7
8		from fabric import Connection as Connection_, Config as Config_
9	8	from fabric.main import make_program
10		from paramiko import SSHConfig
11	9
12	10
13	11	support = os.path.join(os.path.abspath(os.path.dirname(__file__)), "_support")

50	48	assert False, err.format(test)
51	49
52	50
53		# Locally override Connection, Config with versions that supply a dummy
54		# SSHConfig and thus don't load any test-running user's own ssh_config files.
55		# TODO: find a cleaner way to do this, though I don't really see any that isn't
56		# adding a ton of fixtures everywhere (and thus, opening up to forgetting it
57		# for new tests...)
58		class Config(Config_):
59		def __init__(self, args, *kwargs):
60		wat = "You're giving ssh_config explicitly, please use Config_!"
61		assert "ssh_config" not in kwargs, wat
62		# Give ssh_config explicitly -> shorter way of turning off loading
63		kwargs["ssh_config"] = SSHConfig()
64		super(Config, self).__init__(args, *kwargs)
65
66
67		class Connection(Connection_):
68		def __init__(self, args, *kwargs):
69		# Make sure we're using our tweaked Config if none was given.
70		kwargs.setdefault("config", Config())
71		super(Connection, self).__init__(args, *kwargs)
72
73
74	51	def faux_v1_env():
75	52	# Close enough to v1 _AttributeDict...
76	53	# Contains a copy of enough of v1's defaults to prevent us having to do a

+29

-0

tests/conftest.py less more

0	0	# flake8: noqa
1	1	from fabric.testing.fixtures import client, remote, sftp, sftp_objs, transfer
	2
	3	from os.path import isfile, expanduser
	4
	5	from pytest import fixture
	6
	7	from mock import patch
	8
	9
	10	# TODO: does this want to end up in the public fixtures module too?
	11	@fixture(autouse=True)
	12	def no_user_ssh_config():
	13	"""
	14	Cowardly refuse to ever load what looks like user SSH config paths.
	15
	16	Prevents the invoking user's real config from gumming up test results or
	17	inflating test runtime (eg if it sets canonicalization on, which will incur
	18	DNS lookups for nearly all of this suite's bogus names).
	19	"""
	20	# An ugly, but effective, hack. I am not proud. I also don't see anything
	21	# that's >= as bulletproof and less ugly?
	22	# TODO: ideally this should expand to cover system config paths too, but
	23	# that's even less likely to be an issue.
	24	def no_config_for_you(path):
	25	if path == expanduser("~/.ssh/config"):
	26	return False
	27	return isfile(path)
	28
	29	with patch("fabric.config.os.path.isfile", no_config_for_you):
	30	yield

-6

tests/connection.py less more

19	19	from invoke.config import Config as InvokeConfig
20	20	from invoke.exceptions import ThreadException
21	21
22		from fabric import Config as Config_
	22	from fabric import Config, Connection
23	23	from fabric.exceptions import InvalidV1Env
24	24	from fabric.util import get_local_user
25	25
26		from _util import support, Connection, Config, faux_v1_env
	26	from _util import support, faux_v1_env
27	27
28	28
29	29	# Remote is woven in as a config default, so must be patched there

264	264	runtime_path = join(support, "ssh_config", confname)
265	265	if overrides is None:
266	266	overrides = {}
267		return Config_(
	267	return Config(
268	268	runtime_ssh_path=runtime_path, overrides=overrides
269	269	)
270	270

273	273	return Connection("runtime", config=config)
274	274
275	275	def effectively_blank_when_no_loaded_config(self):
276		c = Config_(ssh_config=SSHConfig())
	276	c = Config(ssh_config=SSHConfig())
277	277	cxn = Connection("host", config=c)
278	278	# NOTE: paramiko always injects this even if you look up a host
279	279	# that has no rules, even wildcard ones.

305	305	path = join(
306	306	support, "ssh_config", "overridden_hostname.conf"
307	307	)
308		config = Config_(runtime_ssh_path=path)
	308	config = Config(runtime_ssh_path=path)
309	309	cxn = Connection("aliasname", config=config)
310	310	assert cxn.host == "realname"
311	311	assert cxn.original_host == "aliasname"

858	858	config_kwargs["overrides"] = {
859	859	"connect_kwargs": {"key_filename": ["configured.key"]}
860	860	}
861		conf = Config_(**config_kwargs)
	861	conf = Config(**config_kwargs)
862	862	connect_kwargs = {}
863	863	if kwarg:
864	864	# Stitch in connect_kwargs value

+222

-168

tests/group.py less more

0	0	from mock import Mock, patch, call
1		from pytest_relaxed import raises
	1	from pytest import mark, raises
2	2
3	3	from fabric import Connection, Group, SerialGroup, ThreadingGroup, GroupResult
4	4	from fabric.group import thread_worker
5	5	from fabric.exceptions import GroupException
	6
	7
	8	RUNNER_METHODS = ("run", "sudo")
	9	TRANSFER_METHODS = ("put", "get")
	10	ALL_METHODS = RUNNER_METHODS + TRANSFER_METHODS
	11	runner_args = ("command",)
	12	runner_kwargs = dict(hide=True, warn=True)
	13	transfer_args = tuple()
	14	transfer_kwargs = dict(local="yokel", remote="goat")
	15	ARGS_BY_METHOD = dict(
	16	run=runner_args, sudo=runner_args, put=transfer_args, get=transfer_args
	17	)
	18	KWARGS_BY_METHOD = dict(
	19	run=runner_kwargs,
	20	sudo=runner_kwargs,
	21	put=transfer_kwargs,
	22	get=transfer_kwargs,
	23	)
6	24
7	25
8	26	class Group_:

40	58	for c in g:
41	59	assert isinstance(c, Connection)
42	60
43		class run:
44		@raises(NotImplementedError)
45		def not_implemented_in_base_class(self):
46		Group().run()
	61	@mark.parametrize("method", ALL_METHODS)
	62	def abstract_methods_not_implemented(self, method):
	63	group = Group()
	64	with raises(NotImplementedError):
	65	getattr(group, method)()
47	66
48	67	class close_and_contextmanager_behavior:
49	68	def close_closes_all_member_connections(self):

61	80	for c in cxns:
62	81	c.close.assert_called_once_with()
63	82
64
65		def _make_serial_tester(cxns, index, args, kwargs):
	83	class get:
	84	class local_defaults_to_host_interpolated_path:
	85	def when_no_arg_or_kwarg_given(self):
	86	g = Group("host1", "host2")
	87	g._do = Mock()
	88	g.get(remote="whatever")
	89	g._do.assert_called_with(
	90	"get", remote="whatever", local="{host}/"
	91	)
	92
	93	def not_when_arg_given(self):
	94	g = Group("host1", "host2")
	95	g._do = Mock()
	96	g.get("whatever", "lol")
	97	# No local kwarg passed.
	98	g._do.assert_called_with("get", "whatever", "lol")
	99
	100	def not_when_kwarg_given(self):
	101	g = Group("host1", "host2")
	102	g._do = Mock()
	103	g.get(remote="whatever", local="lol")
	104	# Doesn't stomp given local arg
	105	g._do.assert_called_with("get", remote="whatever", local="lol")
	106
	107
	108	def _make_serial_tester(method, cxns, index, args, kwargs):
66	109	args = args[:]
67	110	kwargs = kwargs.copy()
68	111

71	114	predecessors = cxns[:car]
72	115	successors = cxns[cdr:]
73	116	for predecessor in predecessors:
74		predecessor.run.assert_called_with(args, *kwargs)
	117	getattr(predecessor, method).assert_called_with(args, *kwargs)
75	118	for successor in successors:
76		assert not successor.run.called
	119	assert not getattr(successor, method).called
77	120
78	121	return tester
79	122
80	123
81	124	class SerialGroup_:
82		class run:
83		def executes_arguments_on_contents_run_serially(self):
84		"executes arguments on contents' run() serially"
85		cxns = [Connection(x) for x in ("host1", "host2", "host3")]
86		args = ("command",)
87		kwargs = {"hide": True, "warn": True}
88		for index, cxn in enumerate(cxns):
89		side_effect = _make_serial_tester(cxns, index, args, kwargs)
90		cxn.run = Mock(side_effect=side_effect)
91		g = SerialGroup.from_connections(cxns)
92		g.run(args, *kwargs)
93		# Sanity check, e.g. in case none of them were actually run
94		for cxn in cxns:
95		cxn.run.assert_called_with(args, *kwargs)
96
97		def errors_in_execution_capture_and_continue_til_end(self):
98		cxns = [Mock(name=x) for x in ("host1", "host2", "host3")]
99
100		class OhNoz(Exception):
101		pass
102
103		onoz = OhNoz()
104		cxns[1].run.side_effect = onoz
105		g = SerialGroup.from_connections(cxns)
106		try:
107		g.run("whatever", hide=True)
108		except GroupException as e:
109		result = e.result
110		else:
111		assert False, "Did not raise GroupException!"
112		succeeded = {
113		cxns[0]: cxns[0].run.return_value,
114		cxns[2]: cxns[2].run.return_value,
115		}
116		failed = {cxns[1]: onoz}
117		expected = succeeded.copy()
118		expected.update(failed)
119		assert result == expected
120		assert result.succeeded == succeeded
121		assert result.failed == failed
122
123		def returns_results_mapping(self):
124		cxns = [Mock(name=x) for x in ("host1", "host2", "host3")]
125		g = SerialGroup.from_connections(cxns)
126		result = g.run("whatever", hide=True)
127		assert isinstance(result, GroupResult)
128		expected = {x: x.run.return_value for x in cxns}
129		assert result == expected
130		assert result.succeeded == expected
131		assert result.failed == {}
	125	@mark.parametrize("method", ALL_METHODS)
	126	def executes_arguments_on_contents_run_serially(self, method):
	127	"executes arguments on contents' run() serially"
	128	cxns = [Connection(x) for x in ("host1", "host2", "host3")]
	129	args = ARGS_BY_METHOD[method]
	130	kwargs = KWARGS_BY_METHOD[method]
	131	for index, cxn in enumerate(cxns):
	132	side_effect = _make_serial_tester(
	133	method, cxns, index, args, kwargs
	134	)
	135	setattr(cxn, method, Mock(side_effect=side_effect))
	136	g = SerialGroup.from_connections(cxns)
	137	getattr(g, method)(args, *kwargs)
	138	# Sanity check, e.g. in case none of them were actually run
	139	for cxn in cxns:
	140	getattr(cxn, method).assert_called_with(args, *kwargs)
	141
	142	@mark.parametrize("method", ALL_METHODS)
	143	def errors_in_execution_capture_and_continue_til_end(self, method):
	144	cxns = [Mock(name=x) for x in ("host1", "host2", "host3")]
	145
	146	class OhNoz(Exception):
	147	pass
	148
	149	onoz = OhNoz()
	150	getattr(cxns[1], method).side_effect = onoz
	151	g = SerialGroup.from_connections(cxns)
	152	try:
	153	getattr(g, method)("whatever", hide=True)
	154	except GroupException as e:
	155	result = e.result
	156	else:
	157	assert False, "Did not raise GroupException!"
	158	succeeded = {
	159	cxns[0]: getattr(cxns[0], method).return_value,
	160	cxns[2]: getattr(cxns[2], method).return_value,
	161	}
	162	failed = {cxns[1]: onoz}
	163	expected = succeeded.copy()
	164	expected.update(failed)
	165	assert result == expected
	166	assert result.succeeded == succeeded
	167	assert result.failed == failed
	168
	169	@mark.parametrize("method", ALL_METHODS)
	170	def returns_results_mapping(self, method):
	171	cxns = [Mock(name=x) for x in ("host1", "host2", "host3")]
	172	g = SerialGroup.from_connections(cxns)
	173	result = getattr(g, method)("whatever", hide=True)
	174	assert isinstance(result, GroupResult)
	175	expected = {x: getattr(x, method).return_value for x in cxns}
	176	assert result == expected
	177	assert result.succeeded == expected
	178	assert result.failed == {}
132	179
133	180
134	181	class ThreadingGroup_:
135	182	def setup(self):
136	183	self.cxns = [Connection(x) for x in ("host1", "host2", "host3")]
137		self.args = ("command",)
138		self.kwargs = {"hide": True, "warn": True}
139
140		class run:
141		@patch("fabric.group.Queue")
142		@patch("fabric.group.ExceptionHandlingThread")
143		def executes_arguments_on_contents_run_via_threading(
144		self, Thread, Queue
	184
	185	@mark.parametrize("method", ALL_METHODS)
	186	@patch("fabric.group.Queue")
	187	@patch("fabric.group.ExceptionHandlingThread")
	188	def executes_arguments_on_contents_run_via_threading(
	189	self, Thread, Queue, method
	190	):
	191	queue = Queue.return_value
	192	g = ThreadingGroup.from_connections(self.cxns)
	193	# Make sure .exception() doesn't yield truthy Mocks. Otherwise we
	194	# end up with 'exceptions' that cause errors due to all being the
	195	# same.
	196	Thread.return_value.exception.return_value = None
	197	args = ARGS_BY_METHOD[method]
	198	kwargs = KWARGS_BY_METHOD[method]
	199	getattr(g, method)(args, *kwargs)
	200	# Testing that threads were used the way we expect is mediocre but
	201	# I honestly can't think of another good way to assert "threading
	202	# was used & concurrency occurred"...
	203	instantiations = [
	204	call(
	205	target=thread_worker,
	206	kwargs=dict(
	207	cxn=cxn,
	208	queue=queue,
	209	method=method,
	210	args=args,
	211	kwargs=kwargs,
	212	),
	213	)
	214	for cxn in self.cxns
	215	]
	216	Thread.assert_has_calls(instantiations, any_order=True)
	217	# These ought to work as by default a Mock.return_value is a
	218	# singleton mock object
	219	expected = len(self.cxns)
	220	for name, got in (
	221	("start", Thread.return_value.start.call_count),
	222	("join", Thread.return_value.join.call_count),
145	223	):
146		queue = Queue.return_value
147		g = ThreadingGroup.from_connections(self.cxns)
148		# Make sure .exception() doesn't yield truthy Mocks. Otherwise we
149		# end up with 'exceptions' that cause errors due to all being the
150		# same.
151		Thread.return_value.exception.return_value = None
152		g.run(self.args, *self.kwargs)
153		# Testing that threads were used the way we expect is mediocre but
154		# I honestly can't think of another good way to assert "threading
155		# was used & concurrency occurred"...
156		instantiations = [
157		call(
158		target=thread_worker,
159		kwargs=dict(
160		cxn=cxn,
161		queue=queue,
162		args=self.args,
163		kwargs=self.kwargs,
164		),
165		)
166		for cxn in self.cxns
167		]
168		Thread.assert_has_calls(instantiations, any_order=True)
169		# These ought to work as by default a Mock.return_value is a
170		# singleton mock object
171		expected = len(self.cxns)
172		for name, got in (
173		("start", Thread.return_value.start.call_count),
174		("join", Thread.return_value.join.call_count),
175		):
176		err = (
177		"Expected {} calls to ExceptionHandlingThread.{}, got {}"
178		) # noqa
179		err = err.format(expected, name, got)
180		assert expected, got == err
181
182		@patch("fabric.group.Queue")
183		def queue_used_to_return_results(self, Queue):
184		# Regular, explicit, mocks for Connections
185		cxns = [Mock(host=x) for x in ("host1", "host2", "host3")]
186		# Set up Queue with enough behavior to work / assert
187		queue = Queue.return_value
188		# Ending w/ a True will terminate a while-not-empty loop
189		queue.empty.side_effect = (False, False, False, True)
190		fakes = [(x, x.run.return_value) for x in cxns]
191		queue.get.side_effect = fakes[:]
192		# Execute & inspect results
193		g = ThreadingGroup.from_connections(cxns)
194		results = g.run(self.args, *self.kwargs)
195		expected = {x: x.run.return_value for x in cxns}
196		assert results == expected
197		# Make sure queue was used as expected within worker &
198		# ThreadingGroup.run()
199		puts = [call(x) for x in fakes]
200		queue.put.assert_has_calls(puts, any_order=True)
201		assert queue.empty.called
202		gets = [call(block=False) for _ in cxns]
203		queue.get.assert_has_calls(gets)
204
205		def bubbles_up_errors_within_threads(self):
206		# TODO: I feel like this is the first spot where a raw
207		# ThreadException might need tweaks, at least presentation-wise,
208		# since we're no longer dealing with truly background threads (IO
209		# workers and tunnels), but "middle-ground" threads the user is
210		# kind of expecting (and which they might expect to encounter
211		# failures).
212		cxns = [Mock(host=x) for x in ("host1", "host2", "host3")]
213
214		class OhNoz(Exception):
215		pass
216
217		onoz = OhNoz()
218		cxns[1].run.side_effect = onoz
219		g = ThreadingGroup.from_connections(cxns)
220		try:
221		g.run(self.args, *self.kwargs)
222		except GroupException as e:
223		result = e.result
224		else:
225		assert False, "Did not raise GroupException!"
226		succeeded = {
227		cxns[0]: cxns[0].run.return_value,
228		cxns[2]: cxns[2].run.return_value,
229		}
230		failed = {cxns[1]: onoz}
231		expected = succeeded.copy()
232		expected.update(failed)
233		assert result == expected
234		assert result.succeeded == succeeded
235		assert result.failed == failed
236
237		def returns_results_mapping(self):
238		# TODO: update if/when we implement ResultSet
239		cxns = [Mock(name=x) for x in ("host1", "host2", "host3")]
240		g = ThreadingGroup.from_connections(cxns)
241		result = g.run("whatever", hide=True)
242		assert isinstance(result, GroupResult)
243		expected = {x: x.run.return_value for x in cxns}
244		assert result == expected
245		assert result.succeeded == expected
246		assert result.failed == {}
	224	err = (
	225	"Expected {} calls to ExceptionHandlingThread.{}, got {}"
	226	) # noqa
	227	err = err.format(expected, name, got)
	228	assert expected, got == err
	229
	230	@mark.parametrize("method", ALL_METHODS)
	231	@patch("fabric.group.Queue")
	232	def queue_used_to_return_results(self, Queue, method):
	233	# Regular, explicit, mocks for Connections
	234	cxns = [Mock(host=x) for x in ("host1", "host2", "host3")]
	235	# Set up Queue with enough behavior to work / assert
	236	queue = Queue.return_value
	237	# Ending w/ a True will terminate a while-not-empty loop
	238	queue.empty.side_effect = (False, False, False, True)
	239	fakes = [(x, getattr(x, method).return_value) for x in cxns]
	240	queue.get.side_effect = fakes[:]
	241	# Execute & inspect results
	242	g = ThreadingGroup.from_connections(cxns)
	243	results = getattr(g, method)(
	244	ARGS_BY_METHOD[method], *KWARGS_BY_METHOD[method]
	245	)
	246	expected = {x: getattr(x, method).return_value for x in cxns}
	247	assert results == expected
	248	# Make sure queue was used as expected within worker &
	249	# ThreadingGroup.run()
	250	puts = [call(x) for x in fakes]
	251	queue.put.assert_has_calls(puts, any_order=True)
	252	assert queue.empty.called
	253	gets = [call(block=False) for _ in cxns]
	254	queue.get.assert_has_calls(gets)
	255
	256	@mark.parametrize("method", ALL_METHODS)
	257	def bubbles_up_errors_within_threads(self, method):
	258	# TODO: I feel like this is the first spot where a raw
	259	# ThreadException might need tweaks, at least presentation-wise,
	260	# since we're no longer dealing with truly background threads (IO
	261	# workers and tunnels), but "middle-ground" threads the user is
	262	# kind of expecting (and which they might expect to encounter
	263	# failures).
	264	cxns = [Mock(host=x) for x in ("host1", "host2", "host3")]
	265
	266	class OhNoz(Exception):
	267	pass
	268
	269	onoz = OhNoz()
	270	getattr(cxns[1], method).side_effect = onoz
	271	g = ThreadingGroup.from_connections(cxns)
	272	try:
	273	getattr(g, method)(
	274	ARGS_BY_METHOD[method], *KWARGS_BY_METHOD[method]
	275	)
	276	except GroupException as e:
	277	result = e.result
	278	else:
	279	assert False, "Did not raise GroupException!"
	280	succeeded = {
	281	cxns[0]: getattr(cxns[0], method).return_value,
	282	cxns[2]: getattr(cxns[2], method).return_value,
	283	}
	284	failed = {cxns[1]: onoz}
	285	expected = succeeded.copy()
	286	expected.update(failed)
	287	assert result == expected
	288	assert result.succeeded == succeeded
	289	assert result.failed == failed
	290
	291	@mark.parametrize("method", ALL_METHODS)
	292	def returns_results_mapping(self, method):
	293	cxns = [Mock(name=x) for x in ("host1", "host2", "host3")]
	294	g = ThreadingGroup.from_connections(cxns)
	295	result = getattr(g, method)("whatever", hide=True)
	296	assert isinstance(result, GroupResult)
	297	expected = {x: getattr(x, method).return_value for x in cxns}
	298	assert result == expected
	299	assert result.succeeded == expected
	300	assert result.failed == {}

+48

-1

tests/transfer.py less more

2	2	except ImportError:
3	3	from six import StringIO
4	4
5		from mock import Mock, call
	5	from mock import Mock, call, patch
6	6	from pytest_relaxed import raises
7	7	from pytest import skip # noqa
8	8	from paramiko import SFTPAttributes

91	91	def remote_arg_cannot_be_empty_string(self, transfer):
92	92	transfer.get("")
93	93
	94	class local_arg_interpolation:
	95	def connection_params(self, transfer):
	96	result = transfer.get("somefile", "{user}@{host}-{port}")
	97	expected = "/local/{}@host-22".format(transfer.connection.user)
	98	assert result.local == expected
	99
	100	def connection_params_as_dir(self, transfer):
	101	result = transfer.get("somefile", "{host}/")
	102	assert result.local == "/local/host/somefile"
	103
	104	def remote_path_posixpath_bits(self, transfer):
	105	result = transfer.get(
	106	"parent/mid/leaf", "foo/{dirname}/bar/{basename}"
	107	)
	108	# Recall that test harness sets remote apparent cwd as
	109	# /remote/, thus dirname is /remote/parent/mid
	110	assert result.local == "/local/foo/remote/parent/mid/bar/leaf"
	111
94	112	class file_like_local_paths:
95	113	"file-like local paths"
96	114

131	149	client.stat.return_value = self.attrs
132	150	transfer.get("file", local="meh", preserve_mode=False)
133	151	assert not mock_os.chmod.called
	152
	153	class local_directory_creation:
	154	@patch("fabric.transfer.Path")
	155	def without_trailing_slash_means_leaf_file(self, Path, sftp_objs):
	156	transfer, client = sftp_objs
	157	transfer.get(remote="file", local="top/middle/leaf")
	158	client.get.assert_called_with(
	159	localpath="/local/top/middle/leaf",
	160	remotepath="/remote/file",
	161	)
	162	Path.assert_called_with("top/middle")
	163	Path.return_value.mkdir.assert_called_with(
	164	parents=True, exist_ok=True
	165	)
	166
	167	@patch("fabric.transfer.Path")
	168	def with_trailing_slash_means_mkdir_entire_arg(
	169	self, Path, sftp_objs
	170	):
	171	transfer, client = sftp_objs
	172	transfer.get(remote="file", local="top/middle/leaf/")
	173	client.get.assert_called_with(
	174	localpath="/local/top/middle/leaf/file",
	175	remotepath="/remote/file",
	176	)
	177	Path.assert_called_with("top/middle/leaf/")
	178	Path.return_value.mkdir.assert_called_with(
	179	parents=True, exist_ok=True
	180	)
134	181
135	182	class put:
136	183	class basics: