diff --git a/CHANGELOG.md b/CHANGELOG.md index ba1dfe5..dea2785 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# 3.8.0 + +2020-01-09 + +* `k8s_raw`: added support for items without a namespace +* `k8s_raw`: fixed overriding resource name in YAML +* `k8s_raw`: allow using builtin item types if there are no actual conflicts +* decryption keys can now be set within encrypted files +* improved detection of incorrect metadata processor usage +* fixed excessive skipping of items because of concurrency dependencies +* fixed `preceded_by` not working for actions + + # 3.7.0 2019-10-07 diff --git a/bundlewrap/__init__.py b/bundlewrap/__init__.py index 727937d..81aac6f 100644 --- a/bundlewrap/__init__.py +++ b/bundlewrap/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -VERSION = (3, 7, 0) +VERSION = (3, 8, 0) VERSION_STRING = ".".join([str(v) for v in VERSION]) diff --git a/bundlewrap/deps.py b/bundlewrap/deps.py index a0d1c28..3c8b37e 100644 --- a/bundlewrap/deps.py +++ b/bundlewrap/deps.py @@ -644,6 +644,10 @@ # don't skip dummy items because of untriggered members # see issue #151; separate elif for clarity item._deps.remove(dep_item.id) + elif dep_item.id in item._concurrency_deps: + # don't skip items just because of concurrency deps + # separate elif for clarity + item._deps.remove(dep_item.id) else: removed_items.append(item) diff --git a/bundlewrap/items/__init__.py b/bundlewrap/items/__init__.py index e300dde..99ca5e5 100644 --- a/bundlewrap/items/__init__.py +++ b/bundlewrap/items/__init__.py @@ -322,11 +322,10 @@ if self.ITEM_TYPE_NAME == 'action': # so we have an action where 'unless' says it must be run # but the 'interactive' attribute might still override that - if self.attributes['interactive'] != interactive or \ - self.attributes['interactive'] is None: + if self.attributes['interactive'] and not interactive: + return False + else: return True - else: - return False return not self.cached_status.correct def _prepare_deps(self, items): diff --git a/bundlewrap/items/kubernetes.py b/bundlewrap/items/kubernetes.py index 26d5f03..9034f5c 100644 --- a/bundlewrap/items/kubernetes.py +++ b/bundlewrap/items/kubernetes.py @@ -122,7 +122,7 @@ self.attributes['manifest_file'].endswith(".yaml") or self.attributes['manifest_file'].endswith(".yml") ): - user_manifest = yaml.load(content_processor(self)) + user_manifest = yaml.load(content_processor(self), Loader=yaml.SafeLoader) elif self.attributes['manifest_file'].endswith(".json"): user_manifest = json.loads(content_processor(self)) @@ -131,7 +131,7 @@ 'apiVersion': self.KUBERNETES_APIVERSION, 'kind': self.KIND, 'metadata': { - 'name': self.name.split("/", 1)[-1], + 'name': self.name.split("/")[-1], }, }, user_manifest, @@ -227,8 +227,24 @@ BUNDLE_ATTRIBUTE_NAME = "k8s_raw" ITEM_TYPE_NAME = "k8s_raw" KUBERNETES_APIVERSION = None - NAME_REGEX = r"^([a-z0-9-\.]{1,253}/)?[a-zA-Z0-9-\.]{1,253}/[a-z0-9-\.]{1,253}$" - NAME_REGEX_COMPILED = re.compile(NAME_REGEX) + NAME_REGEX = r"^([a-z0-9-\.]{1,253})?/[a-zA-Z0-9-\.]{1,253}/[a-z0-9-\.]{1,253}$" + NAME_REGEX_COMPILED = re.compile(NAME_REGEX) + + def _check_bundle_collisions(self, items): + super(KubernetesRawItem, self)._check_bundle_collisions(items) + for item in items: + if item == self or not isinstance(item, KubernetesItem): + continue + if item.KIND == self.KIND and item.resource_name == self.resource_name: + raise BundleError(_( + "duplicate definition of {item} (from bundle {bundle}) " + "as {item2} (from bundle {bundle2}) on {node}" + ).format( + item=self.id, + bundle=self.bundle.name, + item2=item.id, + bundle2=item.bundle.name, + )) def get_auto_deps(self, items): deps = super(KubernetesRawItem, self).get_auto_deps(items) @@ -242,38 +258,7 @@ @property def KIND(self): - name = self.name.split("/", 2)[1] - if name.lower() in ( - "clusterrole", - "clusterrolebinding", - "configmap", - "cronjob", - "customresourcedefinition", - "daemonset", - "deployment", - "ingress", - "namespace", - "persistentvolumeclaim", - "service", - "secret", - "statefulset", - ): - raise BundleError(_( - "Kind of {item_type}:{name} (bundle '{bundle}') " - "on {node} clashes with builtin k8s_* item" - ).format( - item_type=self.ITEM_TYPE_NAME, - name=self.name, - bundle=self.bundle.name, - node=self.bundle.node.name, - regex=self.NAME_REGEX, - )) - else: - return name - - @property - def resource_name(self): - return self.name.split("/", 2)[2] + return self.name.split("/", 2)[1] class KubernetesClusterRole(KubernetesItem): @@ -405,7 +390,7 @@ KIND = "NetworkPolicy" KUBERNETES_APIVERSION = "networking.k8s.io/v1" ITEM_TYPE_NAME = "k8s_networkpolicy" - NAME_REGEX = r"^([a-z0-9-\.]{1,253}/)?[a-z0-9-\.]{1,253}$" + NAME_REGEX = r"^([a-z0-9-\.]{1,253})?/[a-z0-9-\.]{1,253}$" NAME_REGEX_COMPILED = re.compile(NAME_REGEX) diff --git a/bundlewrap/metadata.py b/bundlewrap/metadata.py index daa6f96..e284ca8 100644 --- a/bundlewrap/metadata.py +++ b/bundlewrap/metadata.py @@ -93,7 +93,7 @@ )) -def check_metadata_processor_result(result, node_name, metadata_processor_name): +def check_metadata_processor_result(input_metadata, result, node_name, metadata_processor_name): """ Validates the return value of a metadata processor and splits it into metadata and options. @@ -111,6 +111,17 @@ raise ValueError(_( "metadata processor {metaproc} for node {node} did not return " "a dict as the first element" + ).format( + metaproc=metadata_processor_name, + node=node_name, + )) + if ( + (DEFAULTS in options or OVERWRITE in options) and + id(input_metadata) == id(result_dict) + ): + raise ValueError(_( + "metadata processor {metaproc} for node {node} returned original " + "metadata dict plus DEFAULTS or OVERWRITE" ).format( metaproc=metadata_processor_name, node=node_name, diff --git a/bundlewrap/repo.py b/bundlewrap/repo.py index a3d1e03..1a39f2c 100644 --- a/bundlewrap/repo.py +++ b/bundlewrap/repo.py @@ -577,6 +577,7 @@ )) raise exc processed_dict, options = check_metadata_processor_result( + input_metadata, processed, node.name, metadata_processor_name, diff --git a/bundlewrap/secrets.py b/bundlewrap/secrets.py index b7a2012..13732c9 100644 --- a/bundlewrap/secrets.py +++ b/bundlewrap/secrets.py @@ -77,71 +77,71 @@ self.keys = self._load_keys() self._call_log = {} - def _decrypt(self, cryptotext=None, key='encrypt'): + def _decrypt(self, cryptotext=None, key=None): """ Decrypts a given encrypted password. """ if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0": return "decrypted text" + + key, cryptotext = self._determine_key_to_use(cryptotext.encode('utf-8'), key, cryptotext) + return Fernet(key).decrypt(cryptotext).decode('utf-8') + + def _decrypt_file(self, source_path=None, key=None): + """ + Decrypts the file at source_path (relative to data/) and + returns the plaintext as unicode. + """ + if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0": + return "decrypted file" + + cryptotext = get_file_contents(join(self.repo.data_dir, source_path)) + key, cryptotext = self._determine_key_to_use(cryptotext, key, source_path) + + f = Fernet(key) + return f.decrypt(cryptotext).decode('utf-8') + + def _decrypt_file_as_base64(self, source_path=None, key=None): + """ + Decrypts the file at source_path (relative to data/) and + returns the plaintext as base64. + """ + if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0": + return b64encode("decrypted file as base64").decode('utf-8') + + cryptotext = get_file_contents(join(self.repo.data_dir, source_path)) + key, cryptotext = self._determine_key_to_use(cryptotext, key, source_path) + + f = Fernet(key) + return b64encode(f.decrypt(cryptotext)).decode('utf-8') + + def _determine_key_to_use(self, cryptotext, key, entity_description): + key_delim = cryptotext.find(b'$') + if key_delim > -1: + key_from_text = cryptotext[:key_delim].decode('utf-8') + cryptotext = cryptotext[key_delim + 1:] + else: + key_from_text = None + + if key is None: + if key_from_text is not None: + key = key_from_text + else: + key = 'encrypt' + try: key = self.keys[key] except KeyError: raise FaultUnavailable(_( - "Key '{key}' not available for decryption of the following cryptotext, " - "check your {file}: {cryptotext}" - ).format( - cryptotext=cryptotext, - file=FILENAME_SECRETS, - key=key, - )) - - return Fernet(key).decrypt(cryptotext.encode('utf-8')).decode('utf-8') - - def _decrypt_file(self, source_path=None, key='encrypt'): - """ - Decrypts the file at source_path (relative to data/) and - returns the plaintext as unicode. - """ - if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0": - return "decrypted file" - try: - key = self.keys[key] - except KeyError: - raise FaultUnavailable(_( - "Key '{key}' not available for decryption of the following file, " - "check your {file}: {source_path}" + "Key '{key}' not available for decryption of the following entity, " + "check your {file}: {entity_description}" ).format( file=FILENAME_SECRETS, key=key, - source_path=source_path, + entity_description=entity_description, )) - f = Fernet(key) - return f.decrypt(get_file_contents(join(self.repo.data_dir, source_path))).decode('utf-8') - - def _decrypt_file_as_base64(self, source_path=None, key='encrypt'): - """ - Decrypts the file at source_path (relative to data/) and - returns the plaintext as base64. - """ - if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0": - return b64encode("decrypted file as base64").decode('utf-8') - try: - key = self.keys[key] - except KeyError: - raise FaultUnavailable(_( - "Key '{key}' not available for decryption of the following file, " - "check your {file}: {source_path}" - ).format( - file=FILENAME_SECRETS, - key=key, - source_path=source_path, - )) - - f = Fernet(key) - return b64encode(f.decrypt(get_file_contents( - join(self.repo.data_dir, source_path), - ))).decode('utf-8') + return key, cryptotext def _generate_human_password( self, identifier=None, digits=2, key='generate', per_word=3, words=4, @@ -255,21 +255,21 @@ result[section] = config.get(section, 'key').encode('utf-8') return result - def decrypt(self, cryptotext, key='encrypt'): + def decrypt(self, cryptotext, key=None): return Fault( self._decrypt, cryptotext=cryptotext, key=key, ) - def decrypt_file(self, source_path, key='encrypt'): + def decrypt_file(self, source_path, key=None): return Fault( self._decrypt_file, source_path=source_path, key=key, ) - def decrypt_file_as_base64(self, source_path, key='encrypt'): + def decrypt_file_as_base64(self, source_path, key=None): return Fault( self._decrypt_file_as_base64, source_path=source_path, @@ -281,6 +281,7 @@ Encrypts a given plaintext password and returns a string that can be fed into decrypt() to get the password back. """ + key_name = key try: key = self.keys[key] except KeyError: @@ -291,7 +292,7 @@ key=key, )) - return Fernet(key).encrypt(plaintext.encode('utf-8')).decode('utf-8') + return key_name + '$' + Fernet(key).encrypt(plaintext.encode('utf-8')).decode('utf-8') def encrypt_file(self, source_path, target_path, key='encrypt'): """ @@ -299,6 +300,7 @@ target_path. The source_path is relative to CWD or absolute, while target_path is relative to data/. """ + key_name = key try: key = self.keys[key] except KeyError: @@ -313,6 +315,7 @@ fernet = Fernet(key) target_file = join(self.repo.data_dir, target_path) with open(target_file, 'wb') as f: + f.write(key_name.encode('utf-8') + b'$') f.write(fernet.encrypt(plaintext)) return target_file diff --git a/debian/changelog b/debian/changelog index 96e6cb9..35c1967 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +bundlewrap (3.8.0-1) unstable; urgency=medium + + * New upstream release + * Fix standards version and set to 4.4.1 + * Update copyright years + * Declare Rules-Requires-Root + + -- Jonathan Carter Fri, 10 Jan 2020 12:05:02 +0200 + bundlewrap (3.7.0-1) unstable; urgency=medium [ Ondřej Nový ] diff --git a/debian/control b/debian/control index c2e6ee9..4db37e3 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,8 @@ python3-setuptools, python3-requests, python3-cryptography -Standards-Version: 4.5.0 +Standards-Version: 4.4.1 +Rules-Requires-Root: no Homepage: http://bundlewrap.org/ Vcs-Git: https://salsa.debian.org/python-team/applications/bundlewrap.git Vcs-Browser: https://salsa.debian.org/python-team/applications/bundlewrap diff --git a/debian/copyright b/debian/copyright index c60bd20..3578c3b 100644 --- a/debian/copyright +++ b/debian/copyright @@ -3,7 +3,7 @@ Source: https://github.com/bundlewrap/bundlewrap Files: * -Copyright: 2016-2019 Torsten Rehn +Copyright: 2016-2020 Torsten Rehn Comment: Copyrights are assigned to Torsten Rehn (see: CAA.md) Additional author: Peter Hofmann Additional author: Tim Buchwaldt @@ -12,7 +12,7 @@ License: GPL-3 Files: debian/* -Copyright: 2016-2019 Jonathan Carter +Copyright: 2016-2020 Jonathan Carter License: GPL-3 License: GPL-3 diff --git a/docs/content/guide/secrets.md b/docs/content/guide/secrets.md index a4a3788..118361d 100644 --- a/docs/content/guide/secrets.md +++ b/docs/content/guide/secrets.md @@ -103,9 +103,11 @@ You can always add more keys to your `.secrets.cfg`, but you should keep the defaults around. Adding more keys makes it possible to give different keys to different teams. **By default, BundleWrap will skip items it can't find the required keys for**. -When using `.password_for()`, `.decrypt()` etc., you can provide a `key` argument to select the key: +When using `.password_for()`, `.encrypt()` etc., you can provide a `key` argument to select the key: repo.vault.password_for("some database", key="devops") + +The encrypted data will be prefixed by `yourkeyname$...` to indicate that the key `yourkeyname` was used for encryption. Thus, during decryption, you can omit the `key=` parameter.
diff --git a/docs/content/repo/metadata.py.md b/docs/content/repo/metadata.py.md index fecd950..a6bd1e4 100644 --- a/docs/content/repo/metadata.py.md +++ b/docs/content/repo/metadata.py.md @@ -36,8 +36,8 @@ OptionDescription DONEIndicates that this metadata processor has done all it can and need not be called again. Return this whenever possible. RUN_ME_AGAINIndicates that this metadata processor is still waiting for metadata from another metadata processor to become available. -DEFAULTSThe returned metadata dictionary will only be used to provide default values. The actual metadata generated so far will be recursively merged into the returned dict. -OVERWRITEThe returned metadata dictionary will be recursively merged into the actual metadata generated so far (inverse of DEFAULTS). +DEFAULTSThe returned metadata dictionary will only be used to provide default values. The actual metadata generated so far will be recursively merged into the returned dict. When using this flag, you must not return the original metadata dictionary but construct a new one as in the example below. +OVERWRITEThe returned metadata dictionary will be recursively merged into the actual metadata generated so far (inverse of DEFAULTS). When using this flag, you must not return the original metadata dictionary but construct a new one as in the `DEFAULTS` example below. Here is an example of how to use `DEFAULTS`: diff --git a/setup.py b/setup.py index ca289d0..c0cd3dd 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name="bundlewrap", - version="3.7.0", + version="3.8.0", description="Config management with Python", long_description=( "By allowing for easy and low-overhead config management, BundleWrap fills the gap between complex deployments using Chef or Puppet and old school system administration over SSH.\n" diff --git a/tests/integration/bw_lock.py b/tests/integration/bw_lock.py index d3a1324..ea39be9 100644 --- a/tests/integration/bw_lock.py +++ b/tests/integration/bw_lock.py @@ -6,7 +6,7 @@ def get_lock_id(output): - return search("locked with ID (\w+) ", output).groups()[0] + return search(r"locked with ID (\w+) ", output).groups()[0] def test_add_lock_apply_remove(tmpdir): diff --git a/tests/integration/bw_metadata.py b/tests/integration/bw_metadata.py index 6e61e4f..3fbf911 100644 --- a/tests/integration/bw_metadata.py +++ b/tests/integration/bw_metadata.py @@ -258,6 +258,174 @@ assert rcode == 0 +def test_metadatapy_invalid_number_of_elements(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {"foo": "bar"}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +"""@metadata_processor +def foo(metadata): + return metadata +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert rcode != 0 + + +def test_metadatapy_invalid_first_element_not_dict(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {"foo": "bar"}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +"""@metadata_processor +def foo(metadata): + return DONE, metadata +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert rcode != 0 + + +def test_metadatapy_invalid_defaults_plus_original_dict(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {"foo": "bar"}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +"""@metadata_processor +def foo(metadata): + return metadata, DONE, DEFAULTS +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert rcode != 0 + + +def test_metadatapy_invalid_overwrite_plus_original_dict(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {"foo": "bar"}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +"""@metadata_processor +def foo(metadata): + return metadata, DONE, OVERWRITE +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert rcode != 0 + + +def test_metadatapy_invalid_option(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {"foo": "bar"}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +"""@metadata_processor +def foo(metadata): + return metadata, DONE, 1000 +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert rcode != 0 + + +def test_metadatapy_invalid_done_and_again(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {"foo": "bar"}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +"""@metadata_processor +def foo(metadata): + return metadata, DONE, RUN_ME_AGAIN +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert rcode != 0 + + +def test_metadatapy_invalid_no_done_or_again(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {"foo": "bar"}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +"""@metadata_processor +def foo(metadata): + return {}, DEFAULTS +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert rcode != 0 + + +def test_metadatapy_invalid_defaults_and_overwrite(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {"foo": "bar"}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +"""@metadata_processor +def foo(metadata): + return {}, DEFAULTS, OVERWRITE, DONE +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert rcode != 0 + + def test_table(tmpdir): make_repo( tmpdir, diff --git a/tests/integration/secrets.py b/tests/integration/secrets.py index eb10774..2750fd7 100644 --- a/tests/integration/secrets.py +++ b/tests/integration/secrets.py @@ -29,6 +29,19 @@ assert rcode == 0 +def test_encrypt_different_key_autodetect(tmpdir): + make_repo(tmpdir) + + stdout, stderr, rcode = run("bw debug -c 'print(repo.vault.encrypt(\"test\", key=\"generate\"))'", path=str(tmpdir)) + assert stderr == b"" + assert rcode == 0 + + stdout, stderr, rcode = run("bw debug -c 'print(repo.vault.decrypt(\"{}\"))'".format(stdout.decode('utf-8').strip()), path=str(tmpdir)) + assert stdout == b"test\n" + assert stderr == b"" + assert rcode == 0 + + def test_encrypt_file(tmpdir): make_repo(tmpdir) @@ -40,6 +53,35 @@ "bw debug -c 'repo.vault.encrypt_file(\"{}\", \"{}\")'".format( source_file, "encrypted", + ), + path=str(tmpdir), + ) + assert stderr == b"" + assert rcode == 0 + + stdout, stderr, rcode = run( + "bw debug -c 'print(repo.vault.decrypt_file(\"{}\"))'".format( + "encrypted", + ), + path=str(tmpdir), + ) + assert stdout == b"ohai\n" + assert stderr == b"" + assert rcode == 0 + + +def test_encrypt_file_different_key_autodetect(tmpdir): + make_repo(tmpdir) + + source_file = join(str(tmpdir), "data", "source") + with open(source_file, 'w') as f: + f.write("ohai") + + stdout, stderr, rcode = run( + "bw debug -c 'repo.vault.encrypt_file(\"{}\", \"{}\", \"{}\")'".format( + source_file, + "encrypted", + "generate", ), path=str(tmpdir), )