Codebase list bundlewrap / 8edbecd
New upstream release Jonathan Carter 4 years ago
17 changed file(s) with 344 addition(s) and 106 deletion(s). Raw diff Collapse all Expand all
0 # 3.8.0
1
2 2020-01-09
3
4 * `k8s_raw`: added support for items without a namespace
5 * `k8s_raw`: fixed overriding resource name in YAML
6 * `k8s_raw`: allow using builtin item types if there are no actual conflicts
7 * decryption keys can now be set within encrypted files
8 * improved detection of incorrect metadata processor usage
9 * fixed excessive skipping of items because of concurrency dependencies
10 * fixed `preceded_by` not working for actions
11
12
013 # 3.7.0
114
215 2019-10-07
00 # -*- coding: utf-8 -*-
11 from __future__ import unicode_literals
22
3 VERSION = (3, 7, 0)
3 VERSION = (3, 8, 0)
44 VERSION_STRING = ".".join([str(v) for v in VERSION])
643643 # don't skip dummy items because of untriggered members
644644 # see issue #151; separate elif for clarity
645645 item._deps.remove(dep_item.id)
646 elif dep_item.id in item._concurrency_deps:
647 # don't skip items just because of concurrency deps
648 # separate elif for clarity
649 item._deps.remove(dep_item.id)
646650 else:
647651 removed_items.append(item)
648652
321321 if self.ITEM_TYPE_NAME == 'action':
322322 # so we have an action where 'unless' says it must be run
323323 # but the 'interactive' attribute might still override that
324 if self.attributes['interactive'] != interactive or \
325 self.attributes['interactive'] is None:
324 if self.attributes['interactive'] and not interactive:
325 return False
326 else:
326327 return True
327 else:
328 return False
329328 return not self.cached_status.correct
330329
331330 def _prepare_deps(self, items):
121121 self.attributes['manifest_file'].endswith(".yaml") or
122122 self.attributes['manifest_file'].endswith(".yml")
123123 ):
124 user_manifest = yaml.load(content_processor(self))
124 user_manifest = yaml.load(content_processor(self), Loader=yaml.SafeLoader)
125125 elif self.attributes['manifest_file'].endswith(".json"):
126126 user_manifest = json.loads(content_processor(self))
127127
130130 'apiVersion': self.KUBERNETES_APIVERSION,
131131 'kind': self.KIND,
132132 'metadata': {
133 'name': self.name.split("/", 1)[-1],
133 'name': self.name.split("/")[-1],
134134 },
135135 },
136136 user_manifest,
226226 BUNDLE_ATTRIBUTE_NAME = "k8s_raw"
227227 ITEM_TYPE_NAME = "k8s_raw"
228228 KUBERNETES_APIVERSION = None
229 NAME_REGEX = r"^([a-z0-9-\.]{1,253}/)?[a-zA-Z0-9-\.]{1,253}/[a-z0-9-\.]{1,253}$"
230 NAME_REGEX_COMPILED = re.compile(NAME_REGEX)
229 NAME_REGEX = r"^([a-z0-9-\.]{1,253})?/[a-zA-Z0-9-\.]{1,253}/[a-z0-9-\.]{1,253}$"
230 NAME_REGEX_COMPILED = re.compile(NAME_REGEX)
231
232 def _check_bundle_collisions(self, items):
233 super(KubernetesRawItem, self)._check_bundle_collisions(items)
234 for item in items:
235 if item == self or not isinstance(item, KubernetesItem):
236 continue
237 if item.KIND == self.KIND and item.resource_name == self.resource_name:
238 raise BundleError(_(
239 "duplicate definition of {item} (from bundle {bundle}) "
240 "as {item2} (from bundle {bundle2}) on {node}"
241 ).format(
242 item=self.id,
243 bundle=self.bundle.name,
244 item2=item.id,
245 bundle2=item.bundle.name,
246 ))
231247
232248 def get_auto_deps(self, items):
233249 deps = super(KubernetesRawItem, self).get_auto_deps(items)
241257
242258 @property
243259 def KIND(self):
244 name = self.name.split("/", 2)[1]
245 if name.lower() in (
246 "clusterrole",
247 "clusterrolebinding",
248 "configmap",
249 "cronjob",
250 "customresourcedefinition",
251 "daemonset",
252 "deployment",
253 "ingress",
254 "namespace",
255 "persistentvolumeclaim",
256 "service",
257 "secret",
258 "statefulset",
259 ):
260 raise BundleError(_(
261 "Kind of {item_type}:{name} (bundle '{bundle}') "
262 "on {node} clashes with builtin k8s_* item"
263 ).format(
264 item_type=self.ITEM_TYPE_NAME,
265 name=self.name,
266 bundle=self.bundle.name,
267 node=self.bundle.node.name,
268 regex=self.NAME_REGEX,
269 ))
270 else:
271 return name
272
273 @property
274 def resource_name(self):
275 return self.name.split("/", 2)[2]
260 return self.name.split("/", 2)[1]
276261
277262
278263 class KubernetesClusterRole(KubernetesItem):
404389 KIND = "NetworkPolicy"
405390 KUBERNETES_APIVERSION = "networking.k8s.io/v1"
406391 ITEM_TYPE_NAME = "k8s_networkpolicy"
407 NAME_REGEX = r"^([a-z0-9-\.]{1,253}/)?[a-z0-9-\.]{1,253}$"
392 NAME_REGEX = r"^([a-z0-9-\.]{1,253})?/[a-z0-9-\.]{1,253}$"
408393 NAME_REGEX_COMPILED = re.compile(NAME_REGEX)
409394
410395
9292 ))
9393
9494
95 def check_metadata_processor_result(result, node_name, metadata_processor_name):
95 def check_metadata_processor_result(input_metadata, result, node_name, metadata_processor_name):
9696 """
9797 Validates the return value of a metadata processor and splits it
9898 into metadata and options.
110110 raise ValueError(_(
111111 "metadata processor {metaproc} for node {node} did not return "
112112 "a dict as the first element"
113 ).format(
114 metaproc=metadata_processor_name,
115 node=node_name,
116 ))
117 if (
118 (DEFAULTS in options or OVERWRITE in options) and
119 id(input_metadata) == id(result_dict)
120 ):
121 raise ValueError(_(
122 "metadata processor {metaproc} for node {node} returned original "
123 "metadata dict plus DEFAULTS or OVERWRITE"
113124 ).format(
114125 metaproc=metadata_processor_name,
115126 node=node_name,
576576 ))
577577 raise exc
578578 processed_dict, options = check_metadata_processor_result(
579 input_metadata,
579580 processed,
580581 node.name,
581582 metadata_processor_name,
7676 self.keys = self._load_keys()
7777 self._call_log = {}
7878
79 def _decrypt(self, cryptotext=None, key='encrypt'):
79 def _decrypt(self, cryptotext=None, key=None):
8080 """
8181 Decrypts a given encrypted password.
8282 """
8383 if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0":
8484 return "decrypted text"
85
86 key, cryptotext = self._determine_key_to_use(cryptotext.encode('utf-8'), key, cryptotext)
87 return Fernet(key).decrypt(cryptotext).decode('utf-8')
88
89 def _decrypt_file(self, source_path=None, key=None):
90 """
91 Decrypts the file at source_path (relative to data/) and
92 returns the plaintext as unicode.
93 """
94 if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0":
95 return "decrypted file"
96
97 cryptotext = get_file_contents(join(self.repo.data_dir, source_path))
98 key, cryptotext = self._determine_key_to_use(cryptotext, key, source_path)
99
100 f = Fernet(key)
101 return f.decrypt(cryptotext).decode('utf-8')
102
103 def _decrypt_file_as_base64(self, source_path=None, key=None):
104 """
105 Decrypts the file at source_path (relative to data/) and
106 returns the plaintext as base64.
107 """
108 if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0":
109 return b64encode("decrypted file as base64").decode('utf-8')
110
111 cryptotext = get_file_contents(join(self.repo.data_dir, source_path))
112 key, cryptotext = self._determine_key_to_use(cryptotext, key, source_path)
113
114 f = Fernet(key)
115 return b64encode(f.decrypt(cryptotext)).decode('utf-8')
116
117 def _determine_key_to_use(self, cryptotext, key, entity_description):
118 key_delim = cryptotext.find(b'$')
119 if key_delim > -1:
120 key_from_text = cryptotext[:key_delim].decode('utf-8')
121 cryptotext = cryptotext[key_delim + 1:]
122 else:
123 key_from_text = None
124
125 if key is None:
126 if key_from_text is not None:
127 key = key_from_text
128 else:
129 key = 'encrypt'
130
85131 try:
86132 key = self.keys[key]
87133 except KeyError:
88134 raise FaultUnavailable(_(
89 "Key '{key}' not available for decryption of the following cryptotext, "
90 "check your {file}: {cryptotext}"
91 ).format(
92 cryptotext=cryptotext,
93 file=FILENAME_SECRETS,
94 key=key,
95 ))
96
97 return Fernet(key).decrypt(cryptotext.encode('utf-8')).decode('utf-8')
98
99 def _decrypt_file(self, source_path=None, key='encrypt'):
100 """
101 Decrypts the file at source_path (relative to data/) and
102 returns the plaintext as unicode.
103 """
104 if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0":
105 return "decrypted file"
106 try:
107 key = self.keys[key]
108 except KeyError:
109 raise FaultUnavailable(_(
110 "Key '{key}' not available for decryption of the following file, "
111 "check your {file}: {source_path}"
135 "Key '{key}' not available for decryption of the following entity, "
136 "check your {file}: {entity_description}"
112137 ).format(
113138 file=FILENAME_SECRETS,
114139 key=key,
115 source_path=source_path,
140 entity_description=entity_description,
116141 ))
117142
118 f = Fernet(key)
119 return f.decrypt(get_file_contents(join(self.repo.data_dir, source_path))).decode('utf-8')
120
121 def _decrypt_file_as_base64(self, source_path=None, key='encrypt'):
122 """
123 Decrypts the file at source_path (relative to data/) and
124 returns the plaintext as base64.
125 """
126 if environ.get("BW_VAULT_DUMMY_MODE", "0") != "0":
127 return b64encode("decrypted file as base64").decode('utf-8')
128 try:
129 key = self.keys[key]
130 except KeyError:
131 raise FaultUnavailable(_(
132 "Key '{key}' not available for decryption of the following file, "
133 "check your {file}: {source_path}"
134 ).format(
135 file=FILENAME_SECRETS,
136 key=key,
137 source_path=source_path,
138 ))
139
140 f = Fernet(key)
141 return b64encode(f.decrypt(get_file_contents(
142 join(self.repo.data_dir, source_path),
143 ))).decode('utf-8')
143 return key, cryptotext
144144
145145 def _generate_human_password(
146146 self, identifier=None, digits=2, key='generate', per_word=3, words=4,
254254 result[section] = config.get(section, 'key').encode('utf-8')
255255 return result
256256
257 def decrypt(self, cryptotext, key='encrypt'):
257 def decrypt(self, cryptotext, key=None):
258258 return Fault(
259259 self._decrypt,
260260 cryptotext=cryptotext,
261261 key=key,
262262 )
263263
264 def decrypt_file(self, source_path, key='encrypt'):
264 def decrypt_file(self, source_path, key=None):
265265 return Fault(
266266 self._decrypt_file,
267267 source_path=source_path,
268268 key=key,
269269 )
270270
271 def decrypt_file_as_base64(self, source_path, key='encrypt'):
271 def decrypt_file_as_base64(self, source_path, key=None):
272272 return Fault(
273273 self._decrypt_file_as_base64,
274274 source_path=source_path,
280280 Encrypts a given plaintext password and returns a string that can
281281 be fed into decrypt() to get the password back.
282282 """
283 key_name = key
283284 try:
284285 key = self.keys[key]
285286 except KeyError:
290291 key=key,
291292 ))
292293
293 return Fernet(key).encrypt(plaintext.encode('utf-8')).decode('utf-8')
294 return key_name + '$' + Fernet(key).encrypt(plaintext.encode('utf-8')).decode('utf-8')
294295
295296 def encrypt_file(self, source_path, target_path, key='encrypt'):
296297 """
298299 target_path. The source_path is relative to CWD or absolute,
299300 while target_path is relative to data/.
300301 """
302 key_name = key
301303 try:
302304 key = self.keys[key]
303305 except KeyError:
312314 fernet = Fernet(key)
313315 target_file = join(self.repo.data_dir, target_path)
314316 with open(target_file, 'wb') as f:
317 f.write(key_name.encode('utf-8') + b'$')
315318 f.write(fernet.encrypt(plaintext))
316319 return target_file
317320
0 bundlewrap (3.8.0-1) unstable; urgency=medium
1
2 * New upstream release
3 * Fix standards version and set to 4.4.1
4 * Update copyright years
5 * Declare Rules-Requires-Root
6
7 -- Jonathan Carter <jcc@debian.org> Fri, 10 Jan 2020 12:05:02 +0200
8
09 bundlewrap (3.7.0-1) unstable; urgency=medium
110
211 [ Ondřej Nový ]
99 python3-setuptools,
1010 python3-requests,
1111 python3-cryptography
12 Standards-Version: 4.5.0
12 Standards-Version: 4.4.1
13 Rules-Requires-Root: no
1314 Homepage: http://bundlewrap.org/
1415 Vcs-Git: https://salsa.debian.org/python-team/applications/bundlewrap.git
1516 Vcs-Browser: https://salsa.debian.org/python-team/applications/bundlewrap
22 Source: https://github.com/bundlewrap/bundlewrap
33
44 Files: *
5 Copyright: 2016-2019 Torsten Rehn <torsten@rehn.email>
5 Copyright: 2016-2020 Torsten Rehn <torsten@rehn.email>
66 Comment: Copyrights are assigned to Torsten Rehn (see: CAA.md)
77 Additional author: Peter Hofmann <scm@uninformativ.de>
88 Additional author: Tim Buchwaldt <tim@buchwaldt.ws>
1111 License: GPL-3
1212
1313 Files: debian/*
14 Copyright: 2016-2019 Jonathan Carter <jcarter@linux.com>
14 Copyright: 2016-2020 Jonathan Carter <jcc@debian.org>
1515 License: GPL-3
1616
1717 License: GPL-3
102102
103103 You can always add more keys to your `.secrets.cfg`, but you should keep the defaults around. Adding more keys makes it possible to give different keys to different teams. **By default, BundleWrap will skip items it can't find the required keys for**.
104104
105 When using `.password_for()`, `.decrypt()` etc., you can provide a `key` argument to select the key:
105 When using `.password_for()`, `.encrypt()` etc., you can provide a `key` argument to select the key:
106106
107107 repo.vault.password_for("some database", key="devops")
108
109 The encrypted data will be prefixed by `yourkeyname$...` to indicate that the key `yourkeyname` was used for encryption. Thus, during decryption, you can omit the `key=` parameter.
108110
109111 <br>
110112
3535 <tr><th>Option</th><th>Description</th></tr>
3636 <tr><td><code>DONE</code></td><td>Indicates that this metadata processor has done all it can and need not be called again. Return this whenever possible.</td></tr>
3737 <tr><td><code>RUN_ME_AGAIN</code></td><td>Indicates that this metadata processor is still waiting for metadata from another metadata processor to become available.</td></tr>
38 <tr><td><code>DEFAULTS</code></td><td>The returned metadata dictionary will only be used to provide default values. The actual metadata generated so far will be recursively merged into the returned dict.</td></tr>
39 <tr><td><code>OVERWRITE</code></td><td>The returned metadata dictionary will be recursively merged into the actual metadata generated so far (inverse of <code>DEFAULTS</code>).</td></tr>
38 <tr><td><code>DEFAULTS</code></td><td>The returned metadata dictionary will only be used to provide default values. The actual metadata generated so far will be recursively merged into the returned dict. When using this flag, you must not return the original metadata dictionary but construct a new one as in the example below.</td></tr>
39 <tr><td><code>OVERWRITE</code></td><td>The returned metadata dictionary will be recursively merged into the actual metadata generated so far (inverse of <code>DEFAULTS</code>). When using this flag, you must not return the original metadata dictionary but construct a new one as in the `DEFAULTS` example below.</td></tr>
4040 </table>
4141
4242 Here is an example of how to use `DEFAULTS`:
1616
1717 setup(
1818 name="bundlewrap",
19 version="3.7.0",
19 version="3.8.0",
2020 description="Config management with Python",
2121 long_description=(
2222 "By allowing for easy and low-overhead config management, BundleWrap fills the gap between complex deployments using Chef or Puppet and old school system administration over SSH.\n"
55
66
77 def get_lock_id(output):
8 return search("locked with ID (\w+) ", output).groups()[0]
8 return search(r"locked with ID (\w+) ", output).groups()[0]
99
1010
1111 def test_add_lock_apply_remove(tmpdir):
257257 assert rcode == 0
258258
259259
260 def test_metadatapy_invalid_number_of_elements(tmpdir):
261 make_repo(
262 tmpdir,
263 bundles={"test": {}},
264 nodes={
265 "node1": {
266 'bundles': ["test"],
267 'metadata': {"foo": "bar"},
268 },
269 },
270 )
271 with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f:
272 f.write(
273 """@metadata_processor
274 def foo(metadata):
275 return metadata
276 """)
277 stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir))
278 assert rcode != 0
279
280
281 def test_metadatapy_invalid_first_element_not_dict(tmpdir):
282 make_repo(
283 tmpdir,
284 bundles={"test": {}},
285 nodes={
286 "node1": {
287 'bundles': ["test"],
288 'metadata': {"foo": "bar"},
289 },
290 },
291 )
292 with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f:
293 f.write(
294 """@metadata_processor
295 def foo(metadata):
296 return DONE, metadata
297 """)
298 stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir))
299 assert rcode != 0
300
301
302 def test_metadatapy_invalid_defaults_plus_original_dict(tmpdir):
303 make_repo(
304 tmpdir,
305 bundles={"test": {}},
306 nodes={
307 "node1": {
308 'bundles': ["test"],
309 'metadata': {"foo": "bar"},
310 },
311 },
312 )
313 with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f:
314 f.write(
315 """@metadata_processor
316 def foo(metadata):
317 return metadata, DONE, DEFAULTS
318 """)
319 stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir))
320 assert rcode != 0
321
322
323 def test_metadatapy_invalid_overwrite_plus_original_dict(tmpdir):
324 make_repo(
325 tmpdir,
326 bundles={"test": {}},
327 nodes={
328 "node1": {
329 'bundles': ["test"],
330 'metadata': {"foo": "bar"},
331 },
332 },
333 )
334 with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f:
335 f.write(
336 """@metadata_processor
337 def foo(metadata):
338 return metadata, DONE, OVERWRITE
339 """)
340 stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir))
341 assert rcode != 0
342
343
344 def test_metadatapy_invalid_option(tmpdir):
345 make_repo(
346 tmpdir,
347 bundles={"test": {}},
348 nodes={
349 "node1": {
350 'bundles': ["test"],
351 'metadata': {"foo": "bar"},
352 },
353 },
354 )
355 with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f:
356 f.write(
357 """@metadata_processor
358 def foo(metadata):
359 return metadata, DONE, 1000
360 """)
361 stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir))
362 assert rcode != 0
363
364
365 def test_metadatapy_invalid_done_and_again(tmpdir):
366 make_repo(
367 tmpdir,
368 bundles={"test": {}},
369 nodes={
370 "node1": {
371 'bundles': ["test"],
372 'metadata': {"foo": "bar"},
373 },
374 },
375 )
376 with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f:
377 f.write(
378 """@metadata_processor
379 def foo(metadata):
380 return metadata, DONE, RUN_ME_AGAIN
381 """)
382 stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir))
383 assert rcode != 0
384
385
386 def test_metadatapy_invalid_no_done_or_again(tmpdir):
387 make_repo(
388 tmpdir,
389 bundles={"test": {}},
390 nodes={
391 "node1": {
392 'bundles': ["test"],
393 'metadata': {"foo": "bar"},
394 },
395 },
396 )
397 with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f:
398 f.write(
399 """@metadata_processor
400 def foo(metadata):
401 return {}, DEFAULTS
402 """)
403 stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir))
404 assert rcode != 0
405
406
407 def test_metadatapy_invalid_defaults_and_overwrite(tmpdir):
408 make_repo(
409 tmpdir,
410 bundles={"test": {}},
411 nodes={
412 "node1": {
413 'bundles': ["test"],
414 'metadata': {"foo": "bar"},
415 },
416 },
417 )
418 with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f:
419 f.write(
420 """@metadata_processor
421 def foo(metadata):
422 return {}, DEFAULTS, OVERWRITE, DONE
423 """)
424 stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir))
425 assert rcode != 0
426
427
260428 def test_table(tmpdir):
261429 make_repo(
262430 tmpdir,
2828 assert rcode == 0
2929
3030
31 def test_encrypt_different_key_autodetect(tmpdir):
32 make_repo(tmpdir)
33
34 stdout, stderr, rcode = run("bw debug -c 'print(repo.vault.encrypt(\"test\", key=\"generate\"))'", path=str(tmpdir))
35 assert stderr == b""
36 assert rcode == 0
37
38 stdout, stderr, rcode = run("bw debug -c 'print(repo.vault.decrypt(\"{}\"))'".format(stdout.decode('utf-8').strip()), path=str(tmpdir))
39 assert stdout == b"test\n"
40 assert stderr == b""
41 assert rcode == 0
42
43
3144 def test_encrypt_file(tmpdir):
3245 make_repo(tmpdir)
3346
3952 "bw debug -c 'repo.vault.encrypt_file(\"{}\", \"{}\")'".format(
4053 source_file,
4154 "encrypted",
55 ),
56 path=str(tmpdir),
57 )
58 assert stderr == b""
59 assert rcode == 0
60
61 stdout, stderr, rcode = run(
62 "bw debug -c 'print(repo.vault.decrypt_file(\"{}\"))'".format(
63 "encrypted",
64 ),
65 path=str(tmpdir),
66 )
67 assert stdout == b"ohai\n"
68 assert stderr == b""
69 assert rcode == 0
70
71
72 def test_encrypt_file_different_key_autodetect(tmpdir):
73 make_repo(tmpdir)
74
75 source_file = join(str(tmpdir), "data", "source")
76 with open(source_file, 'w') as f:
77 f.write("ohai")
78
79 stdout, stderr, rcode = run(
80 "bw debug -c 'repo.vault.encrypt_file(\"{}\", \"{}\", \"{}\")'".format(
81 source_file,
82 "encrypted",
83 "generate",
4284 ),
4385 path=str(tmpdir),
4486 )