diff --git a/CHANGELOG.md b/CHANGELOG.md index ce9dfb8..cdc400e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +# 3.1.0 + +2017-10-10 + +* added pkg_opkg items +* added `bw test -s` +* improved error messages for unknown reverse triggers +* fixed hash_method md5 on user items +* fixed cursor sometimes not being restored + + # 3.0.3 2017-10-04 diff --git a/bundlewrap/__init__.py b/bundlewrap/__init__.py index 9526cbf..1fe3d6c 100644 --- a/bundlewrap/__init__.py +++ b/bundlewrap/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -VERSION = (3, 0, 3) +VERSION = (3, 1, 0) VERSION_STRING = ".".join([str(v) for v in VERSION]) diff --git a/bundlewrap/cmdline/__init__.py b/bundlewrap/cmdline/__init__.py index 7c38eee..f301bf5 100644 --- a/bundlewrap/cmdline/__init__.py +++ b/bundlewrap/cmdline/__init__.py @@ -7,7 +7,7 @@ from os.path import abspath, dirname from pipes import quote from sys import argv, exit, stderr, stdout -from traceback import print_exc +from traceback import format_exc, print_exc from ..exceptions import NoSuchRepository, MissingRepoDependency @@ -133,11 +133,17 @@ "{x} {path} " "is not a BundleWrap repository." ).format(path=quote(abspath(pargs.repo_path)), x=red("!!!"))) + io.deactivate() exit(1) else: path = dirname(path) except MissingRepoDependency as exc: io.stderr(str(exc)) + io.deactivate() + exit(1) + except Exception: + io.stderr(format_exc()) + io.deactivate() exit(1) # convert all string args into text diff --git a/bundlewrap/cmdline/parser.py b/bundlewrap/cmdline/parser.py index 4d0ed6d..60f0eed 100644 --- a/bundlewrap/cmdline/parser.py +++ b/bundlewrap/cmdline/parser.py @@ -852,6 +852,19 @@ help=_("check for bundles not assigned to any node"), ) parser_test.add_argument( + "-s", + "--secret-rotation", + default=None, + dest='ignore_secret_identifiers', + help=_("ensure every string passed to repo.vault.[human_]password_for() is used at least " + "twice (using it only once means you're probably managing only one end of an " + "authentication, making it dangerous to rotate your .secrets.cfg); PATTERNS is a " + "comma-separated list of regex patterns for strings to ignore in this check " + "(just pass an empty string if you don't need to ignore anything)"), + metavar="PATTERNS", + type=str, + ) + parser_test.add_argument( "-S", "--subgroup-loops", action='store_true', diff --git a/bundlewrap/cmdline/test.py b/bundlewrap/cmdline/test.py index f910ad9..0701cf0 100644 --- a/bundlewrap/cmdline/test.py +++ b/bundlewrap/cmdline/test.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from copy import copy +from re import compile as compile_regex from sys import exit from ..deps import DummyItem @@ -120,6 +121,39 @@ exit(1) +def test_secret_identifiers(repo, ignore_patterns): + # create a new object to make sure we don't double-count any calls + # from previous tests + pristine_repo = Repository(repo.path) + pristine_repo.hash() # shortest way to create all configuration + patterns = set() + for raw_pattern in ignore_patterns.split(","): + if raw_pattern: + patterns.add(compile_regex(raw_pattern)) + found_something = False + for identifier, call_count in pristine_repo.vault._call_log.items(): + if call_count == 1: + ignore = False + for pattern in patterns: + if pattern.search(identifier): + ignore = True + break + if not ignore: + io.stderr(_( + "{x} identifier passed only once to repo.vault.[human_]password_for(): {i}" + ).format( + i=bold(identifier), + x=red("✘"), + )) + found_something = True + if found_something: + exit(1) + else: + io.stdout(_( + "{x} all arguments to repo.vault.[human_]password_for() used at least twice" + ).format(x=green("✓"))) + + def test_empty_groups(repo): empty_groups = set() for group in repo.groups: @@ -249,6 +283,7 @@ args['determinism_metadata'] > 1 or args['hooks_node'] or args['hooks_repo'] or + args['ignore_secret_identifiers'] is not None or args['items'] or args['metadata_collisions'] or args['orphaned_bundles'] or @@ -271,6 +306,9 @@ args['metadata_collisions'] = True args['subgroup_loops'] = True + if args['ignore_secret_identifiers'] is not None and not QUIT_EVENT.is_set(): + test_secret_identifiers(repo, args['ignore_secret_identifiers']) + if args['plugin_conflicts'] and not QUIT_EVENT.is_set(): test_plugin_conflicts(repo) diff --git a/bundlewrap/deps.py b/bundlewrap/deps.py index 57e224a..bc3c10a 100644 --- a/bundlewrap/deps.py +++ b/bundlewrap/deps.py @@ -441,7 +441,17 @@ """ for item in items.values(): for triggering_item_id in item.triggered_by: - triggering_item = items[triggering_item_id] + try: + triggering_item = items[triggering_item_id] + except KeyError: + raise ItemDependencyError(_( + "'{item}' in bundle '{bundle}' has a reverse trigger (triggered_by) " + "on '{dep}', which doesn't exist" + ).format( + item=item.id, + bundle=item.bundle.name, + dep=triggering_item_id, + )) if triggering_item.id.startswith("bundle:"): # bundle items bundle_name = triggering_item.id.split(":")[1] for actual_triggering_item in items.values(): @@ -459,7 +469,17 @@ else: triggering_item.triggers.append(item.id) for preceded_item_id in item.precedes: - preceded_item = items[preceded_item_id] + try: + preceded_item = items[preceded_item_id] + except KeyError: + raise ItemDependencyError(_( + "'{item}' in bundle '{bundle}' has a reverse trigger (precedes) " + "on '{dep}', which doesn't exist" + ).format( + item=item.id, + bundle=item.bundle.name, + dep=preceded_item_id, + )) if preceded_item.id.startswith("bundle:"): # bundle items bundle_name = preceded_item.id.split(":")[1] for actual_preceded_item in items.values(): diff --git a/bundlewrap/items/pkg_opkg.py b/bundlewrap/items/pkg_opkg.py new file mode 100644 index 0000000..8e9b773 --- /dev/null +++ b/bundlewrap/items/pkg_opkg.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from pipes import quote + +from bundlewrap.items.pkg import Pkg + + +class OpkgPkg(Pkg): + """ + A package installed by opkg. + """ + BUNDLE_ATTRIBUTE_NAME = "pkg_opkg" + ITEM_TYPE_NAME = "pkg_opkg" + + @classmethod + def block_concurrent(cls, node_os, node_os_version): + return ["pkg_opkg"] + + def pkg_all_installed(self): + result = self.node.run("opkg list-installed") + for line in result.stdout.decode('utf-8').strip().split("\n"): + if line: + yield "{}:{}".format(self.ITEM_TYPE_NAME, line.split()[0]) + + def pkg_install(self): + self.node.run("opkg install {}".format(quote(self.name)), may_fail=True) + + def pkg_installed(self): + result = self.node.run( + "opkg status {} | grep ^Status: | grep installed".format(quote(self.name)), + may_fail=True, + ) + return result.return_code == 0 + + def pkg_remove(self): + self.node.run("opkg remove {}".format(quote(self.name)), may_fail=True) diff --git a/bundlewrap/items/users.py b/bundlewrap/items/users.py index c70467f..7aecba5 100644 --- a/bundlewrap/items/users.py +++ b/bundlewrap/items/users.py @@ -244,6 +244,11 @@ rounds=8, # default rounds for OpenBSD accounts salt=_DEFAULT_BCRYPT_SALT if salt is None else salt, ) + elif attributes.get('hash_method') == 'md5': + attributes['password_hash'] = hash_method.encrypt( + force_text(attributes['password']), + salt=_DEFAULT_SALT if salt is None else salt, + ) else: attributes['password_hash'] = hash_method.encrypt( force_text(attributes['password']), diff --git a/bundlewrap/node.py b/bundlewrap/node.py index cbaa46b..ffeeec3 100644 --- a/bundlewrap/node.py +++ b/bundlewrap/node.py @@ -328,6 +328,7 @@ 'amazonlinux', 'arch', 'opensuse', + 'openwrt', 'gentoo', 'linux', ) + \ diff --git a/bundlewrap/secrets.py b/bundlewrap/secrets.py index 4c05f67..9a2618c 100644 --- a/bundlewrap/secrets.py +++ b/bundlewrap/secrets.py @@ -75,6 +75,7 @@ def __init__(self, repo): self.repo = repo self.keys = self._load_keys() + self._call_log = {} def _decrypt(self, cryptotext=None, key='encrypt'): """ @@ -311,6 +312,8 @@ def human_password_for( self, identifier, digits=2, key='generate', per_word=3, words=4, ): + self._call_log.setdefault(identifier, 0) + self._call_log[identifier] += 1 return Fault( self._generate_human_password, identifier=identifier, @@ -321,6 +324,8 @@ ) def password_for(self, identifier, key='generate', length=32, symbols=False): + self._call_log.setdefault(identifier, 0) + self._call_log[identifier] += 1 return Fault( self._generate_password, identifier=identifier, diff --git a/bundlewrap/utils/ui.py b/bundlewrap/utils/ui.py index e245b30..e7d78b5 100644 --- a/bundlewrap/utils/ui.py +++ b/bundlewrap/utils/ui.py @@ -384,6 +384,8 @@ except ProcessLookupError: pass self._clear_last_job() + if TTY: + write_to_stream(STDOUT_WRITER, SHOW_CURSOR) _exit(1) else: if SHUTDOWN_EVENT_SOFT.wait(0.1): diff --git a/debian/changelog b/debian/changelog index 851b8ef..d33d9a6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +bundlewrap (3.1.0-1) unstable; urgency=medium + + * New upstream release + + -- Jonathan Carter Tue, 10 Oct 2017 14:39:46 +0200 + bundlewrap (3.0.3-1) unstable; urgency=medium * New upstream release diff --git a/docs/content/items/pkg_opkg.md b/docs/content/items/pkg_opkg.md new file mode 100644 index 0000000..2bf829c --- /dev/null +++ b/docs/content/items/pkg_opkg.md @@ -0,0 +1,24 @@ +# opkg package items + +Handles packages installed by `opkg` on OpenWRT/LEDE. + + pkg_opkg = { + "foopkg": { + "installed": True, # default + }, + "bar": { + "installed": False, + }, + } + +

+ +# Attribute reference + +See also: [The list of generic builtin item attributes](../repo/items.py.md#builtin-item-attributes) + +
+ +## installed + +`True` when the package is expected to be present on the system; `False` if it should be removed. diff --git a/docs/content/repo/items.py.md b/docs/content/repo/items.py.md index e3b5a72..a2eddb3 100644 --- a/docs/content/repo/items.py.md +++ b/docs/content/repo/items.py.md @@ -42,6 +42,7 @@ groupgroupsManages groups by wrapping groupadd, groupmod and groupdel pkg_aptpkg_aptInstalls and removes packages with APT pkg_dnfpkg_dnfInstalls and removes packages with dnf +pkg_opkgpkg_opkgInstalls and removes packages with opkg pkg_pacmanpkg_pacmanInstalls and removes packages with pacman pkg_pippkg_pipInstalls and removes Python packages with pip pkg_snappkg_snapInstalls and removes packages with snap diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index d00bf27..fc20ce9 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -40,6 +40,7 @@ - group: items/group.md - pkg_apt: items/pkg_apt.md - pkg_dnf: items/pkg_dnf.md + - pkg_opkg: items/pkg_opkg.md - pkg_pacman: items/pkg_pacman.md - pkg_pip: items/pkg_pip.md - pkg_snap: items/pkg_snap.md diff --git a/setup.py b/setup.py index 8884010..da34fb2 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="bundlewrap", - version="3.0.3", + version="3.1.0", description="Config management with Python", long_description=( "By allowing for easy and low-overhead config management, BundleWrap fills the gap between complex deployments using Chef or Puppet and old school system administration over SSH.\n" diff --git a/tests/integration/bw_run.py b/tests/integration/bw_run.py new file mode 100644 index 0000000..b09f8d5 --- /dev/null +++ b/tests/integration/bw_run.py @@ -0,0 +1,31 @@ +from bundlewrap.utils.testing import host_os, make_repo, run + + +def test_run_ok(tmpdir): + make_repo( + tmpdir, + nodes={ + "localhost": { + 'os': host_os(), + }, + }, + ) + stdout, stderr, rcode = run("BW_TABLE_STYLE=grep bw run localhost true", path=str(tmpdir)) + assert rcode == 0 + assert b"localhost\t0" in stdout + assert stderr == b"" + + +def test_run_fail(tmpdir): + make_repo( + tmpdir, + nodes={ + "localhost": { + 'os': host_os(), + }, + }, + ) + stdout, stderr, rcode = run("BW_TABLE_STYLE=grep bw run localhost false", path=str(tmpdir)) + assert rcode == 0 + assert b"localhost\t1" in stdout + assert stderr == b"" diff --git a/tests/integration/bw_test.py b/tests/integration/bw_test.py index 3059a3e..437a25c 100644 --- a/tests/integration/bw_test.py +++ b/tests/integration/bw_test.py @@ -584,3 +584,54 @@ }, ) assert run("bw test -I", path=str(tmpdir))[2] == 1 + + +def test_secret_identifier_only_once(tmpdir): + make_repo( + tmpdir, + nodes={ + "node1": { + 'bundles': ["bundle1"], + }, + }, + bundles={ + "bundle1": { + 'files': { + "/test": { + 'content': "${repo.vault.password_for('testing')}", + 'content_type': 'mako', + }, + }, + }, + }, + ) + assert run("bw test -s ''", path=str(tmpdir))[2] == 1 + assert run("bw test -s 'test'", path=str(tmpdir))[2] == 0 + assert run("bw test -s 'test,foo'", path=str(tmpdir))[2] == 0 + + +def test_secret_identifier_twice(tmpdir): + make_repo( + tmpdir, + nodes={ + "node1": { + 'bundles': ["bundle1"], + }, + "node2": { + 'bundles': ["bundle1"], + }, + }, + bundles={ + "bundle1": { + 'files': { + "/test": { + 'content': "${repo.vault.password_for('testing')}", + 'content_type': 'mako', + }, + }, + }, + }, + ) + assert run("bw test -s ''", path=str(tmpdir))[2] == 0 + assert run("bw test -s 'test'", path=str(tmpdir))[2] == 0 + assert run("bw test -s 'test,foo'", path=str(tmpdir))[2] == 0