diff --git a/CHANGELOG.md b/CHANGELOG.md index 2336f91..dcbe830 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +# 4.1.0 + +2020-07-27 + +* added `bw test --quiet` +* `apply_start` hook can now raise GracefulApplyException +* performance improvements in metadata generation +* improved reporting of persistent metadata KeyErrors +* clashing metadata keys are now allowed for equal values +* git_deploy: fixed attempted shallow clones over HTTP +* k8s: improved handling of absent `apiVersion` +* fixed `cascade_skip` not affecting recursively skipped items +* fixed `bw metadata -b -k` +* fixed metadata reactors seeing their own previous results +* fixed SCM information being returned as bytes + + # 4.0.0 2020-06-22 diff --git a/bundlewrap/__init__.py b/bundlewrap/__init__.py index 2a92389..81d28df 100644 --- a/bundlewrap/__init__.py +++ b/bundlewrap/__init__.py @@ -1,2 +1,2 @@ -VERSION = (4, 0, 0) +VERSION = (4, 1, 0) VERSION_STRING = ".".join([str(v) for v in VERSION]) diff --git a/bundlewrap/cmdline/apply.py b/bundlewrap/cmdline/apply.py index 410c514..389bf5f 100644 --- a/bundlewrap/cmdline/apply.py +++ b/bundlewrap/cmdline/apply.py @@ -30,12 +30,19 @@ io.progress_set_total(count_items(pending_nodes)) - repo.hooks.apply_start( - repo, - args['targets'], - target_nodes, - interactive=args['interactive'], - ) + try: + repo.hooks.apply_start( + repo, + args['targets'], + target_nodes, + interactive=args['interactive'], + ) + except GracefulApplyException as exc: + io.stderr(_("{x} apply aborted by hook ({reason})").format( + reason=str(exc) or _("no reason given"), + x=red("!!!"), + )) + exit(1) start_time = datetime.now() results = [] diff --git a/bundlewrap/cmdline/metadata.py b/bundlewrap/cmdline/metadata.py index 57a52b2..2b7808f 100644 --- a/bundlewrap/cmdline/metadata.py +++ b/bundlewrap/cmdline/metadata.py @@ -138,10 +138,13 @@ table = [[bold(_("path")), bold(_("source"))], ROW_SEPARATOR] for path, blamed in sorted(node.metadata_blame.items()): joined_path = "/".join(path) - for key_path in key_paths: - if joined_path.startswith(key_path): - table.append([joined_path, ", ".join(blamed)]) - break + if key_paths: + for key_path in key_paths: + if _list_starts_with(path, key_path): + table.append([joined_path, ", ".join(blamed)]) + break + else: + table.append([joined_path, ", ".join(blamed)]) page_lines(render_table(table)) else: metadata = deepcopy_metadata(node.metadata) diff --git a/bundlewrap/cmdline/parser.py b/bundlewrap/cmdline/parser.py index 7c1048b..3dce336 100644 --- a/bundlewrap/cmdline/parser.py +++ b/bundlewrap/cmdline/parser.py @@ -531,7 +531,7 @@ ) # bw metadata - help_metadata = ("View a JSON representation of a node's metadata (defaults blue, reactors green, groups yellow, node red) or a table of selected metadata keys from multiple nodes") + help_metadata = ("View a JSON representation of a node's metadata (defaults blue, reactors green, groups yellow, node red, uncolored if mixed-source) or a table of selected metadata keys from multiple nodes") parser_metadata = subparsers.add_parser( "metadata", description=help_metadata, @@ -906,6 +906,13 @@ help=_("check for bundles not assigned to any node"), ) parser_test.add_argument( + "-q", + "--quiet", + action='store_true', + dest='quiet', + help=_("don't show successful tests"), + ) + parser_test.add_argument( "-S", "--subgroup-loops", action='store_true', diff --git a/bundlewrap/cmdline/test.py b/bundlewrap/cmdline/test.py index 10674dc..5da4f2e 100644 --- a/bundlewrap/cmdline/test.py +++ b/bundlewrap/cmdline/test.py @@ -12,7 +12,7 @@ from ..utils.ui import io, QUIT_EVENT -def test_items(nodes, ignore_missing_faults): +def test_items(nodes, ignore_missing_faults, quiet): io.progress_set_total(count_items(nodes)) for node in nodes: if QUIT_EVENT.is_set(): @@ -60,12 +60,13 @@ if item.id.count(":") < 2: # don't count canned actions io.progress_advance() - io.stdout("{x} {node} {bundle} {item}".format( - bundle=bold(item.bundle.name), - item=item.id, - node=bold(node.name), - x=green("✓"), - )) + if not quiet: + io.stdout("{x} {node} {bundle} {item}".format( + bundle=bold(item.bundle.name), + item=item.id, + node=bold(node.name), + x=green("✓"), + )) if item_queue.items_with_deps and not QUIT_EVENT.is_set(): exception = ItemDependencyLoop(item_queue.items_with_deps) for line in explain_item_dependency_loop(exception, node.name): @@ -74,7 +75,7 @@ io.progress_set_total(0) -def test_subgroup_loops(repo): +def test_subgroup_loops(repo, quiet): checked_groups = [] for group in repo.groups: if QUIT_EVENT.is_set(): @@ -83,19 +84,21 @@ continue with io.job(_("{group} checking for subgroup loops").format(group=bold(group.name))): checked_groups.extend(group.subgroups) # the subgroups property has the check built in - io.stdout(_("{x} {group} has no subgroup loops").format( - x=green("✓"), - group=bold(group.name), - )) - - -def test_metadata_conflicts(node): + if not quiet: + io.stdout(_("{x} {group} has no subgroup loops").format( + x=green("✓"), + group=bold(group.name), + )) + + +def test_metadata_conflicts(node, quiet): with io.job(_("{node} checking for metadata conflicts").format(node=bold(node.name))): check_for_metadata_conflicts(node) - io.stdout(_("{x} {node} has no metadata conflicts").format( - x=green("✓"), - node=bold(node.name), - )) + if not quiet: + io.stdout(_("{x} {node} has no metadata conflicts").format( + x=green("✓"), + node=bold(node.name), + )) def test_orphaned_bundles(repo): @@ -132,7 +135,7 @@ exit(1) -def test_determinism_config(repo, nodes, iterations): +def test_determinism_config(repo, nodes, iterations, quiet): """ Generate configuration a couple of times for every node and see if anything changes between iterations @@ -167,13 +170,14 @@ exit(1) io.progress_advance() io.progress_set_total(0) - io.stdout(_("{x} Configuration remained the same after being generated {n} times").format( - n=iterations, - x=green("✓"), - )) - - -def test_determinism_metadata(repo, nodes, iterations): + if not quiet: + io.stdout(_("{x} Configuration remained the same after being generated {n} times").format( + n=iterations, + x=green("✓"), + )) + + +def test_determinism_metadata(repo, nodes, iterations, quiet): """ Generate metadata a couple of times for every node and see if anything changes between iterations @@ -208,10 +212,11 @@ exit(1) io.progress_advance() io.progress_set_total(0) - io.stdout(_("{x} Metadata remained the same after being generated {n} times").format( - n=iterations, - x=green("✓"), - )) + if not quiet: + io.stdout(_("{x} Metadata remained the same after being generated {n} times").format( + n=iterations, + x=green("✓"), + )) def bw_test(repo, args): @@ -244,7 +249,7 @@ args['subgroup_loops'] = True if args['subgroup_loops'] and not QUIT_EVENT.is_set(): - test_subgroup_loops(repo) + test_subgroup_loops(repo, args['quiet']) if args['empty_groups'] and not QUIT_EVENT.is_set(): test_empty_groups(repo) @@ -257,18 +262,18 @@ for node in nodes: if QUIT_EVENT.is_set(): break - test_metadata_conflicts(node) + test_metadata_conflicts(node, args['quiet']) io.progress_advance() io.progress_set_total(0) if args['items']: - test_items(nodes, args['ignore_missing_faults']) + test_items(nodes, args['ignore_missing_faults'], args['quiet']) if args['determinism_metadata'] > 1 and not QUIT_EVENT.is_set(): - test_determinism_metadata(repo, nodes, args['determinism_metadata']) + test_determinism_metadata(repo, nodes, args['determinism_metadata'], args['quiet']) if args['determinism_config'] > 1 and not QUIT_EVENT.is_set(): - test_determinism_config(repo, nodes, args['determinism_config']) + test_determinism_config(repo, nodes, args['determinism_config'], args['quiet']) if args['hooks_node'] and not QUIT_EVENT.is_set(): io.progress_set_total(len(nodes)) diff --git a/bundlewrap/deps.py b/bundlewrap/deps.py index e9d460c..29fd4c9 100644 --- a/bundlewrap/deps.py +++ b/bundlewrap/deps.py @@ -7,6 +7,7 @@ class DummyItem: bundle = None + cascade_skip = True triggered = False def __init__(self, *args, **kwargs): @@ -668,9 +669,12 @@ all_recursively_removed_items = [] for removed_item in removed_items: - items, recursively_removed_items = \ - remove_item_dependents(items, removed_item, skipped=skipped) - all_recursively_removed_items += recursively_removed_items + if removed_item.cascade_skip: + items, recursively_removed_items = \ + remove_item_dependents(items, removed_item, skipped=skipped) + all_recursively_removed_items += recursively_removed_items + else: + items = remove_dep_from_items(items, removed_item.id) return (items, removed_items + all_recursively_removed_items) diff --git a/bundlewrap/exceptions.py b/bundlewrap/exceptions.py index 4207b9c..930a5f0 100644 --- a/bundlewrap/exceptions.py +++ b/bundlewrap/exceptions.py @@ -107,6 +107,13 @@ pass +class MetadataPersistentKeyError(RepositoryError): + """ + Raised when metadata reactors keep raising KeyErrors indefinitely. + """ + pass + + class MissingRepoDependency(RepositoryError): """ Raised when a dependency from requirements.txt is missing. diff --git a/bundlewrap/items/git_deploy.py b/bundlewrap/items/git_deploy.py index 7ac7ce2..5f3d48d 100644 --- a/bundlewrap/items/git_deploy.py +++ b/bundlewrap/items/git_deploy.py @@ -36,7 +36,7 @@ Returns the path to the directory. """ tmpdir = mkdtemp() - if is_ref(rev): + if is_ref(rev) and not remote_url.startswith('http'): git_cmdline = ["clone", "--bare", "--depth", "1", "--no-single-branch", remote_url, "."] else: git_cmdline = ["clone", "--bare", remote_url, "."] diff --git a/bundlewrap/items/kubernetes.py b/bundlewrap/items/kubernetes.py index 9ea840d..bb172d7 100644 --- a/bundlewrap/items/kubernetes.py +++ b/bundlewrap/items/kubernetes.py @@ -133,7 +133,7 @@ user_manifest, ) - if merged_manifest['apiVersion'] is None: + if merged_manifest.get('apiVersion') is None: raise BundleError(_( "{item} from bundle '{bundle}' needs an apiVersion in its manifest" ).format(item=self.id, bundle=self.bundle.name)) diff --git a/bundlewrap/metadata.py b/bundlewrap/metadata.py index a21e484..3085af2 100644 --- a/bundlewrap/metadata.py +++ b/bundlewrap/metadata.py @@ -104,44 +104,49 @@ TYPE_SET = 2 TYPE_OTHER = 3 - def paths_with_types(d): + def paths_with_values_and_types(d): for path in map_dict_keys(d): value = value_at_key_path(d, path) if isinstance(value, dict): - yield path, TYPE_DICT + yield path, value, TYPE_DICT elif isinstance(value, set): - yield path, TYPE_SET + yield path, value, TYPE_SET else: - yield path, TYPE_OTHER + yield path, value, TYPE_OTHER for prefix in ("metadata_defaults:", "metadata_reactor:"): paths = {} - for identifier, layer in node._metadata_stack._layers.items(): - if identifier.startswith(prefix): - for path, current_type in paths_with_types(layer): - try: - prev_type, prev_identifier = paths[path] - except KeyError: - paths[path] = current_type, identifier - else: - if ( - prev_type == TYPE_DICT - and current_type == TYPE_DICT - ): - pass - elif ( - prev_type == TYPE_SET - and current_type == TYPE_SET - ): - pass + for partition in node._metadata_stack._partitions: + for identifier, layer in partition.items(): + if identifier.startswith(prefix): + for path, value, current_type in paths_with_values_and_types(layer): + try: + prev_type, prev_identifier, prev_value = paths[path] + except KeyError: + paths[path] = current_type, identifier, value else: - raise ValueError(_( - "{a} and {b} are clashing over this key path: {path}" - ).format( - a=identifier, - b=prev_identifier, - path="/".join(path), - )) + if ( + prev_type == TYPE_DICT + and current_type == TYPE_DICT + ): + pass + elif ( + prev_type == TYPE_SET + and current_type == TYPE_SET + ): + pass + elif value != prev_value: + raise ValueError(_( + "{node}: {a} and {b} are clashing over this key path: {path} " + "(\"{val_a}\" vs. \"{val_b}\")" + ).format( + a=identifier, + b=prev_identifier, + node=node.name, + path="/".join(path), + val_a=value, + val_b=prev_value, + )) def check_for_metadata_conflicts_between_groups(node): diff --git a/bundlewrap/metagen.py b/bundlewrap/metagen.py new file mode 100644 index 0000000..89b2971 --- /dev/null +++ b/bundlewrap/metagen.py @@ -0,0 +1,365 @@ +from collections import Counter +from os import environ +from traceback import TracebackException + +from .exceptions import MetadataPersistentKeyError +from .metadata import DoNotRunAgain +from .node import _flatten_group_hierarchy +from .utils import randomize_order +from .utils.ui import io, QUIT_EVENT +from .utils.metastack import Metastack +from .utils.text import bold, mark_for_translation as _, red + + +MAX_METADATA_ITERATIONS = int(environ.get("BW_MAX_METADATA_ITERATIONS", "5000")) + + +class MetadataGenerator: + # are we currently executing a reactor? + __in_a_reactor = False + + def __reset(self): + # reactors that raise DoNotRunAgain + self.__do_not_run_again = set() + # reactors that raised KeyErrors (and which ones) + self.__keyerrors = {} + # a Metastack for every node + self.__metastacks = {} + # mapping each node to all nodes that depend on it + self.__node_deps = {} + # A node is 'stable' when all its reactors return unchanged + # metadata, except for those reactors that look at other nodes. + # This dict maps node names to True/False indicating stable status. + self.__node_stable = {} + # nodes we encountered as a dependency through partial_metadata, + # but haven't run yet + self.__nodes_that_never_ran = set() + # nodes whose dependencies changed and that have to rerun their + # reactors depending on those nodes + self.__triggered_nodes = set() + # nodes we already did initial processing on + self.__nodes_that_ran_at_least_once = set() + # how often we called reactors + self.__reactors_run = 0 + # how often each reactor changed + self.__reactor_changes = {} + # tracks which reactors on a node have look at other nodes + # through partial_metadata + self.__reactors_with_deps = {} + + def _metadata_for_node(self, node_name, blame=False, stack=False): + """ + Returns full or partial metadata for this node. This is the + primary entrypoint accessed from node.metadata. + + Partial metadata may only be requested from inside a metadata + reactor. + + If necessary, this method will build complete metadata for this + node and all related nodes. Related meaning nodes that this node + depends on in one of its metadata reactors. + """ + if self.__in_a_reactor: + if node_name in self._node_metadata_complete: + # We already completed metadata for this node, but partial must + # return a Metastack, so we build a single-layered one just for + # the interface. + metastack = Metastack() + metastack._set_layer( + 0, + "flattened", + self._node_metadata_complete[node_name], + ) + return metastack + else: + self.__partial_metadata_accessed_for.add(node_name) + return self.__metastacks.setdefault(node_name, Metastack()) + + if blame or stack: + # cannot return cached result here, force rebuild + try: + del self._node_metadata_complete[node_name] + except KeyError: + pass + + try: + return self._node_metadata_complete[node_name] + except KeyError: + pass + + # Different worker threads might request metadata at the same time. + + with self._node_metadata_lock: + try: + # maybe our metadata got completed while waiting for the lock + return self._node_metadata_complete[node_name] + except KeyError: + pass + + self.__build_node_metadata(node_name) + + # now that we have completed all metadata for this + # node and all related nodes, copy that data over + # to the complete dict + for some_node_name in self.__nodes_that_ran_at_least_once: + self._node_metadata_complete[some_node_name] = \ + self.__metastacks[some_node_name]._as_dict() + + if blame: + blame_result = self.__metastacks[node_name]._as_blame() + elif stack: + stack_result = self.__metastacks[node_name] + + # reset temporary vars (this isn't strictly necessary, but might + # free up some memory and avoid confusion) + self.__reset() + + if blame: + return blame_result + elif stack: + return stack_result + else: + return self._node_metadata_complete[node_name] + + def __build_node_metadata(self, initial_node_name): + self.__reset() + self.__nodes_that_never_ran.add(initial_node_name) + + iterations = 0 + while not QUIT_EVENT.is_set(): + iterations += 1 + if iterations > MAX_METADATA_ITERATIONS: + top_changers = Counter(self.__reactor_changes).most_common(25) + msg = _( + "MAX_METADATA_ITERATIONS({m}) exceeded, " + "likely an infinite loop between flip-flopping metadata reactors.\n" + "These are the reactors that changed most often:\n\n" + ).format(m=MAX_METADATA_ITERATIONS) + for reactor, count in top_changers: + msg += f" {count}\t{reactor[0]}\t{reactor[1]}\n" + raise RuntimeError(msg) + + io.debug(f"metadata iteration #{iterations}") + + jobmsg = _("{b} ({i} iterations, {n} nodes, {r} reactors, {e} runs)").format( + b=bold(_("running metadata reactors")), + i=iterations, + n=len(self.__nodes_that_never_ran) + len(self.__nodes_that_ran_at_least_once), + r=len(self.__reactor_changes), + e=self.__reactors_run, + ) + with io.job(jobmsg): + try: + node_name = self.__nodes_that_never_ran.pop() + except KeyError: + pass + else: + self.__nodes_that_ran_at_least_once.add(node_name) + self.__initial_run_for_node(node_name) + continue + + # at this point, we have run all relevant nodes at least once + + # if we have any triggered nodes from below, run their reactors + # with deps to see if they become unstable + + try: + node_name = self.__triggered_nodes.pop() + except KeyError: + pass + else: + io.debug(f"triggered metadata run for {node_name}") + self.__run_reactors( + self.get_node(node_name), + with_deps=True, + without_deps=False, + ) + continue + + # now (re)stabilize all nodes + + encountered_unstable_node = False + for node, stable in self.__node_stable.items(): + if stable: + continue + self.__run_reactors(node, with_deps=False, without_deps=True) + if self.__node_stable[node]: + io.debug(f"metadata stabilized for {node_name}") + else: + io.debug(f"metadata remains unstable for {node_name}") + encountered_unstable_node = True + if encountered_unstable_node: + # start over until everything is stable + continue + + # at this point, all nodes should be stable except for their reactors with deps + + encountered_unstable_node = False + for node in randomize_order(self.__node_stable.keys()): + self.__run_reactors(node, with_deps=True, without_deps=False) + if not self.__node_stable[node]: + encountered_unstable_node = True + if encountered_unstable_node: + # start over until everything is stable + continue + + # if we get here, we're done! + break + + if self.__keyerrors and not QUIT_EVENT.is_set(): + msg = _( + "These metadata reactors raised a KeyError " + "even after all other reactors were done:" + ) + for source, exc in sorted(self.__keyerrors.items()): + node_name, reactor = source + msg += f"\n\n {node_name} {reactor}\n\n" + for line in TracebackException.from_exception(exc).format(): + msg += " " + line + raise MetadataPersistentKeyError(msg) + + def __initial_run_for_node(self, node_name): + io.debug(f"initial metadata run for {node_name}") + node = self.get_node(node_name) + self.__metastacks[node_name] = Metastack() + + # randomize order to increase chance of exposing clashing defaults + for defaults_name, defaults in randomize_order(node.metadata_defaults): + self.__metastacks[node_name]._set_layer( + 2, + defaults_name, + defaults, + ) + self.__metastacks[node_name]._cache_partition(2) + + group_order = _flatten_group_hierarchy(node.groups) + for group_name in group_order: + self.__metastacks[node_name]._set_layer( + 0, + "group:{}".format(group_name), + self.get_group(group_name)._attributes.get('metadata', {}), + ) + + self.__metastacks[node_name]._set_layer( + 0, + "node:{}".format(node_name), + node._attributes.get('metadata', {}), + ) + self.__metastacks[node_name]._cache_partition(0) + + self.__reactors_with_deps[node_name] = set() + # run all reactors once to get started + self.__run_reactors(node, with_deps=True, without_deps=True) + + def __run_reactors(self, node, with_deps=True, without_deps=True): + any_reactor_changed = False + + for depsonly in (True, False): + if depsonly and not with_deps: + # skip reactors with deps + continue + if not depsonly and not without_deps: + # skip reactors without deps + continue + # TODO ideally, we should run the least-run reactors first + for reactor_name, reactor in randomize_order(node.metadata_reactors): + if ( + (depsonly and reactor_name not in self.__reactors_with_deps[node.name]) or + (not depsonly and reactor_name in self.__reactors_with_deps[node.name]) + ): + # this if makes sure we run reactors with deps first + continue + reactor_changed, deps = self.__run_reactor(node.name, reactor_name, reactor) + io.debug(f"{node.name}:{reactor_name} changed={reactor_changed} deps={deps}") + if reactor_changed: + any_reactor_changed = True + if deps: + # record that this reactor has dependencies + self.__reactors_with_deps[node.name].add(reactor_name) + # we could also remove this marker if we end up without + # deps again in future iterations, but that is too + # unlikely and the housekeeping cost too great + for required_node_name in deps: + if required_node_name not in self.__nodes_that_ran_at_least_once: + # we found a node that we didn't need until now + self.__nodes_that_never_ran.add(required_node_name) + # this is so we know the current node needs to be run + # again if the required node changes + self.__node_deps.setdefault(required_node_name, set()) + self.__node_deps[required_node_name].add(node.name) + + if any_reactor_changed: + # something changed on this node, mark all dependent nodes as unstable + for required_node_name in self.__node_deps.get(node.name, set()): + io.debug(f"{node.name} triggering metadata rerun on {required_node_name}") + self.__triggered_nodes.add(required_node_name) + + if with_deps and any_reactor_changed: + self.__node_stable[node] = False + elif without_deps: + self.__node_stable[node] = not any_reactor_changed + + def __run_reactor(self, node_name, reactor_name, reactor): + if (node_name, reactor_name) in self.__do_not_run_again: + return False, set() + self.__partial_metadata_accessed_for = set() + self.__reactors_run += 1 + self.__reactor_changes.setdefault((node_name, reactor_name), 0) + # make sure the reactor doesn't react to its own output + old_metadata = self.__metastacks[node_name]._pop_layer(1, reactor_name) + self.__in_a_reactor = True + try: + new_metadata = reactor(self.__metastacks[node_name]) + except KeyError as exc: + self.__keyerrors[(node_name, reactor_name)] = exc + return False, self.__partial_metadata_accessed_for + except DoNotRunAgain: + self.__do_not_run_again.add((node_name, reactor_name)) + # clear any previously stored exception + try: + del self.__keyerrors[(node_name, reactor_name)] + except KeyError: + pass + return False, set() + except Exception as exc: + io.stderr(_( + "{x} Exception while executing metadata reactor " + "{metaproc} for node {node}:" + ).format( + x=red("!!!"), + metaproc=reactor_name, + node=node_name, + )) + raise exc + finally: + self.__in_a_reactor = False + + # reactor terminated normally, clear any previously stored exception + try: + del self.__keyerrors[(node_name, reactor_name)] + except KeyError: + pass + + try: + self.__metastacks[node_name]._set_layer( + 1, + reactor_name, + new_metadata, + ) + except TypeError as exc: + # TODO catch validation errors better + io.stderr(_( + "{x} Exception after executing metadata reactor " + "{metaproc} for node {node}:" + ).format( + x=red("!!!"), + metaproc=reactor_name, + node=node_name, + )) + raise exc + + changed = old_metadata != new_metadata + if changed: + self.__reactor_changes[(node_name, reactor_name)] += 1 + + return changed, self.__partial_metadata_accessed_for diff --git a/bundlewrap/node.py b/bundlewrap/node.py index 6ac2b4d..8e2e9c3 100644 --- a/bundlewrap/node.py +++ b/bundlewrap/node.py @@ -677,19 +677,15 @@ @property def metadata(self): - """ - Returns full metadata for a node. MUST NOT be used from inside a - metadata processor. Use .partial_metadata instead. - """ - return self.repo._metadata_for_node(self.name, partial=False) + return self.repo._metadata_for_node(self.name) @property def metadata_blame(self): - return self.repo._metadata_for_node(self.name, partial=False, blame=True) + return self.repo._metadata_for_node(self.name, blame=True) @property def _metadata_stack(self): - return self.repo._metadata_for_node(self.name, partial=False, stack=True) + return self.repo._metadata_for_node(self.name, stack=True) def metadata_get(self, path, default=NO_DEFAULT): if not isinstance(path, (tuple, list)): @@ -729,15 +725,9 @@ @property def partial_metadata(self): """ - Only to be used from inside metadata reactors. Can't use the - normal .metadata there because it might deadlock when nodes - have interdependent metadata. - - It's OK for metadata reactors to work with partial metadata - because they will be fed all metadata updates until no more - changes are made by any metadata reactor. + Deprecated, remove in 5.0.0 """ - return self.repo._metadata_for_node(self.name, partial=True) + return self.metadata def run(self, command, data_stdin=None, may_fail=False, log_output=False): assert self.os in self.OS_FAMILY_UNIX diff --git a/bundlewrap/repo.py b/bundlewrap/repo.py index 32bd1aa..4a2ff08 100644 --- a/bundlewrap/repo.py +++ b/bundlewrap/repo.py @@ -1,6 +1,6 @@ from importlib.machinery import SourceFileLoader from inspect import isabstract -from os import environ, listdir, mkdir, walk +from os import listdir, mkdir, walk from os.path import abspath, dirname, isdir, isfile, join from threading import Lock @@ -17,21 +17,19 @@ RepositoryError, ) from .group import Group -from .metadata import DoNotRunAgain -from .node import _flatten_group_hierarchy, Node +from .metagen import MetadataGenerator +from .node import Node from .secrets import FILENAME_SECRETS, generate_initial_secrets_cfg, SecretProxy from .utils import ( cached_property, error_context, get_file_contents, names, - randomize_order, ) from .utils.scm import get_git_branch, get_git_clean, get_rev from .utils.dicts import hash_statedict -from .utils.metastack import Metastack -from .utils.text import bold, mark_for_translation as _, red, validate_name -from .utils.ui import io, QUIT_EVENT +from .utils.text import mark_for_translation as _, red, validate_name +from .utils.ui import io DIRNAME_BUNDLES = "bundles" DIRNAME_DATA = "data" @@ -41,7 +39,6 @@ FILENAME_GROUPS = "groups.py" FILENAME_NODES = "nodes.py" FILENAME_REQUIREMENTS = "requirements.txt" -MAX_METADATA_ITERATIONS = int(environ.get("BW_MAX_METADATA_ITERATIONS", "100")) HOOK_EVENTS = ( 'action_run_end', @@ -179,7 +176,7 @@ return self.__module_cache[attrname] -class Repository: +class Repository(MetadataGenerator): def __init__(self, repo_path=None): if repo_path is None: self.path = "/dev/null" @@ -193,6 +190,8 @@ self.node_dict = {} self._get_all_attr_code_cache = {} self._get_all_attr_result_cache = {} + + # required by MetadataGenerator self._node_metadata_complete = {} self._node_metadata_lock = Lock() @@ -459,250 +458,6 @@ Returns a list of nodes in the given group. """ return self.nodes_in_all_groups([group_name]) - - def _metadata_for_node(self, node_name, partial=False, blame=False, stack=False): - """ - Returns full or partial metadata for this node. - - Partial metadata may only be requested from inside a metadata - reactor. - - If necessary, this method will build complete metadata for this - node and all related nodes. Related meaning nodes that this node - depends on in one of its metadata processors. - """ - if partial: - if node_name in self._node_metadata_complete: - # We already completed metadata for this node, but partial must - # return a Metastack, so we build a single-layered one just for - # the interface. - metastack = Metastack() - metastack._set_layer( - "flattened", - self._node_metadata_complete[node_name], - ) - return metastack - else: - # Return the WIP Metastack or an empty one if we didn't start - # yet. - self._nodes_we_need_metadata_for.add(node_name) - return self._metastacks.setdefault(node_name, Metastack()) - - if blame or stack: - # cannot return cached result here, force rebuild - try: - del self._node_metadata_complete[node_name] - except KeyError: - pass - - try: - return self._node_metadata_complete[node_name] - except KeyError: - pass - - # Different worker threads might request metadata at the same time. - # This creates problems for the following variables: - # - # self._metastacks - # self._nodes_we_need_metadata_for - # - # Chaos would ensue if we allowed multiple instances of - # _build_node_metadata() running in parallel, messing with these - # vars. So we use a lock and reset the vars before and after. - - with self._node_metadata_lock: - try: - # maybe our metadata got completed while waiting for the lock - return self._node_metadata_complete[node_name] - except KeyError: - pass - - # set up temporary vars - self._metastacks = {} - self._nodes_we_need_metadata_for = {node_name} - - self._build_node_metadata() - - io.debug("completed metadata for {} nodes".format( - len(self._nodes_we_need_metadata_for), - )) - # now that we have completed all metadata for this - # node and all related nodes, copy that data over - # to the complete dict - for node_name in self._nodes_we_need_metadata_for: - self._node_metadata_complete[node_name] = \ - self._metastacks[node_name]._as_dict() - - if blame: - blame_result = self._metastacks[node_name]._as_blame() - elif stack: - stack_result = self._metastacks[node_name] - - # reset temporary vars (this isn't strictly necessary, but might - # free up some memory and avoid confusion) - self._metastacks = {} - self._nodes_we_need_metadata_for = set() - - if blame: - return blame_result - elif stack: - return stack_result - else: - return self._node_metadata_complete[node_name] - - def _build_node_metadata(self): - """ - Builds complete metadata for all nodes that appear in - self._nodes_we_need_metadata_for. - """ - # Prevents us from reassembling static metadata needlessly and - # helps us detect nodes pulled into self._nodes_we_need_metadata_for - # by node.partial_metadata. - nodes_with_completed_static_metadata = set() - # these reactors have indicated that they do not need to be run again - do_not_run_again = set() - # these reactors have raised KeyErrors - keyerrors = {} - # loop detection - iterations = 0 - reactors_that_changed_something_in_last_iteration = set() - - while not QUIT_EVENT.is_set(): - iterations += 1 - if iterations > MAX_METADATA_ITERATIONS: - reactors = "" - for node, reactor in sorted(reactors_that_changed_something_in_last_iteration): - reactors += node + " " + reactor + "\n" - raise ValueError(_( - "Infinite loop detected between these metadata reactors:\n" - ) + reactors) - - # First, get the static metadata out of the way - for node_name in list(self._nodes_we_need_metadata_for): - if QUIT_EVENT.is_set(): - break - node = self.get_node(node_name) - # check if static metadata for this node is already done - if node_name in nodes_with_completed_static_metadata: - continue - self._metastacks[node_name] = Metastack() - - with io.job(_("{node} adding metadata defaults").format(node=bold(node.name))): - # randomize order to increase chance of exposing clashing defaults - for defaults_name, defaults in randomize_order(node.metadata_defaults): - self._metastacks[node_name]._set_layer( - defaults_name, - defaults, - ) - - with io.job(_("{node} adding group metadata").format(node=bold(node.name))): - group_order = _flatten_group_hierarchy(node.groups) - for group_name in group_order: - self._metastacks[node_name]._set_layer( - "group:{}".format(group_name), - self.get_group(group_name)._attributes.get('metadata', {}), - ) - - with io.job(_("{node} adding node metadata").format(node=bold(node.name))): - self._metastacks[node_name]._set_layer( - "node:{}".format(node_name), - node._attributes.get('metadata', {}), - ) - - # This will ensure node/group metadata and defaults are - # skipped over in future iterations. - nodes_with_completed_static_metadata.add(node_name) - - # Now for the interesting part: We run all metadata reactors - # until none of them return changed metadata anymore. - any_reactor_returned_changed_metadata = False - reactors_that_changed_something_in_last_iteration = set() - - # randomize order to increase chance of exposing unintended - # non-deterministic effects of execution order - for node_name in randomize_order(self._nodes_we_need_metadata_for): - if QUIT_EVENT.is_set(): - break - node = self.get_node(node_name) - - with io.job(_("{node} running metadata reactors").format(node=bold(node.name))): - for reactor_name, reactor in randomize_order(node.metadata_reactors): - if (node_name, reactor_name) in do_not_run_again: - continue - try: - new_metadata = reactor(self._metastacks[node.name]) - except KeyError as exc: - keyerrors[(node_name, reactor_name)] = exc - except DoNotRunAgain: - do_not_run_again.add((node_name, reactor_name)) - except Exception as exc: - io.stderr(_( - "{x} Exception while executing metadata reactor " - "{metaproc} for node {node}:" - ).format( - x=red("!!!"), - metaproc=reactor_name, - node=node.name, - )) - raise exc - else: - # reactor terminated normally, clear any previously stored exception - try: - del keyerrors[(node_name, reactor_name)] - except KeyError: - pass - - try: - this_changed = self._metastacks[node_name]._set_layer( - reactor_name, - new_metadata, - ) - except TypeError as exc: - # TODO catch validation errors better - io.stderr(_( - "{x} Exception after executing metadata reactor " - "{metaproc} for node {node}:" - ).format( - x=red("!!!"), - metaproc=reactor_name, - node=node.name, - )) - raise exc - if this_changed: - reactors_that_changed_something_in_last_iteration.add( - (node_name, reactor_name), - ) - any_reactor_returned_changed_metadata = True - - if not any_reactor_returned_changed_metadata: - if nodes_with_completed_static_metadata != self._nodes_we_need_metadata_for: - # During metadata reactor execution, partial metadata may - # have been requested for nodes we did not previously - # consider. We still need to make sure to generate static - # metadata for these new nodes, as that may trigger - # additional results from metadata reactors. - continue - else: - # Now that we're done, re-sort static metadata to - # overrule reactors. - for node_name, metastack in self._metastacks.items(): - for identifier in list(metastack._layers.keys()): - if ( - identifier.startswith("group:") or - identifier.startswith("node:") - ): - metastack._layers[identifier] = metastack._layers.pop(identifier) - break - - if keyerrors: - reactors = "" - for source, exc in keyerrors.items(): - node_name, reactor = source - reactors += "{} {} {}\n".format(node_name, reactor, exc) - raise ValueError(_( - "These metadata reactors raised a KeyError " - "even after all other reactors were done:\n" - ) + reactors) def metadata_hash(self): repo_dict = {} diff --git a/bundlewrap/utils/dicts.py b/bundlewrap/utils/dicts.py index af95eb2..c8d0d81 100644 --- a/bundlewrap/utils/dicts.py +++ b/bundlewrap/utils/dicts.py @@ -432,4 +432,9 @@ if not path: return dict_obj else: - return value_at_key_path(dict_obj[path[0]], path[1:]) + nested_dict = dict_obj[path[0]] + remaining_path = path[1:] + if remaining_path and not isinstance(nested_dict, dict): + raise KeyError("/".join(path)) + else: + return value_at_key_path(nested_dict, remaining_path) diff --git a/bundlewrap/utils/metastack.py b/bundlewrap/utils/metastack.py index 9f2d112..08633b5 100644 --- a/bundlewrap/utils/metastack.py +++ b/bundlewrap/utils/metastack.py @@ -1,9 +1,12 @@ from collections import OrderedDict from sys import version_info -from ..metadata import deepcopy_metadata, validate_metadata, value_at_key_path +from ..metadata import METADATA_TYPES, deepcopy_metadata, validate_metadata, value_at_key_path from . import NO_DEFAULT -from .dicts import map_dict_keys, merge_dict +from .dicts import ATOMIC_TYPES, map_dict_keys, merge_dict + + +UNMERGEABLE = tuple(METADATA_TYPES) + tuple(ATOMIC_TYPES.values()) class Metastack: @@ -15,11 +18,13 @@ in their ability to revise their own layer each time they are run. """ def __init__(self): - # We rely heavily on insertion order in this dict. - if version_info < (3, 7): - self._layers = OrderedDict() - else: - self._layers = {} + self._partitions = ( + # We rely heavily on insertion order in these dicts. + {} if version_info >= (3, 7) else OrderedDict(), # node/groups + {} if version_info >= (3, 7) else OrderedDict(), # reactors + {} if version_info >= (3, 7) else OrderedDict(), # defaults + ) + self._cached_partitions = {} def get(self, path, default=NO_DEFAULT): """ @@ -42,18 +47,24 @@ result = None undef = True - for layer in self._layers.values(): - try: - value = value_at_key_path(layer, path) - except KeyError: - pass - else: - if undef: - # First time we see anything. - result = {'data': value} - undef = False + for part_index, partition in enumerate(self._partitions): + # prefer cached partitions if available + partition = self._cached_partitions.get(part_index, partition) + for layer in reversed(list(partition.values())): + try: + value = value_at_key_path(layer, path) + except KeyError: + pass else: - result = merge_dict(result, {'data': value}) + if undef: + # First time we see anything. If we can't merge + # it anyway, then return early. + if isinstance(value, UNMERGEABLE): + return value + result = {'data': value} + undef = False + else: + result = merge_dict({'data': value}, result) if undef: if default != NO_DEFAULT: @@ -63,11 +74,19 @@ else: return deepcopy_metadata(result['data']) - def _as_dict(self): + def _as_dict(self, partitions=None): final_dict = {} - for layer in self._layers.values(): - final_dict = merge_dict(final_dict, layer) + if partitions is None: + partitions = tuple(range(len(self._partitions))) + else: + partitions = sorted(partitions) + + for part_index in partitions: + # prefer cached partitions if available + partition = self._cached_partitions.get(part_index, self._partitions[part_index]) + for layer in reversed(list(partition.values())): + final_dict = merge_dict(layer, final_dict) return final_dict @@ -75,19 +94,27 @@ keymap = map_dict_keys(self._as_dict()) blame = {} for path in keymap: - for identifier, layer in self._layers.items(): - try: - value_at_key_path(layer, path) - except KeyError: - pass - else: - blame.setdefault(path, []).append(identifier) + for partition in self._partitions: + for identifier, layer in partition.items(): + try: + value_at_key_path(layer, path) + except KeyError: + pass + else: + blame.setdefault(path, []).append(identifier) return blame - def _set_layer(self, identifier, new_layer): - # Marked with an underscore because only the internal metadata - # reactor routing is supposed to call this method. + def _pop_layer(self, partition_index, identifier): + try: + return self._partitions[partition_index].pop(identifier) + except (KeyError, IndexError): + return {} + + def _set_layer(self, partition_index, identifier, new_layer): validate_metadata(new_layer) - changed = self._layers.get(identifier, {}) != new_layer - self._layers[identifier] = new_layer - return changed + self._partitions[partition_index][identifier] = new_layer + + def _cache_partition(self, partition_index): + self._cached_partitions[partition_index] = { + 'merged layers': self._as_dict(partitions=[partition_index]), + } diff --git a/bundlewrap/utils/scm.py b/bundlewrap/utils/scm.py index 8e71012..cefec09 100644 --- a/bundlewrap/utils/scm.py +++ b/bundlewrap/utils/scm.py @@ -10,7 +10,7 @@ "git rev-parse --abbrev-ref HEAD", shell=True, stderr=STDOUT, - ).strip() + ).decode().strip() except CalledProcessError: return None @@ -21,7 +21,7 @@ "git status --porcelain", shell=True, stderr=STDOUT, - ).strip()) + ).decode().strip()) except CalledProcessError: return None @@ -32,7 +32,7 @@ "bzr revno", shell=True, stderr=STDOUT, - ).strip() + ).decode().strip() except CalledProcessError: return None @@ -43,7 +43,7 @@ "git rev-parse HEAD", shell=True, stderr=STDOUT, - ).strip() + ).decode().strip() except CalledProcessError: return None @@ -54,7 +54,7 @@ "hg --debug id -i", shell=True, stderr=STDOUT, - ).strip().rstrip("+") + ).decode().strip().rstrip("+") except CalledProcessError: return None diff --git a/bundlewrap/utils/ui.py b/bundlewrap/utils/ui.py index 50563a8..31dfc01 100644 --- a/bundlewrap/utils/ui.py +++ b/bundlewrap/utils/ui.py @@ -1,12 +1,11 @@ from contextlib import contextmanager from datetime import datetime -import fcntl from functools import wraps from os import _exit, environ, getpid, kill from os.path import join from select import select +from shutil import get_terminal_size from signal import signal, SIG_DFL, SIGINT, SIGQUIT, SIGTERM -import struct from subprocess import PIPE, Popen import sys import termios @@ -102,15 +101,6 @@ yield c -def term_width(): - if not TTY: - return 0 - - fd = sys.stdout.fileno() - _, width = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, 'aaaa')) - return width - - def page_lines(lines): """ View the given list of Unicode lines in a pager (e.g. `less`). @@ -125,7 +115,10 @@ env=env, stdin=PIPE, ) - pager.stdin.write("\n".join(lines).encode('utf-8')) + try: + pager.stdin.write("\n".join(lines).encode('utf-8')) + except BrokenPipeError: + pass pager.stdin.close() pager.communicate() write_to_stream(STDOUT_WRITER, HIDE_CURSOR) @@ -423,7 +416,7 @@ progress_text = "{:.1f}% ".format(progress * 100) line += bold(progress_text) visible_length += len(progress_text) - line += self.jobs[-1][:term_width() - 1 - visible_length] + line += self.jobs[-1][:get_terminal_size().columns - 1 - visible_length] write_to_stream(STDOUT_WRITER, line) self._status_line_present = True diff --git a/debian/changelog b/debian/changelog index f32e54e..345787e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,14 +1,21 @@ +bundlewrap (4.1.0-1) unstable; urgency=medium + + * New upstream release + * Add python3-tomlkit to build-depends (needed for tests) + + -- Jonathan Carter Sat, 08 Aug 2020 12:24:31 +0200 + bundlewrap (4.0.0-1) unstable; urgency=medium * New upstream release - * Add python3-tomlkit to build-dependencies (needed for tests) + * Add python3-tomlkit to build-depends (needed for tests) -- Jonathan Carter Tue, 30 Jun 2020 20:38:26 +0200 bundlewrap (3.10.0-1) unstable; urgency=medium * New upstream release - * Add python3-jinja2 and python3-mako to build-depends (for tests) + * Add python3-jinja2 and python3-mako to build-depends (needed for tests) -- Jonathan Carter Mon, 18 May 2020 11:39:02 +0200 diff --git a/debian/control b/debian/control index e2bfcb0..42c7dd4 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,8 @@ python3-mako, python3-minimal, python3-requests, - python3-setuptools + python3-setuptools, + python3-tomlkit Standards-Version: 4.5.0 Rules-Requires-Root: no Homepage: http://bundlewrap.org/ diff --git a/docs/content/repo/groups.py.md b/docs/content/repo/groups.py.md index a7144aa..8951357 100644 --- a/docs/content/repo/groups.py.md +++ b/docs/content/repo/groups.py.md @@ -93,7 +93,7 @@
BundleWrap will consider group hierarchy when merging metadata. For example, it is possible to define a default nameserver for the "eu" group and then override it for the "eu.frankfurt" subgroup. The catch is that this only works for groups that are connected through a subgroup hierarchy. Independent groups will have their metadata merged in an undefined order. bw test will report conflicting metadata in independent groups as a metadata collision.
-
Also see the documentation for node.metadata for more information.
+
Also see the documentation for node.metadata and metadata.py for more information.

diff --git a/docs/content/repo/hooks.md b/docs/content/repo/hooks.md index d9101a3..0d90400 100644 --- a/docs/content/repo/hooks.md +++ b/docs/content/repo/hooks.md @@ -64,6 +64,13 @@ `interactive` Indicates whether the apply is interactive or not. +To abort the entire apply operation: + +``` +from bundlewrap.exceptions import GracefulApplyException +raise GracefulApplyException("reason goes here") +``` + --- **`apply_end(repo, target, nodes, duration=None, **kwargs)`** diff --git a/docs/content/repo/metadata.py.md b/docs/content/repo/metadata.py.md index 3ecb026..f6d5468 100644 --- a/docs/content/repo/metadata.py.md +++ b/docs/content/repo/metadata.py.md @@ -16,7 +16,7 @@ ## Reactors -So let's look at reactors next. Metadata reactors are functions that take the metadata generated so far as their single argument. You must then return a new dictionary with any metadata you wish to have added: +So let's look at reactors next. Metadata reactors are functions that take the metadata generated for this node so far as their single argument. You must then return a new dictionary with any metadata you wish to have added: @metadata_reactor def bar(metadata): @@ -30,7 +30,14 @@ While node and group metadata and metadata defaults will always be available to reactors, you should not rely on that for the simple reason that you may one day move some metadata from those static sources into another reactor, which may be run later. Thus you may need to wait for some iterations before that data shows up in `metadata`. Note that BundleWrap will catch any `KeyError`s raised in metadata reactors and only report them if they don't go away after all other relevant reactors are done. -To avoid deadlocks when accessing *other* nodes' metadata from within a metadata reactor, use `other_node.partial_metadata` instead of `other_node.metadata`. For the same reason, always use the `metadata` parameter to access the current node's metadata, never `node.metadata`. +You can also access other nodes' metadata: + + @metadata_reactor + def baz(metadata): + frob = set() + for n in repo.nodes: + frob.add(n.metadata.get('sizzle')) + return {'frob': frob} ### DoNotRunAgain @@ -46,3 +53,21 @@
For your convenience, you can access repo, node, metadata_reactors, and DoNotRunAgain in metadata.py without importing them.
+ + +## Priority + +For atomic ("primitive") data types like `int` or `bool`: + +1. Nodes +2. Groups +3. Reactors +4. Defaults + +Node metadata wins over group metadata, groups win over reactors, reactors win over defaults. + +This also applies to type conflicts: For example, specifying a boolean flag in node metadata will win over a list returned by a metadata reactor. (You should probably avoid situations like this entirely.) + +Set-like data types will be merged recursively. + +
Also see the documentation for node.metadata and group.metadata for more information.
diff --git a/docs/content/repo/nodes.py.md b/docs/content/repo/nodes.py.md index 1af42c5..b02b550 100644 --- a/docs/content/repo/nodes.py.md +++ b/docs/content/repo/nodes.py.md @@ -119,7 +119,7 @@ * `None` * `bundlewrap.utils.Fault` -
Also see the documentation for group.metadata for more information.
+
Also see the documentation for group.metadata and metadata.py for more information.

diff --git a/setup.py b/setup.py index bca9912..79e0d62 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="bundlewrap", - version="4.0.0", + version="4.1.0", description="Config management with Python", long_description=( "By allowing for easy and low-overhead config management, BundleWrap fills the gap between complex deployments using Chef or Puppet and old school system administration over SSH.\n" diff --git a/tests/integration/bw_metadata.py b/tests/integration/bw_metadata.py index 1da6031..0dea406 100644 --- a/tests/integration/bw_metadata.py +++ b/tests/integration/bw_metadata.py @@ -385,28 +385,27 @@ f.write( """ @metadata_reactor -def foo(metadata): - bar_ran = metadata.get('bar_ran', False) - if not bar_ran: - return {'foo_ran': True} - else: - return {'foo': metadata.get('bar'), 'foo_ran': True} - - -@metadata_reactor -def bar(metadata): - foo_ran = metadata.get('foo_ran', False) - if not foo_ran: - return {'bar_ran': False} - else: - return {'bar': 47, 'bar_ran': True} -""") - stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) - assert loads(stdout.decode()) == { - "bar": 47, - "bar_ran": True, - "foo": 47, - "foo_ran": True, +def one(metadata): + return {'one': True} + +@metadata_reactor +def two(metadata): + return {'two': metadata.get('one')} + +@metadata_reactor +def three(metadata): + return {'three': metadata.get('two')} + +@metadata_reactor +def four(metadata): + return {'four': metadata.get('three')} +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert loads(stdout.decode()) == { + "one": True, + "two": True, + "three": True, + "four": True, } assert stderr == b"" assert rcode == 0 @@ -427,11 +426,113 @@ """ @metadata_reactor def plusone(metadata): - return {'foo': metadata.get('foo', 0) + 1 } + return {'foo': metadata.get('bar', 0) + 1 } @metadata_reactor def plustwo(metadata): - return {'foo': metadata.get('foo', 0) + 2 } + return {'bar': metadata.get('foo', 0) + 2 } """) stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) assert rcode == 1 + + +def test_metadatapy_no_self_react(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +""" +@metadata_reactor +def reactor1(metadata): + assert not metadata.get('broken', False) + return {'broken': True} + +@metadata_reactor +def reactor2(metadata): + # just to make sure reactor1 runs again + return {'again': True} +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert loads(stdout.decode()) == { + "broken": True, + "again": True, + } + + +def test_own_node_metadata(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {'number': 47}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +""" +@metadata_reactor +def reactor1(metadata): + return {'plusone': node.metadata.get('number') + 1} +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert loads(stdout.decode()) == { + "number": 47, + "plusone": 48, + } + + +def test_other_node_metadata(tmpdir): + make_repo( + tmpdir, + bundles={"test": {}}, + nodes={ + "node1": { + 'bundles': ["test"], + 'metadata': {'number': 47}, + }, + "node2": { + 'bundles': ["test"], + 'metadata': {'number': 42}, + }, + "node3": { + 'bundles': ["test"], + 'metadata': {'number': 23}, + }, + }, + ) + with open(join(str(tmpdir), "bundles", "test", "metadata.py"), 'w') as f: + f.write( +""" +@metadata_reactor +def reactor1(metadata): + numbers = set() + for n in repo.nodes: + if n != node: + numbers.add(n.metadata.get('number')) + return {'other_numbers': numbers} +""") + stdout, stderr, rcode = run("bw metadata node1", path=str(tmpdir)) + assert loads(stdout.decode()) == { + "number": 47, + "other_numbers": [23, 42], + } + stdout, stderr, rcode = run("bw metadata node2", path=str(tmpdir)) + assert loads(stdout.decode()) == { + "number": 42, + "other_numbers": [23, 47], + } + stdout, stderr, rcode = run("bw metadata node3", path=str(tmpdir)) + assert loads(stdout.decode()) == { + "number": 23, + "other_numbers": [42, 47], + } diff --git a/tests/unit/metastack.py b/tests/unit/metastack.py index a911ffe..42a5bd4 100644 --- a/tests/unit/metastack.py +++ b/tests/unit/metastack.py @@ -11,20 +11,20 @@ def test_has_no_subpath(): stack = Metastack() - stack._set_layer('base', {'something': {'in': {}}}) + stack._set_layer(0, 'base', {'something': {'in': {}}}) with raises(KeyError): stack.get('something/in/a/path') def test_get_top(): stack = Metastack() - stack._set_layer('base', {'something': 123}) + stack._set_layer(0, 'base', {'something': 123}) assert stack.get('something') == 123 def test_get_subpath(): stack = Metastack() - stack._set_layer('base', {'something': {'in': {'a': 'subpath'}}}) + stack._set_layer(0, 'base', {'something': {'in': {'a': 'subpath'}}}) assert stack.get('something/in/a', None) == 'subpath' @@ -35,51 +35,51 @@ def test_get_default_with_base(): stack = Metastack() - stack._set_layer('', {'foo': 'bar'}) + stack._set_layer(0, '', {'foo': 'bar'}) assert stack.get('something', 123) == 123 def test_get_default_with_overlay(): stack = Metastack() - stack._set_layer('base', {'foo': 'bar'}) - stack._set_layer('overlay', {'baz': 'boing'}) + stack._set_layer(0, 'base', {'foo': 'bar'}) + stack._set_layer(0, 'overlay', {'baz': 'boing'}) assert stack.get('something', 123) == 123 def test_overlay_value(): stack = Metastack() - stack._set_layer('base', {'something': {'a_list': [1, 2], 'a_value': 5}}) - stack._set_layer('overlay', {'something': {'a_value': 10}}) + stack._set_layer(0, 'base', {'something': {'a_list': [1, 2], 'a_value': 5}}) + stack._set_layer(0, 'overlay', {'something': {'a_value': 10}}) assert stack.get('something/a_value', None) == 10 def test_merge_lists(): stack = Metastack() - stack._set_layer('base', {'something': {'a_list': [1, 2], 'a_value': 5}}) - stack._set_layer('overlay', {'something': {'a_list': [3]}}) + stack._set_layer(0, 'base', {'something': {'a_list': [1, 2], 'a_value': 5}}) + stack._set_layer(0, 'overlay', {'something': {'a_list': [3]}}) assert sorted(stack.get('something/a_list', None)) == sorted([1, 2, 3]) def test_merge_sets(): stack = Metastack() - stack._set_layer('base', {'something': {'a_set': {1, 2}, 'a_value': 5}}) - stack._set_layer('overlay', {'something': {'a_set': {3}}}) + stack._set_layer(0, 'base', {'something': {'a_set': {1, 2}, 'a_value': 5}}) + stack._set_layer(0, 'overlay', {'something': {'a_set': {3}}}) assert stack.get('something/a_set', None) == {1, 2, 3} def test_overlay_value_multi_layers(): stack = Metastack() - stack._set_layer('base', {'something': {'a_list': [1, 2], 'a_value': 5}}) - stack._set_layer('overlay', {'something': {'a_value': 10}}) - stack._set_layer('unrelated', {'something': {'another_value': 10}}) + stack._set_layer(0, 'base', {'something': {'a_list': [1, 2], 'a_value': 5}}) + stack._set_layer(0, 'overlay', {'something': {'a_value': 10}}) + stack._set_layer(0, 'unrelated', {'something': {'another_value': 10}}) assert stack.get('something/a_value', None) == 10 def test_merge_lists_multi_layers(): stack = Metastack() - stack._set_layer('base', {'something': {'a_list': [1, 2], 'a_value': 5}}) - stack._set_layer('overlay', {'something': {'a_list': [3]}}) - stack._set_layer('unrelated', {'something': {'another_value': 10}}) + stack._set_layer(0, 'base', {'something': {'a_list': [1, 2], 'a_value': 5}}) + stack._set_layer(0, 'overlay', {'something': {'a_list': [3]}}) + stack._set_layer(0, 'unrelated', {'something': {'another_value': 10}}) # Objects in Metastacks are frozen. This converts lists to tuples. # Unlike set and frozenset, list and tuple doesn't naturally support @@ -97,53 +97,53 @@ def test_merge_sets_multi_layers(): stack = Metastack() - stack._set_layer('base', {'something': {'a_set': {1, 2}, 'a_value': 5}}) - stack._set_layer('overlay', {'something': {'a_set': {3}}}) - stack._set_layer('unrelated', {'something': {'another_value': 10}}) + stack._set_layer(0, 'base', {'something': {'a_set': {1, 2}, 'a_value': 5}}) + stack._set_layer(0, 'overlay', {'something': {'a_set': {3}}}) + stack._set_layer(0, 'unrelated', {'something': {'another_value': 10}}) assert stack.get('something/a_set', None) == {1, 2, 3} def test_merge_lists_with_empty_layer(): stack = Metastack() - stack._set_layer('base', {'something': {'a_list': [1, 2], 'a_value': 5}}) - stack._set_layer('overlay1', {'something': {'a_list': []}}) - stack._set_layer('overlay2', {'something': {'a_list': [3]}}) + stack._set_layer(0, 'base', {'something': {'a_list': [1, 2], 'a_value': 5}}) + stack._set_layer(0, 'overlay1', {'something': {'a_list': []}}) + stack._set_layer(0, 'overlay2', {'something': {'a_list': [3]}}) assert sorted(stack.get('something/a_list', None)) == sorted([1, 2, 3]) def test_merge_sets_with_empty_layer(): stack = Metastack() - stack._set_layer('base', {'something': {'a_set': {1, 2}, 'a_value': 5}}) - stack._set_layer('overlay1', {'something': {'a_set': set()}}) - stack._set_layer('overlay2', {'something': {'a_set': {3}}}) + stack._set_layer(0, 'base', {'something': {'a_set': {1, 2}, 'a_value': 5}}) + stack._set_layer(0, 'overlay1', {'something': {'a_set': set()}}) + stack._set_layer(0, 'overlay2', {'something': {'a_set': {3}}}) assert stack.get('something/a_set', None) == {1, 2, 3} def test_merge_lists_with_multiple_used_layers(): stack = Metastack() - stack._set_layer('base', {'something': {'a_list': [1, 2], 'a_value': 5}}) - stack._set_layer('overlay1', {'something': {'a_list': [3]}}) - stack._set_layer('overlay2', {'something': {'a_list': [4]}}) - stack._set_layer('overlay3', {'something': {'a_list': [6, 5]}}) + stack._set_layer(0, 'base', {'something': {'a_list': [1, 2], 'a_value': 5}}) + stack._set_layer(0, 'overlay1', {'something': {'a_list': [3]}}) + stack._set_layer(0, 'overlay2', {'something': {'a_list': [4]}}) + stack._set_layer(0, 'overlay3', {'something': {'a_list': [6, 5]}}) assert sorted(stack.get('something/a_list', None)) == sorted([1, 2, 3, 4, 5, 6]) def test_merge_sets_with_multiple_used_layers(): stack = Metastack() - stack._set_layer('base', {'something': {'a_set': {1, 2}, 'a_value': 5}}) - stack._set_layer('overlay1', {'something': {'a_set': {3}}}) - stack._set_layer('overlay2', {'something': {'a_set': {4}}}) - stack._set_layer('overlay3', {'something': {'a_set': {6, 5}}}) + stack._set_layer(0, 'base', {'something': {'a_set': {1, 2}, 'a_value': 5}}) + stack._set_layer(0, 'overlay1', {'something': {'a_set': {3}}}) + stack._set_layer(0, 'overlay2', {'something': {'a_set': {4}}}) + stack._set_layer(0, 'overlay3', {'something': {'a_set': {6, 5}}}) assert stack.get('something/a_set', None) == {1, 2, 3, 4, 5, 6} def test_merge_dicts(): stack = Metastack() - stack._set_layer('overlay1', {'something': {'a_value': 3}}) - stack._set_layer('overlay2', {'something': {'another_value': 5}}) - stack._set_layer('overlay3', {'something': {'this': {'and': 'that'}}}) - stack._set_layer('overlay4', {'something': {'a_set': {1, 2}}}) - stack._set_layer('overlay5', {'something': {'a_set': {3, 4}}}) + stack._set_layer(0, 'overlay1', {'something': {'a_value': 3}}) + stack._set_layer(0, 'overlay2', {'something': {'another_value': 5}}) + stack._set_layer(0, 'overlay3', {'something': {'this': {'and': 'that'}}}) + stack._set_layer(0, 'overlay4', {'something': {'a_set': {1, 2}}}) + stack._set_layer(0, 'overlay5', {'something': {'a_set': {3, 4}}}) assert stack.get('something', None) == { 'a_set': {1, 2, 3, 4}, 'a_value': 3, @@ -156,20 +156,20 @@ def test_requesting_empty_path(): stack = Metastack() - stack._set_layer('base', {'foo': {'bar': 'baz'}}) + stack._set_layer(0, 'base', {'foo': {'bar': 'baz'}}) assert stack.get('', 'default') == 'default' def test_update_layer_for_new_value(): stack = Metastack() - stack._set_layer('base', {'foo': 'bar'}) - - stack._set_layer('overlay', {'something': 123}) + stack._set_layer(0, 'base', {'foo': 'bar'}) + + stack._set_layer(0, 'overlay', {'something': 123}) assert stack.get('foo', None) == 'bar' assert stack.get('boing', 'default') == 'default' assert stack.get('something', None) == 123 - stack._set_layer('overlay', {'something': 456}) + stack._set_layer(0, 'overlay', {'something': 456}) assert stack.get('foo', None) == 'bar' assert stack.get('boing', 'default') == 'default' assert stack.get('something', None) == 456 @@ -177,7 +177,7 @@ def test_deepcopy(): stack = Metastack() - stack._set_layer('base', {'foo': {'bar': {1, 2, 3}}}) + stack._set_layer(0, 'base', {'foo': {'bar': {1, 2, 3}}}) foo = stack.get('foo', None) foo['bar'].add(4) assert stack.get('foo/bar') == {1, 2, 3} @@ -187,33 +187,33 @@ def test_atomic_in_base(): stack = Metastack() - stack._set_layer('base', {'list': atomic([1, 2, 3])}) - stack._set_layer('overlay', {'list': [4]}) + stack._set_layer(0, 'base', {'list': atomic([1, 2, 3])}) + stack._set_layer(0, 'overlay', {'list': [4]}) assert list(stack.get('list', None)) == [4] def test_atomic_in_layer(): stack = Metastack() - stack._set_layer('base', {'list': [1, 2, 3]}) - stack._set_layer('overlay', {'list': atomic([4])}) + stack._set_layer(0, 'base', {'list': [1, 2, 3]}) + stack._set_layer(0, 'overlay', {'list': atomic([4])}) assert list(stack.get('list', None)) == [4] -def test_set_layer_return_code(): - stack = Metastack() - ret = stack._set_layer('overlay', {'foo': 'bar'}) - assert ret is True - ret = stack._set_layer('overlay', {'foo': 'bar'}) - assert ret is False - ret = stack._set_layer('overlay', {'foo': 'baz'}) - assert ret is True - ret = stack._set_layer('overlay', {'foo': 'baz', 'bar': 1}) - assert ret is True +def test_pop_layer(): + stack = Metastack() + stack._set_layer(0, 'overlay', {'foo': 'bar'}) + stack._set_layer(0, 'overlay', {'foo': 'baz'}) + assert stack._pop_layer(0, 'overlay') == {'foo': 'baz'} + with raises(KeyError): + stack.get('foo') + assert stack._pop_layer(0, 'overlay') == {} + assert stack._pop_layer(0, 'unknown') == {} + assert stack._pop_layer(47, 'unknown') == {} def test_as_dict(): stack = Metastack() - stack._set_layer('base', { + stack._set_layer(0, 'base', { 'bool': True, 'bytes': b'howdy', 'dict': {'1': 2}, @@ -224,9 +224,9 @@ 'str': 'howdy', 'tuple': (1, 2), }) - stack._set_layer('overlay1', {'int': 1000}) - stack._set_layer('overlay2', {'list': [2]}) - stack._set_layer('overlay3', {'new_element': True}) + stack._set_layer(0, 'overlay1', {'int': 1000}) + stack._set_layer(0, 'overlay2', {'list': [2]}) + stack._set_layer(0, 'overlay3', {'new_element': True}) assert stack._as_dict() == { 'bool': True, 'bytes': b'howdy', @@ -243,9 +243,9 @@ def test_as_blame(): stack = Metastack() - stack._set_layer('base', {'something': {'a_list': [1, 2], 'a_value': 5}}) - stack._set_layer('overlay', {'something': {'a_list': [3]}}) - stack._set_layer('unrelated', {'something': {'another_value': 10}}) + stack._set_layer(0, 'base', {'something': {'a_list': [1, 2], 'a_value': 5}}) + stack._set_layer(0, 'overlay', {'something': {'a_list': [3]}}) + stack._set_layer(0, 'unrelated', {'something': {'another_value': 10}}) assert stack._as_blame() == { ('something',): ['base', 'overlay', 'unrelated'], ('something', 'a_list'): ['base', 'overlay'],