New upstream snapshot.
Debian Janitor
2 years ago
| 87 | 87 | |
| 88 | 88 | * ``DJES_WSGI_FILE``: path to the ``wsgi.py`` file for the django |
| 89 | 89 | project. If not None, the monitoring of environment configuration will |
| 90 | perform a ``touch`` of the file everytime the env_defaults are updated, so | |
| 90 | perform a ``touch`` of the file every time the env_defaults are updated, so | |
| 91 | 91 | that all processes consuming settings from ``django.conf`` can consume the |
| 92 | 92 | latest settings available as well |
| 93 | 93 | The path can be absolute or relative to the 'manage.py' file. |
| 0 | python-django-etcd-settings (0.1.13+dfsg-4) UNRELEASED; urgency=medium | |
| 0 | python-django-etcd-settings (0.1.15+git20171003.1.5a176a8-1) UNRELEASED; urgency=medium | |
| 1 | 1 | |
| 2 | 2 | [ Debian Janitor ] |
| 3 | 3 | * Set upstream metadata fields: Bug-Database, Bug-Submit, Repository, |
| 13 | 13 | * Apply multi-arch hints. |
| 14 | 14 | + python-django-etcd-settings-doc: Add Multi-Arch: foreign. |
| 15 | 15 | * Bump debhelper from deprecated 9 to 13. |
| 16 | * New upstream snapshot. | |
| 16 | 17 | |
| 17 | -- Debian Janitor <janitor@jelmer.uk> Sat, 02 May 2020 09:29:16 +0000 | |
| 18 | -- Debian Janitor <janitor@jelmer.uk> Mon, 15 Nov 2021 01:59:07 -0000 | |
| 18 | 19 | |
| 19 | 20 | python-django-etcd-settings (0.1.13+dfsg-3) unstable; urgency=medium |
| 20 | 21 | |
Binary diff not shown
| 3 | 3 | |
| 4 | 4 | At the command line:: |
| 5 | 5 | |
| 6 | $ pip install ssh://git@stash.kpnnl.local:7999/de/django-etcd-settings.git | |
| 6 | $ pip install django-etcd-settings |
| 0 | from .manager import EtcdConfigManager | |
| 1 | ||
| 2 | ||
| 3 | def get_overwrites(env, dev_params, etcd_details): | |
| 4 | overwrites = {} | |
| 5 | if etcd_details is not None: | |
| 6 | mgr = EtcdConfigManager(dev_params, **etcd_details) | |
| 7 | overwrites = mgr.get_env_defaults(env) | |
| 8 | else: | |
| 9 | overwrites = EtcdConfigManager.get_dev_params(dev_params) | |
| 10 | return overwrites |
| 0 | import json | |
| 1 | import logging | |
| 2 | import re | |
| 3 | import time | |
| 4 | from importlib import import_module | |
| 5 | from os import utime | |
| 6 | ||
| 7 | from etcd import Client, EtcdException, EtcdKeyNotFound | |
| 8 | ||
| 9 | from .utils import ( | |
| 10 | CustomJSONEncoder, attrs_to_dir, byteify, custom_json_decoder_hook, | |
| 11 | threaded, | |
| 12 | ) | |
| 13 | ||
| 14 | ||
| 15 | class EtcdConfigInvalidValueError(Exception): | |
| 16 | def __init__(self, key, raw_value, value_error): | |
| 17 | self.key = key | |
| 18 | self.raw_value = raw_value | |
| 19 | self.value_error = value_error | |
| 20 | super(EtcdConfigInvalidValueError, self).__init__( | |
| 21 | "Invalid value for key '{}'. Raising '{}', because of value: '{}'" | |
| 22 | .format(key, value_error, raw_value)) | |
| 23 | ||
| 24 | ||
| 25 | class EtcdClusterState(object): | |
| 26 | etcd_index = 0 | |
| 27 | ||
| 28 | ||
| 29 | class EtcdConfigManager(object): | |
| 30 | def __init__( | |
| 31 | self, dev_params=None, prefix='config', protocol='http', | |
| 32 | host='localhost', port=2379, username=None, password=None, | |
| 33 | long_polling_timeout=50, long_polling_safety_delay=5): | |
| 34 | self._client = Client( | |
| 35 | host=host, port=port, protocol=protocol, allow_redirect=True, | |
| 36 | username=username, password=password) | |
| 37 | # Overriding retries for urllib3.PoolManager.connection_pool_kw | |
| 38 | self._client.http.connection_pool_kw['retries'] = 0 | |
| 39 | self._base_config_path = prefix | |
| 40 | self._dev_params = dev_params | |
| 41 | self._base_config_set_path = "{}/extensions" \ | |
| 42 | .format(self._base_config_path) | |
| 43 | r = ('^(?P<path>{}/(?:extensions/)?' | |
| 44 | '(?P<envorset>[\w\-\.]+))/(?P<key>.+)$') | |
| 45 | self._key_regex = re.compile(r.format(self._base_config_path)) | |
| 46 | self.long_polling_timeout = long_polling_timeout | |
| 47 | self.long_polling_safety_delay = long_polling_safety_delay | |
| 48 | self._init_logger() | |
| 49 | ||
| 50 | def _init_logger(self): | |
| 51 | self.logger = logging.getLogger('etcd_config_manager') | |
| 52 | logger_console_handler = logging.StreamHandler() | |
| 53 | logger_console_handler.setLevel(logging.ERROR) | |
| 54 | self.logger.addHandler(logger_console_handler) | |
| 55 | ||
| 56 | def _env_defaults_path(self, env='test'): | |
| 57 | return "{}/{}".format(self._base_config_path, env) | |
| 58 | ||
| 59 | def _config_set_path(self, set_name): | |
| 60 | return "{}/{}".format(self._base_config_set_path, set_name) | |
| 61 | ||
| 62 | def _encode_config_key(self, k): | |
| 63 | return k.lower().replace('_', '/') | |
| 64 | ||
| 65 | def _decode_config_key(self, k): | |
| 66 | [env_or_set, key_path] = re.sub( | |
| 67 | self._key_regex, '\g<envorset>|\g<key>', k).split('|') | |
| 68 | return env_or_set, key_path.upper().replace('/', '_') | |
| 69 | ||
| 70 | def _encode_config_value(self, val): | |
| 71 | return json.dumps(val, cls=CustomJSONEncoder) | |
| 72 | ||
| 73 | def _decode_config_value(self, val): | |
| 74 | decoded = json.loads(val, object_hook=custom_json_decoder_hook) | |
| 75 | return byteify(decoded) | |
| 76 | ||
| 77 | def _process_response_set(self, rset, env_defaults=True): | |
| 78 | d = {} | |
| 79 | EtcdClusterState.etcd_index = rset.etcd_index | |
| 80 | for leaf in rset.leaves: | |
| 81 | try: | |
| 82 | config_set, key = self._decode_config_key(leaf.key) | |
| 83 | except ValueError: | |
| 84 | info = "An error occurred when processing an EtcdResponse" | |
| 85 | if not env_defaults: | |
| 86 | info += " (is '{}' a directory?)".format( | |
| 87 | self._base_config_set_path) | |
| 88 | self.logger.warning(info) | |
| 89 | else: | |
| 90 | if leaf.value is not None: | |
| 91 | try: | |
| 92 | value = self._decode_config_value(leaf.value) | |
| 93 | except ValueError as e: | |
| 94 | raise EtcdConfigInvalidValueError( | |
| 95 | leaf.key, leaf.value, e) | |
| 96 | ||
| 97 | if env_defaults: | |
| 98 | d[key] = value | |
| 99 | else: | |
| 100 | if config_set not in d: | |
| 101 | d[config_set] = {} | |
| 102 | d[config_set][key] = value | |
| 103 | return d | |
| 104 | ||
| 105 | @staticmethod | |
| 106 | def get_dev_params(mod): | |
| 107 | params = {} | |
| 108 | if mod: | |
| 109 | params = attrs_to_dir(import_module(mod)) | |
| 110 | return params | |
| 111 | ||
| 112 | def get_env_defaults(self, env): | |
| 113 | res = self._client.read( | |
| 114 | self._env_defaults_path(env), | |
| 115 | recursive=True) | |
| 116 | conf = self._process_response_set(res) | |
| 117 | conf.update(EtcdConfigManager.get_dev_params(self._dev_params)) | |
| 118 | return conf | |
| 119 | ||
| 120 | def get_config_sets(self): | |
| 121 | conf = {} | |
| 122 | try: | |
| 123 | res = self._client.read( | |
| 124 | self._base_config_set_path, | |
| 125 | recursive=True) | |
| 126 | conf = self._process_response_set(res, env_defaults=False) | |
| 127 | except EtcdKeyNotFound: | |
| 128 | self.logger.warning( | |
| 129 | "Unable to find config sets at '{}' (expected a dict)", | |
| 130 | self._base_config_set_path) | |
| 131 | return conf | |
| 132 | ||
| 133 | @threaded(daemon=True) | |
| 134 | def monitor_env_defaults( | |
| 135 | self, env, conf={}, wsgi_file=None, max_events=None): | |
| 136 | processed_events = 0 | |
| 137 | for event in self._watch( | |
| 138 | self._env_defaults_path(env), conf, wsgi_file, max_events): | |
| 139 | if event is not None: | |
| 140 | conf.update(self._process_response_set(event)) | |
| 141 | conf.update(EtcdConfigManager.get_dev_params(self._dev_params)) | |
| 142 | if wsgi_file: | |
| 143 | with open(wsgi_file, 'a'): | |
| 144 | utime(wsgi_file, None) | |
| 145 | processed_events += 1 | |
| 146 | return processed_events | |
| 147 | ||
| 148 | @threaded(daemon=True) | |
| 149 | def monitor_config_sets(self, conf={}, max_events=None): | |
| 150 | processed_events = 0 | |
| 151 | for event in self._watch( | |
| 152 | self._base_config_set_path, conf=conf, max_events=max_events): | |
| 153 | if event is not None: | |
| 154 | conf.update( | |
| 155 | self._process_response_set(event, env_defaults=False)) | |
| 156 | processed_events += 1 | |
| 157 | return processed_events | |
| 158 | ||
| 159 | def _watch(self, path, conf={}, wsgi_file=None, max_events=None): | |
| 160 | i = 0 | |
| 161 | while (max_events is None) or (i < max_events): | |
| 162 | try: | |
| 163 | i += 1 | |
| 164 | index = EtcdClusterState.etcd_index | |
| 165 | if index > 0: | |
| 166 | index = index + 1 | |
| 167 | res = self._client.watch( | |
| 168 | path, | |
| 169 | index=index, | |
| 170 | recursive=True, | |
| 171 | timeout=self.long_polling_timeout) | |
| 172 | else: | |
| 173 | res = self._client.read(path, recursive=True) | |
| 174 | yield res | |
| 175 | except Exception as e: | |
| 176 | if not (isinstance(e, EtcdException) | |
| 177 | and ('timed out' in str(e))): | |
| 178 | self.logger.error("Long Polling Error: {}".format(e)) | |
| 179 | time.sleep(self.long_polling_safety_delay) | |
| 180 | yield None | |
| 181 | ||
| 182 | def set_env_defaults(self, env, conf={}): | |
| 183 | path = self._env_defaults_path(env) | |
| 184 | errors = {} | |
| 185 | for k, v in conf.items(): | |
| 186 | if k.isupper(): | |
| 187 | try: | |
| 188 | encoded_key = self._encode_config_key(k) | |
| 189 | self._client.write( | |
| 190 | "{}/{}".format(path, encoded_key), | |
| 191 | self._encode_config_value(v)) | |
| 192 | except Exception as e: | |
| 193 | errors[k] = str(e) | |
| 194 | return errors | |
| 195 | ||
| 196 | def set_config_sets(self, config_sets={}): | |
| 197 | errors = {} | |
| 198 | for set_name, config_set in config_sets.items(): | |
| 199 | path = self._config_set_path(set_name) | |
| 200 | for k, v in config_set.items(): | |
| 201 | if k.isupper(): | |
| 202 | try: | |
| 203 | self._client.write( | |
| 204 | "{}/{}".format(path, self._encode_config_key(k)), | |
| 205 | self._encode_config_value(v)) | |
| 206 | except Exception as e: | |
| 207 | errors[k] = str(e) | |
| 208 | return errors |
| 2 | 2 | from importlib import import_module |
| 3 | 3 | |
| 4 | 4 | from django.conf import settings as django_settings |
| 5 | from etcd_config.manager import EtcdConfigManager | |
| 6 | from etcd_config.utils import attrs_to_dir | |
| 5 | 7 | |
| 6 | from .manager import EtcdConfigManager | |
| 7 | from .utils import ( | |
| 8 | attrs_to_dir, copy_if_mutable, dict_rec_update, find_project_root, | |
| 9 | ) | |
| 8 | from .utils import copy_if_mutable, dict_rec_update, find_project_root | |
| 10 | 9 | |
| 11 | 10 | |
| 12 | 11 | class EtcdSettingsProxy(object): |
| 0 | 0 | import copy |
| 1 | import datetime | |
| 2 | import json | |
| 3 | import logging | |
| 4 | 1 | import os |
| 5 | import sys | |
| 6 | 2 | from collections import Mapping |
| 7 | from functools import wraps | |
| 8 | from threading import Thread | |
| 9 | ||
| 10 | import six | |
| 11 | from dateutil.parser import parse as parse_date | |
| 12 | ||
| 13 | ||
| 14 | def attrs_to_dir(mod): | |
| 15 | data = {} | |
| 16 | for attr in dir(mod): | |
| 17 | if attr == attr.upper(): | |
| 18 | data[attr] = getattr(mod, attr) | |
| 19 | return data | |
| 20 | 3 | |
| 21 | 4 | |
| 22 | 5 | def dict_rec_update(d, u): |
| 29 | 12 | else: |
| 30 | 13 | d[k] = u[k] |
| 31 | 14 | return d |
| 32 | ||
| 33 | ||
| 34 | class Task(Thread): | |
| 35 | """ | |
| 36 | The Threaded object returned by the @threaded decorator below | |
| 37 | """ | |
| 38 | ||
| 39 | def __init__(self, method, *args, **kwargs): | |
| 40 | super(Task, self).__init__() | |
| 41 | self.method = method | |
| 42 | self.args = args | |
| 43 | self.kwargs = kwargs | |
| 44 | self._result = None | |
| 45 | self.__exc_info = None | |
| 46 | ||
| 47 | def run(self): | |
| 48 | try: | |
| 49 | self._result = self.method(*self.args, **self.kwargs) | |
| 50 | except: | |
| 51 | self.__exc_info = sys.exc_info() | |
| 52 | ||
| 53 | @property | |
| 54 | def result(self): | |
| 55 | self.join() | |
| 56 | if self.__exc_info is not None: | |
| 57 | six.reraise(*self.__exc_info) | |
| 58 | return self._result | |
| 59 | ||
| 60 | ||
| 61 | def threaded(function=None, daemon=False): | |
| 62 | ||
| 63 | def wrapper_factory(func): | |
| 64 | ||
| 65 | @wraps(func) | |
| 66 | def get_thread(*args, **kwargs): | |
| 67 | t = Task(func, *args, **kwargs) | |
| 68 | if daemon: | |
| 69 | t.daemon = True | |
| 70 | t.start() | |
| 71 | return t | |
| 72 | ||
| 73 | return get_thread | |
| 74 | ||
| 75 | if function: | |
| 76 | return wrapper_factory(function) | |
| 77 | else: | |
| 78 | return wrapper_factory | |
| 79 | ||
| 80 | ||
| 81 | class CustomJSONEncoder(json.JSONEncoder): | |
| 82 | ||
| 83 | custom_type_key = '_custom_type' | |
| 84 | custom_type_value_key = 'value' | |
| 85 | DECODERS = {'datetime': parse_date} | |
| 86 | ||
| 87 | def default(self, obj): | |
| 88 | if isinstance(obj, datetime.datetime): | |
| 89 | return {self.custom_type_key: 'datetime', | |
| 90 | self.custom_type_value_key: obj.isoformat()} | |
| 91 | else: | |
| 92 | return super(CustomJSONEncoder, self).default(obj) | |
| 93 | ||
| 94 | ||
| 95 | def custom_json_decoder_hook(obj): | |
| 96 | ||
| 97 | ct = obj.get(CustomJSONEncoder.custom_type_key, None) | |
| 98 | if ct is not None: | |
| 99 | value = obj.get(CustomJSONEncoder.custom_type_value_key) | |
| 100 | return CustomJSONEncoder.DECODERS[ct](value) | |
| 101 | else: | |
| 102 | return obj | |
| 103 | 15 | |
| 104 | 16 | |
| 105 | 17 | def find_project_root(root_indicator='manage.py', current=os.getcwd()): |
| 120 | 32 | if type(value) in (dict, list): |
| 121 | 33 | return copy.deepcopy(value) |
| 122 | 34 | return value |
| 123 | ||
| 124 | ||
| 125 | def byteify(input): | |
| 126 | if isinstance(input, dict): | |
| 127 | return {byteify(key): byteify(value) for key, value in input.items()} | |
| 128 | elif isinstance(input, list): | |
| 129 | return [byteify(element) for element in input] | |
| 130 | elif (sys.version_info.major == 2) and (isinstance(input, unicode)): | |
| 131 | return input.encode('utf-8') | |
| 132 | else: | |
| 133 | return input | |
| 134 | ||
| 135 | ||
| 136 | class IgnoreMaxEtcdRetries(logging.Filter): | |
| 137 | """ | |
| 138 | Skip etcd.client MaxRetryError on timeout | |
| 139 | """ | |
| 140 | ||
| 141 | def __init__(self, name='etcd.client'): | |
| 142 | super(IgnoreMaxEtcdRetries, self).__init__(name) | |
| 143 | ||
| 144 | def filter(self, record): | |
| 145 | msg = '{}'.format(record.args) | |
| 146 | return not ( | |
| 147 | 'MaxRetryError' in msg and | |
| 148 | 'Read timed out' in msg | |
| 149 | ) | |
| 0 | 0 | Django>=1.7.5 |
| 1 | python-etcd>=0.4.1 | |
| 2 | python-dateutil>=2.2 | |
| 3 | 1 | six>=1.10.0,<2.0.0 |
| 2 | etcd-config>=1.0.4 |
| 0 | from django.test import TestCase | |
| 1 | from etcd_settings.loader import get_overwrites | |
| 2 | ||
| 3 | ||
| 4 | class TestLoader(TestCase): | |
| 5 | ||
| 6 | def test_get_overwrites_without_etcd(self): | |
| 7 | overwrites = get_overwrites( | |
| 8 | 'unittest', 'tests.loader_dev_params', None) | |
| 9 | self.assertEqual('bar', overwrites['FOO']) |
| 0 | import datetime | |
| 1 | import json | |
| 2 | import logging | |
| 3 | import os | |
| 4 | import time | |
| 5 | ||
| 6 | from django.test import TestCase | |
| 7 | from etcd import EtcdKeyNotFound | |
| 8 | from etcd_settings.manager import ( | |
| 9 | EtcdClusterState, EtcdConfigInvalidValueError, EtcdConfigManager, | |
| 10 | ) | |
| 11 | ||
| 12 | from .conftest import settings | |
| 13 | ||
| 14 | ||
| 15 | class TestEtcdConfigManager(TestCase): | |
| 16 | longMessage = True | |
| 17 | ||
| 18 | def _dataset_with_empty_dir(self): | |
| 19 | key = os.path.join(self.mgr._env_defaults_path(self.env), 'dir/empty') | |
| 20 | self.mgr._client.write(key, None, dir=True) | |
| 21 | value = self.mgr._client.get(key) | |
| 22 | return key, value | |
| 23 | ||
| 24 | def _dataset_with_invalid_json(self): | |
| 25 | key = os.path.join( | |
| 26 | self.mgr._env_defaults_path(self.env), 'json/invalid') | |
| 27 | self.mgr._client.set(key, '{') | |
| 28 | value = self.mgr._client.get(key) | |
| 29 | return key, value | |
| 30 | ||
| 31 | def _dataset_for_defaults(self): | |
| 32 | dataset = {} | |
| 33 | k1 = os.path.join(self.mgr._env_defaults_path(self.env), 'foo/bar') | |
| 34 | dataset[k1] = '"baz"' | |
| 35 | k2 = os.path.join(self.mgr._env_defaults_path(self.env), 'foo/baz') | |
| 36 | dataset[k2] = '"bar"' | |
| 37 | k3 = os.path.join(self.mgr._env_defaults_path(self.env), 'foobarbaz') | |
| 38 | dataset[k3] = '"superbaz"' | |
| 39 | expected_env = { | |
| 40 | 'FOO_BAR': 'baz', | |
| 41 | 'FOO_BAZ': 'bar', | |
| 42 | 'FOOBARBAZ': 'superbaz', | |
| 43 | } | |
| 44 | for k, v in dataset.items(): | |
| 45 | self.mgr._client.set(k, v) | |
| 46 | return dataset.keys(), expected_env | |
| 47 | ||
| 48 | def _dataset_for_configsets(self): | |
| 49 | dataset = {} | |
| 50 | k1 = os.path.join(self.mgr._config_set_path('foo'), 'bar') | |
| 51 | dataset[k1] = '1' | |
| 52 | k2 = os.path.join(self.mgr._config_set_path('foo'), 'baz') | |
| 53 | dataset[k2] = '2' | |
| 54 | k3 = os.path.join(self.mgr._config_set_path('foo.bar'), 'baz') | |
| 55 | dataset[k3] = '1' | |
| 56 | k4 = os.path.join(self.mgr._config_set_path('foo.bar'), 'bazbaz') | |
| 57 | dataset[k4] = '2' | |
| 58 | k5 = os.path.join(self.mgr._config_set_path('foo.bar-zoo'), 'bar') | |
| 59 | dataset[k5] = '1' | |
| 60 | expected_sets = { | |
| 61 | 'foo': {'BAR': 1, 'BAZ': 2}, | |
| 62 | 'foo.bar': {'BAZ': 1, 'BAZBAZ': 2}, | |
| 63 | 'foo.bar-zoo': {'BAR': 1}, | |
| 64 | } | |
| 65 | for k, v in dataset.items(): | |
| 66 | self.mgr._client.set(k, v) | |
| 67 | return dataset.keys(), expected_sets | |
| 68 | ||
| 69 | def setUp(self): | |
| 70 | ||
| 71 | self.env = 'unittest' | |
| 72 | EtcdClusterState.etcd_index = 0 | |
| 73 | self.mgr = EtcdConfigManager( | |
| 74 | dev_params=None, prefix=settings.ETCD_PREFIX, protocol='http', | |
| 75 | host=settings.ETCD_HOST, port=settings.ETCD_PORT, | |
| 76 | username=settings.ETCD_USERNAME, password=settings.ETCD_PASSWORD, | |
| 77 | long_polling_timeout=0.1, long_polling_safety_delay=0.1 | |
| 78 | ) | |
| 79 | for l in self.mgr.logger.handlers: | |
| 80 | l.setLevel(logging.CRITICAL) | |
| 81 | ||
| 82 | def tearDown(self): | |
| 83 | def try_unless_not_found(f): | |
| 84 | try: | |
| 85 | f() | |
| 86 | except EtcdKeyNotFound: | |
| 87 | pass | |
| 88 | ||
| 89 | def clean_env(): | |
| 90 | self.mgr._client.delete( | |
| 91 | self.mgr._env_defaults_path(self.env), recursive=True) | |
| 92 | ||
| 93 | def clean_extensions(): | |
| 94 | self.mgr._client.delete( | |
| 95 | self.mgr._base_config_set_path, recursive=True) | |
| 96 | ||
| 97 | try_unless_not_found(clean_extensions) | |
| 98 | try_unless_not_found(clean_env) | |
| 99 | ||
| 100 | def test_init_logger(self): | |
| 101 | self.assertIsNotNone(self.mgr.logger) | |
| 102 | ||
| 103 | def test_encode_config_key(self): | |
| 104 | self.assertEqual( | |
| 105 | 'foo/bar/baz', | |
| 106 | self.mgr._encode_config_key('FOO_BAR_BAZ')) | |
| 107 | ||
| 108 | def test_decode_env_config_key(self): | |
| 109 | key = 'FOO_BAR' | |
| 110 | s = '{}/{}/foo/bar'.format(self.mgr._base_config_path, self.env) | |
| 111 | self.assertEqual((self.env, key), self.mgr._decode_config_key(s)) | |
| 112 | ||
| 113 | def test_decode_set_config_key(self): | |
| 114 | key = 'FOO_BAR' | |
| 115 | configset = 'unit.test' | |
| 116 | s = '{}/{}/foo/bar'.format(self.mgr._base_config_set_path, configset) | |
| 117 | self.assertEqual((configset, key), self.mgr._decode_config_key(s)) | |
| 118 | ||
| 119 | def test_encode_config_value(self): | |
| 120 | self.assertEqual( | |
| 121 | '"abcde"', | |
| 122 | self.mgr._encode_config_value('abcde')) | |
| 123 | self.assertEqual( | |
| 124 | '112', | |
| 125 | self.mgr._encode_config_value(112)) | |
| 126 | self.assertEqual( | |
| 127 | # Tuples are lost in encoding, should be avoided as config values | |
| 128 | '[1, "b"]', | |
| 129 | self.mgr._encode_config_value((1, 'b'))) | |
| 130 | encoded_config = self.mgr._encode_config_value(dict(foo=1, bar='baz')) | |
| 131 | self.assertEqual( | |
| 132 | json.loads('{"foo": 1, "bar": "baz"}'), | |
| 133 | json.loads(encoded_config)) | |
| 134 | ||
| 135 | def test_process_response_set_empty(self): | |
| 136 | key, value = self._dataset_with_empty_dir() | |
| 137 | self.assertEqual({}, self.mgr._process_response_set(value)) | |
| 138 | ||
| 139 | def test_process_response_exception_handling(self): | |
| 140 | with self.assertRaises(EtcdConfigInvalidValueError) as excContext: | |
| 141 | key, value = self._dataset_with_invalid_json() | |
| 142 | self.mgr._process_response_set(value) | |
| 143 | ||
| 144 | exc = excContext.exception | |
| 145 | ||
| 146 | self.assertEqual(key, exc.key) | |
| 147 | self.assertEqual(value.value, exc.raw_value) | |
| 148 | self.assertIn(key, str(exc), "Expect key in message") | |
| 149 | self.assertIn( | |
| 150 | "Expecting", str(exc), msg="Expect detailed error message") | |
| 151 | self.assertIn( | |
| 152 | "line", str(exc), msg="Expect line number in error message") | |
| 153 | self.assertIn( | |
| 154 | "column", str(exc), msg="Expect column number in error message") | |
| 155 | self.assertIn(value.value, str(exc), msg="Expect invalid value") | |
| 156 | ||
| 157 | def test_decode_config_value(self): | |
| 158 | self.assertEqual( | |
| 159 | 'abcde', | |
| 160 | self.mgr._decode_config_value('"abcde"')) | |
| 161 | self.assertEqual( | |
| 162 | 112, | |
| 163 | self.mgr._decode_config_value('112')) | |
| 164 | self.assertEqual( | |
| 165 | dict(foo=1, bar='baz'), | |
| 166 | self.mgr._decode_config_value('{"foo": 1, "bar": "baz"}')) | |
| 167 | self.assertEqual( | |
| 168 | [1, 'b'], | |
| 169 | self.mgr._decode_config_value('[1, "b"]')) | |
| 170 | ||
| 171 | def test_custom_encoding_decoding_values(self): | |
| 172 | d = datetime.datetime(2015, 10, 9, 8, 7, 6) | |
| 173 | encoded_d = self.mgr._encode_config_value(d) | |
| 174 | decoded_d = self.mgr._decode_config_value(encoded_d) | |
| 175 | self.assertEqual(True, isinstance(decoded_d, datetime.datetime)) | |
| 176 | self.assertEqual(d.isoformat(), decoded_d.isoformat()) | |
| 177 | self.assertEqual(d, decoded_d) | |
| 178 | ||
| 179 | def test_get_env_defaults(self): | |
| 180 | keys, expected = self._dataset_for_defaults() | |
| 181 | self.assertEqual(expected, self.mgr.get_env_defaults(self.env)) | |
| 182 | ||
| 183 | def test_get_config_sets(self): | |
| 184 | keys, expected_sets = self._dataset_for_configsets() | |
| 185 | self.assertEqual(expected_sets, self.mgr.get_config_sets()) | |
| 186 | ||
| 187 | def test_monitor_env_defaults(self): | |
| 188 | keys, expected_env = self._dataset_for_defaults() | |
| 189 | d = {} | |
| 190 | old_etcd_index = EtcdClusterState.etcd_index | |
| 191 | t = self.mgr.monitor_env_defaults(env=self.env, conf=d, max_events=1) | |
| 192 | self.assertEqual(1, t.result) | |
| 193 | self.assertEqual(expected_env, d) | |
| 194 | self.assertGreater(EtcdClusterState.etcd_index, old_etcd_index) | |
| 195 | ||
| 196 | def test_monitor_config_sets(self): | |
| 197 | keys, expected_sets = self._dataset_for_configsets() | |
| 198 | d = {} | |
| 199 | old_etcd_index = EtcdClusterState.etcd_index | |
| 200 | t = self.mgr.monitor_config_sets(conf=d, max_events=1) | |
| 201 | self.assertEqual(1, t.result) | |
| 202 | self.assertEqual(expected_sets, d) | |
| 203 | self.assertGreater(EtcdClusterState.etcd_index, old_etcd_index) | |
| 204 | ||
| 205 | def test_monitors_delay_and_continue_on_exception(self): | |
| 206 | d = {} | |
| 207 | max_events = 2 | |
| 208 | lambdas = [ | |
| 209 | lambda: self.mgr.monitor_env_defaults( | |
| 210 | self.env, conf=d, max_events=max_events), | |
| 211 | lambda: self.mgr.monitor_config_sets( | |
| 212 | conf=d, max_events=max_events), | |
| 213 | ] | |
| 214 | try: | |
| 215 | self.mgr._base_config_path = 'Unknown' | |
| 216 | for l in lambdas: | |
| 217 | t0 = time.time() | |
| 218 | t = l() | |
| 219 | self.assertEqual(max_events, t.result) | |
| 220 | self.assertEqual( | |
| 221 | True, | |
| 222 | (time.time() - t0) | |
| 223 | > (max_events * self.mgr.long_polling_safety_delay) | |
| 224 | ) | |
| 225 | self.assertEqual(False, t.is_alive()) | |
| 226 | finally: | |
| 227 | self.mgr._base_config_path = settings.ETCD_PREFIX | |
| 228 | ||
| 229 | def test_monitors_continue_on_etcd_exception(self): | |
| 230 | d = {} | |
| 231 | max_events = 2 | |
| 232 | lambdas = [ | |
| 233 | lambda: self.mgr.monitor_env_defaults( | |
| 234 | self.env, conf=d, max_events=max_events), | |
| 235 | lambda: self.mgr.monitor_config_sets( | |
| 236 | conf=d, max_events=max_events), | |
| 237 | ] | |
| 238 | try: | |
| 239 | old_etcd_index = EtcdClusterState.etcd_index | |
| 240 | self.mgr._etcd_index = 999990 | |
| 241 | for l in lambdas: | |
| 242 | t0 = time.time() | |
| 243 | t = l() | |
| 244 | self.assertEqual(max_events, t.result) | |
| 245 | net_time = time.time() - t0 | |
| 246 | self.assertGreater( | |
| 247 | net_time, | |
| 248 | (max_events * self.mgr.long_polling_timeout) | |
| 249 | ) | |
| 250 | self.assertLess( | |
| 251 | net_time, | |
| 252 | (max_events * | |
| 253 | (self.mgr.long_polling_safety_delay | |
| 254 | + self.mgr.long_polling_timeout)) | |
| 255 | ) | |
| 256 | self.assertEqual(False, t.is_alive()) | |
| 257 | finally: | |
| 258 | EtcdClusterState.etcd_index = old_etcd_index |
| 5 | 5 | from django.http import HttpRequest |
| 6 | 6 | from django.test import TestCase |
| 7 | 7 | from django.test.utils import override_settings |
| 8 | from etcd_settings.loader import get_overwrites | |
| 9 | from etcd_settings.manager import EtcdClusterState, EtcdConfigManager | |
| 8 | from etcd_config.loader import get_overwrites | |
| 9 | from etcd_config.manager import EtcdClusterState, EtcdConfigManager | |
| 10 | 10 | from etcd_settings.proxy import EtcdSettingsProxy |
| 11 | 11 | from mock import MagicMock |
| 12 | 12 | |
| 16 | 16 | @override_settings( |
| 17 | 17 | DJES_ETCD_DETAILS=settings.ETCD_DETAILS, |
| 18 | 18 | DJES_ENV=settings.ETCD_ENV, |
| 19 | DJES_REQUEST_GETTER='etcd_settings.utils.threaded', | |
| 19 | DJES_REQUEST_GETTER='etcd_config.utils.threaded', | |
| 20 | 20 | E=0 |
| 21 | 21 | ) |
| 22 | 22 | class TestEtcdSettingsProxy(TestCase): |