Codebase list ceilometer / 1562d03
* Removed patches applied upstream: - Add_response_handlers_to_support_different_response_types.patch - Add_support_to_namespaces_on_dynamic_pollsters.patch - Add_support_to_host_command_dynamic_pollster_definitions.patch Thomas Goirand 1 year, 2 months ago
5 changed file(s) with 4 addition(s) and 1908 deletion(s). Raw diff Collapse all Expand all
+4
-0
debian/changelog less more
11
22 * New upstream release.
33 * Removed versions from (build-)depends when satisfied in Bookworm.
4 * Removed patches applied upstream:
5 - Add_response_handlers_to_support_different_response_types.patch
6 - Add_support_to_namespaces_on_dynamic_pollsters.patch
7 - Add_support_to_host_command_dynamic_pollster_definitions.patch
48
59 -- Thomas Goirand <zigo@debian.org> Fri, 03 Mar 2023 16:38:18 +0100
610
+0
-563
debian/patches/Add_response_handlers_to_support_different_response_types.patch less more
0 From 225f1cd7765ddb7b725c538944947ada8c52e73f Mon Sep 17 00:00:00 2001
1 From: Pedro Henrique <phpm13@gmail.com>
2 Date: Mon, 18 Jul 2022 16:53:23 -0300
3 Subject: [PATCH] Add response handlers to support different response types
4
5 Problem description
6 ===================
7 The dynamic pollsters only support APIs that produce
8 JSON responses. Therefore the dynamic pollsters do not
9 support APIs where the response is an XML or not Restful
10 compliant APIs with HTTP 200 within a plain text message
11 on errors.
12
13 Proposal
14 ========
15 To allow the dynamic pollsters to support other APIs
16 response formats, we propose to add a response handling
17 that supports multiple response types. It must be
18 configurable in the dynamic pollsters YAML. The default
19 continues to be JSON.
20
21 Change-Id: I4886cefe06eccac2dc24adbc2fad2166bcbfdd2c
22 ---
23
24 Index: ceilometer/ceilometer/declarative.py
25 ===================================================================
26 --- ceilometer.orig/ceilometer/declarative.py
27 +++ ceilometer/ceilometer/declarative.py
28 @@ -49,6 +49,10 @@ class DynamicPollsterDefinitionException
29 pass
30
31
32 +class InvalidResponseTypeException(DynamicPollsterException):
33 + pass
34 +
35 +
36 class NonOpenStackApisDynamicPollsterException\
37 (DynamicPollsterDefinitionException):
38 pass
39 Index: ceilometer/ceilometer/polling/dynamic_pollster.py
40 ===================================================================
41 --- ceilometer.orig/ceilometer/polling/dynamic_pollster.py
42 +++ ceilometer/ceilometer/polling/dynamic_pollster.py
43 @@ -18,8 +18,10 @@
44 similar to the idea used for handling notifications.
45 """
46 import copy
47 +import json
48 import re
49 import time
50 +import xmltodict
51
52 from oslo_log import log
53
54 @@ -46,6 +48,80 @@ def validate_sample_type(sample_type):
55 % (sample_type, ceilometer_sample.TYPES))
56
57
58 +class XMLResponseHandler(object):
59 + """This response handler converts an XML in string format to a dict"""
60 +
61 + @staticmethod
62 + def handle(response):
63 + return xmltodict.parse(response)
64 +
65 +
66 +class JsonResponseHandler(object):
67 + """This response handler converts a JSON in string format to a dict"""
68 +
69 + @staticmethod
70 + def handle(response):
71 + return json.loads(response)
72 +
73 +
74 +class PlainTextResponseHandler(object):
75 + """This response handler converts a string to a dict {'out'=<string>}"""
76 +
77 + @staticmethod
78 + def handle(response):
79 + return {'out': str(response)}
80 +
81 +
82 +VALID_HANDLERS = {
83 + 'json': JsonResponseHandler,
84 + 'xml': XMLResponseHandler,
85 + 'text': PlainTextResponseHandler
86 +}
87 +
88 +
89 +def validate_response_handler(val):
90 + if not isinstance(val, list):
91 + raise declarative.DynamicPollsterDefinitionException(
92 + "Invalid response_handlers configuration. It must be a list. "
93 + "Provided value type: %s" % type(val).__name__)
94 +
95 + for value in val:
96 + if value not in VALID_HANDLERS:
97 + raise declarative.DynamicPollsterDefinitionException(
98 + "Invalid response_handler value [%s]. Accepted values "
99 + "are [%s]" % (value, ', '.join(list(VALID_HANDLERS))))
100 +
101 +
102 +class ResponseHandlerChain(object):
103 + """Tries to convert a string to a dict using the response handlers"""
104 +
105 + def __init__(self, response_handlers, **meta):
106 + if not isinstance(response_handlers, list):
107 + response_handlers = list(response_handlers)
108 +
109 + self.response_handlers = response_handlers
110 + self.meta = meta
111 +
112 + def handle(self, response):
113 + failed_handlers = []
114 + for handler in self.response_handlers:
115 + try:
116 + return handler.handle(response)
117 + except Exception as e:
118 + handler_name = handler.__name__
119 + failed_handlers.append(handler_name)
120 + LOG.debug(
121 + "Error handling response [%s] with handler [%s]: %s. "
122 + "We will try the next one, if multiple handlers were "
123 + "configured.",
124 + response, handler_name, e)
125 +
126 + handlers_str = ', '.join(failed_handlers)
127 + raise declarative.InvalidResponseTypeException(
128 + "No remaining handlers to handle the response [%s], "
129 + "used handlers [%s]. [%s]." % (response, handlers_str, self.meta))
130 +
131 +
132 class PollsterDefinitionBuilder(object):
133
134 def __init__(self, definitions):
135 @@ -440,7 +516,9 @@ class PollsterDefinitions(object):
136 PollsterDefinition(name='timeout', default=30),
137 PollsterDefinition(name='extra_metadata_fields_cache_seconds',
138 default=3600),
139 - PollsterDefinition(name='extra_metadata_fields')
140 + PollsterDefinition(name='extra_metadata_fields'),
141 + PollsterDefinition(name='response_handlers', default=['json'],
142 + validator=validate_response_handler)
143 ]
144
145 extra_definitions = []
146 @@ -655,6 +733,11 @@ class PollsterSampleGatherer(object):
147
148 def __init__(self, definitions):
149 self.definitions = definitions
150 + self.response_handler_chain = ResponseHandlerChain(
151 + map(VALID_HANDLERS.get,
152 + self.definitions.configurations['response_handlers']),
153 + url_path=definitions.configurations['url_path']
154 + )
155
156 @property
157 def default_discovery(self):
158 @@ -668,17 +751,17 @@ class PollsterSampleGatherer(object):
159 resp, url = self._internal_execute_request_get_samples(
160 definitions=definitions, **kwargs)
161
162 - response_json = resp.json()
163 - entry_size = len(response_json)
164 - LOG.debug("Entries [%s] in the JSON for request [%s] "
165 + response_dict = self.response_handler_chain.handle(resp.text)
166 + entry_size = len(response_dict)
167 + LOG.debug("Entries [%s] in the DICT for request [%s] "
168 "for dynamic pollster [%s].",
169 - response_json, url, definitions['name'])
170 + response_dict, url, definitions['name'])
171
172 if entry_size > 0:
173 samples = self.retrieve_entries_from_response(
174 - response_json, definitions)
175 + response_dict, definitions)
176 url_to_next_sample = self.get_url_to_next_sample(
177 - response_json, definitions)
178 + response_dict, definitions)
179
180 self.prepare_samples(definitions, samples, **kwargs)
181
182 Index: ceilometer/ceilometer/tests/unit/polling/test_dynamic_pollster.py
183 ===================================================================
184 --- ceilometer.orig/ceilometer/tests/unit/polling/test_dynamic_pollster.py
185 +++ ceilometer/ceilometer/tests/unit/polling/test_dynamic_pollster.py
186 @@ -14,13 +14,14 @@
187 """Tests for OpenStack dynamic pollster
188 """
189 import copy
190 +import json
191 import logging
192 from unittest import mock
193
194 import requests
195 from urllib import parse as urlparse
196
197 -from ceilometer.declarative import DynamicPollsterDefinitionException
198 +from ceilometer import declarative
199 from ceilometer.polling import dynamic_pollster
200 from ceilometer import sample
201 from oslotest import base
202 @@ -107,6 +108,11 @@ class TestDynamicPollster(base.BaseTestC
203 class FakeResponse(object):
204 status_code = None
205 json_object = None
206 + _text = None
207 +
208 + @property
209 + def text(self):
210 + return self._text or json.dumps(self.json_object)
211
212 def json(self):
213 return self.json_object
214 @@ -242,9 +248,10 @@ class TestDynamicPollster(base.BaseTestC
215 pollster_definition = copy.deepcopy(
216 self.pollster_definition_only_required_fields)
217 pollster_definition.pop(key)
218 - exception = self.assertRaises(DynamicPollsterDefinitionException,
219 - dynamic_pollster.DynamicPollster,
220 - pollster_definition)
221 + exception = self.assertRaises(
222 + declarative.DynamicPollsterDefinitionException,
223 + dynamic_pollster.DynamicPollster,
224 + pollster_definition)
225 self.assertEqual("Required fields ['%s'] not specified."
226 % key, exception.brief_message)
227
228 @@ -252,7 +259,7 @@ class TestDynamicPollster(base.BaseTestC
229 self.pollster_definition_only_required_fields[
230 'sample_type'] = "invalid_sample_type"
231 exception = self.assertRaises(
232 - DynamicPollsterDefinitionException,
233 + declarative.DynamicPollsterDefinitionException,
234 dynamic_pollster.DynamicPollster,
235 self.pollster_definition_only_required_fields)
236 self.assertEqual("Invalid sample type [invalid_sample_type]. "
237 @@ -314,6 +321,147 @@ class TestDynamicPollster(base.BaseTestC
238 self.assertEqual(3, len(samples))
239
240 @mock.patch('keystoneclient.v2_0.client.Client')
241 + def test_execute_request_json_response_handler(
242 + self, client_mock):
243 + pollster = dynamic_pollster.DynamicPollster(
244 + self.pollster_definition_only_required_fields)
245 +
246 + return_value = self.FakeResponse()
247 + return_value.status_code = requests.codes.ok
248 + return_value._text = '{"test": [1,2,3]}'
249 +
250 + client_mock.session.get.return_value = return_value
251 +
252 + samples = pollster.definitions.sample_gatherer. \
253 + execute_request_get_samples(
254 + keystone_client=client_mock,
255 + resource="https://endpoint.server.name/")
256 +
257 + self.assertEqual(3, len(samples))
258 +
259 + @mock.patch('keystoneclient.v2_0.client.Client')
260 + def test_execute_request_xml_response_handler(
261 + self, client_mock):
262 + definitions = copy.deepcopy(
263 + self.pollster_definition_only_required_fields)
264 + definitions['response_handlers'] = ['xml']
265 + pollster = dynamic_pollster.DynamicPollster(definitions)
266 +
267 + return_value = self.FakeResponse()
268 + return_value.status_code = requests.codes.ok
269 + return_value._text = '<test>123</test>'
270 + client_mock.session.get.return_value = return_value
271 +
272 + samples = pollster.definitions.sample_gatherer. \
273 + execute_request_get_samples(
274 + keystone_client=client_mock,
275 + resource="https://endpoint.server.name/")
276 +
277 + self.assertEqual(3, len(samples))
278 +
279 + @mock.patch('keystoneclient.v2_0.client.Client')
280 + def test_execute_request_xml_json_response_handler(
281 + self, client_mock):
282 + definitions = copy.deepcopy(
283 + self.pollster_definition_only_required_fields)
284 + definitions['response_handlers'] = ['xml', 'json']
285 + pollster = dynamic_pollster.DynamicPollster(definitions)
286 +
287 + return_value = self.FakeResponse()
288 + return_value.status_code = requests.codes.ok
289 + return_value._text = '<test>123</test>'
290 + client_mock.session.get.return_value = return_value
291 +
292 + samples = pollster.definitions.sample_gatherer. \
293 + execute_request_get_samples(
294 + keystone_client=client_mock,
295 + resource="https://endpoint.server.name/")
296 +
297 + self.assertEqual(3, len(samples))
298 +
299 + return_value._text = '{"test": [1,2,3,4]}'
300 +
301 + samples = pollster.definitions.sample_gatherer. \
302 + execute_request_get_samples(
303 + keystone_client=client_mock,
304 + resource="https://endpoint.server.name/")
305 +
306 + self.assertEqual(4, len(samples))
307 +
308 + @mock.patch('keystoneclient.v2_0.client.Client')
309 + def test_execute_request_xml_json_response_handler_invalid_response(
310 + self, client_mock):
311 + definitions = copy.deepcopy(
312 + self.pollster_definition_only_required_fields)
313 + definitions['response_handlers'] = ['xml', 'json']
314 + pollster = dynamic_pollster.DynamicPollster(definitions)
315 +
316 + return_value = self.FakeResponse()
317 + return_value.status_code = requests.codes.ok
318 + return_value._text = 'Invalid response'
319 + client_mock.session.get.return_value = return_value
320 +
321 + with self.assertLogs('ceilometer.polling.dynamic_pollster',
322 + level='DEBUG') as logs:
323 + gatherer = pollster.definitions.sample_gatherer
324 + exception = self.assertRaises(
325 + declarative.InvalidResponseTypeException,
326 + gatherer.execute_request_get_samples,
327 + keystone_client=client_mock,
328 + resource="https://endpoint.server.name/")
329 +
330 + xml_handling_error = logs.output[2]
331 + json_handling_error = logs.output[3]
332 +
333 + self.assertIn(
334 + 'DEBUG:ceilometer.polling.dynamic_pollster:'
335 + 'Error handling response [Invalid response] '
336 + 'with handler [XMLResponseHandler]',
337 + xml_handling_error)
338 +
339 + self.assertIn(
340 + 'DEBUG:ceilometer.polling.dynamic_pollster:'
341 + 'Error handling response [Invalid response] '
342 + 'with handler [JsonResponseHandler]',
343 + json_handling_error)
344 +
345 + self.assertEqual(
346 + "InvalidResponseTypeException None: "
347 + "No remaining handlers to handle the response "
348 + "[Invalid response], used handlers "
349 + "[XMLResponseHandler, JsonResponseHandler]. "
350 + "[{'url_path': 'v1/test/endpoint/fake'}].",
351 + str(exception))
352 +
353 + def test_configure_response_handler_definition_invalid_value(self):
354 + definitions = copy.deepcopy(
355 + self.pollster_definition_only_required_fields)
356 + definitions['response_handlers'] = ['jason']
357 +
358 + exception = self.assertRaises(
359 + declarative.DynamicPollsterDefinitionException,
360 + dynamic_pollster.DynamicPollster,
361 + pollster_definitions=definitions)
362 + self.assertEqual("DynamicPollsterDefinitionException None: "
363 + "Invalid response_handler value [jason]. "
364 + "Accepted values are [json, xml, text]",
365 + str(exception))
366 +
367 + def test_configure_response_handler_definition_invalid_type(self):
368 + definitions = copy.deepcopy(
369 + self.pollster_definition_only_required_fields)
370 + definitions['response_handlers'] = 'json'
371 +
372 + exception = self.assertRaises(
373 + declarative.DynamicPollsterDefinitionException,
374 + dynamic_pollster.DynamicPollster,
375 + pollster_definitions=definitions)
376 + self.assertEqual("DynamicPollsterDefinitionException None: "
377 + "Invalid response_handlers configuration. "
378 + "It must be a list. Provided value type: str",
379 + str(exception))
380 +
381 + @mock.patch('keystoneclient.v2_0.client.Client')
382 def test_execute_request_get_samples_exception_on_request(
383 self, client_mock):
384 pollster = dynamic_pollster.DynamicPollster(
385 @@ -728,6 +876,10 @@ class TestDynamicPollster(base.BaseTestC
386
387 def internal_execute_request_get_samples_mock(self, **kwargs):
388 class Response:
389 + @property
390 + def text(self):
391 + return json.dumps([sample])
392 +
393 def json(self):
394 return [sample]
395 return Response(), "url"
396 Index: ceilometer/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
397 ===================================================================
398 --- ceilometer.orig/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
399 +++ ceilometer/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
400 @@ -15,6 +15,7 @@
401 """
402
403 import copy
404 +import json
405 import sys
406 from unittest import mock
407
408 @@ -312,6 +313,11 @@ class TestNonOpenStackApisDynamicPollste
409 def internal_execute_request_get_samples_mock(
410 self, definitions, **kwargs):
411 class Response:
412 +
413 + @property
414 + def text(self):
415 + return json.dumps([sample])
416 +
417 def json(self):
418 return [sample]
419 return Response(), "url"
420 Index: ceilometer/doc/source/admin/telemetry-dynamic-pollster.rst
421 ===================================================================
422 --- ceilometer.orig/doc/source/admin/telemetry-dynamic-pollster.rst
423 +++ ceilometer/doc/source/admin/telemetry-dynamic-pollster.rst
424 @@ -45,7 +45,7 @@ attributes to define a dynamic pollster:
425 the unit or some other meaningful String value;
426
427 * ``value_attribute``: mandatory attribute; defines the attribute in the
428 - JSON response from the URL of the component being polled. We also accept
429 + response from the URL of the component being polled. We also accept
430 nested values dictionaries. To use a nested value one can simply use
431 ``attribute1.attribute2.<asMuchAsNeeded>.lastattribute``. It is also
432 possible to reference the sample itself using ``"." (dot)``; the self
433 @@ -281,6 +281,117 @@ desires):
434 name: "display_name"
435 default_value: 0
436
437 +* ``response_handlers``: optional parameter. Defines the response
438 + handlers used to handle the response. For now, the supported values
439 + are:
440 +
441 + ``json``: This handler will interpret the response as a `JSON` and will
442 + convert it to a `dictionary` which can be manipulated using the
443 + operations options when mapping the attributes:
444 +
445 + .. code-block:: yaml
446 +
447 + ---
448 +
449 + - name: "dynamic.json.response"
450 + sample_type: "gauge"
451 + [...]
452 + response_handlers:
453 + - json
454 +
455 + Response to handle:
456 +
457 + .. code-block:: json
458 +
459 + {
460 + "test": {
461 + "list": [1, 2, 3]
462 + }
463 + }
464 +
465 + Response handled:
466 +
467 + .. code-block:: python
468 +
469 + {
470 + 'test': {
471 + 'list': [1, 2, 3]
472 + }
473 + }
474 +
475 +
476 + ``xml``: This handler will interpret the response as an `XML` and will
477 + convert it to a `dictionary` which can be manipulated using the
478 + operations options when mapping the attributes:
479 +
480 + .. code-block:: yaml
481 +
482 + ---
483 +
484 + - name: "dynamic.json.response"
485 + sample_type: "gauge"
486 + [...]
487 + response_handlers:
488 + - xml
489 +
490 + Response to handle:
491 +
492 + .. code-block:: xml
493 +
494 + <test>
495 + <list>1</list>
496 + <list>2</list>
497 + <list>3</list>
498 + </test>
499 +
500 + Response handled:
501 +
502 + .. code-block:: python
503 +
504 + {
505 + 'test': {
506 + 'list': [1, 2, 3]
507 + }
508 + }
509 +
510 + ``text``: This handler will interpret the response as a `PlainText` and
511 + will convert it to a `dictionary` which can be manipulated using the
512 + operations options when mapping the attributes:
513 +
514 + .. code-block:: yaml
515 +
516 + ---
517 +
518 + - name: "dynamic.json.response"
519 + sample_type: "gauge"
520 + [...]
521 + response_handlers:
522 + - text
523 +
524 + Response to handle:
525 +
526 + .. code-block:: text
527 +
528 + Plain text response
529 +
530 + Response handled:
531 +
532 + .. code-block:: python
533 +
534 + {
535 + 'out': "Plain text response"
536 + }
537 +
538 + They can be used together or individually. If not defined, the
539 + `default` value will be `json`. If you set 2 or more response
540 + handlers, the first configured handler will be used to try to
541 + handle the response, if it is not possible, a `DEBUG` log
542 + message will be displayed, then the next will be used
543 + and so on. If no configured handler was able to handle
544 + the response, an empty dict will be returned and a `WARNING`
545 + log will be displayed to warn operators that the response was
546 + not able to be handled by any configured handler.
547 +
548 The dynamic pollsters system configuration (for non-OpenStack APIs)
549 -------------------------------------------------------------------
550
551 Index: ceilometer/requirements.txt
552 ===================================================================
553 --- ceilometer.orig/requirements.txt
554 +++ ceilometer/requirements.txt
555 @@ -6,6 +6,7 @@
556 # of appearance. Changing the order has an impact on the overall integration
557 # process, which may cause wedges in the gate later.
558
559 +xmltodict>=0.13.0 # MIT License
560 cachetools>=2.1.0 # MIT License
561 cotyledon>=1.3.0 #Apache-2.0
562 futurist>=1.8.0 # Apache-2.0
+0
-1166
debian/patches/Add_support_to_host_command_dynamic_pollster_definitions.patch less more
0 Author: Pedro Henrique <phpm13@gmail.com>
1 Date: Wed, 03 Aug 2022 10:09:00 -0300
2 Description: Add support to host command dynamic pollster definitions
3 Problem description
4 ===================
5 Today we have some hardcoded pollsters that are gathering
6 data from running virtual machines through libvirt or
7 different programs running in the compute nodes. However,
8 the Dynamic pollster definition does not support this kind of
9 operations to gather data, it only supports HTTP Rest
10 requests to collect data. Therefore, it is not possible to
11 use the dynamic pollster definition to create a YML based
12 pollster that runs and collects data from Libvirt in the
13 compute nodes.
14 .
15 Proposal
16 ========
17 To allow host commands/scripts in the Dynamic pollsters,
18 we propose to add a new pollster definition using the
19 `os.subprocess` lib to run host commands to collect
20 Host/VMs data and store them in the configured backend.
21 This will provide more flexibility and make the
22 Dynamic pollsters able to be used in Ceilometer compute
23 instances as well.
24 Change-Id: I50b8dc341ce457780416b41d138e35f5a0d083b6
25 Depends-On: https://review.opendev.org/c/openstack/ceilometer/+/850253
26 Origin: upstream, https://review.opendev.org/c/openstack/ceilometer/+/852021
27 Last-Update: 2022-10-24
28
29 Index: ceilometer/ceilometer/polling/dynamic_pollster.py
30 ===================================================================
31 --- ceilometer.orig/ceilometer/polling/dynamic_pollster.py
32 +++ ceilometer/ceilometer/polling/dynamic_pollster.py
33 @@ -20,6 +20,7 @@
34 import copy
35 import json
36 import re
37 +import subprocess
38 import time
39 import xmltodict
40
41 @@ -205,7 +206,7 @@ class PollsterSampleExtractor(object):
42 metadata=metadata, pollster_definitions=pollster_definitions)
43
44 extra_metadata = self.definitions.retrieve_extra_metadata(
45 - kwargs['manager'], pollster_sample)
46 + kwargs['manager'], pollster_sample, kwargs['conf'])
47
48 for key in extra_metadata.keys():
49 if key in metadata.keys():
50 @@ -518,7 +519,8 @@ class PollsterDefinitions(object):
51 default=3600),
52 PollsterDefinition(name='extra_metadata_fields'),
53 PollsterDefinition(name='response_handlers', default=['json'],
54 - validator=validate_response_handler)
55 + validator=validate_response_handler),
56 + PollsterDefinition(name='base_metadata', default={})
57 ]
58
59 extra_definitions = []
60 @@ -572,114 +574,75 @@ class PollsterDefinitions(object):
61 "Required fields %s not specified."
62 % missing, self.configurations)
63
64 - def retrieve_extra_metadata(self, manager, request_sample):
65 + def retrieve_extra_metadata(self, manager, request_sample, pollster_conf):
66 extra_metadata_fields = self.configurations['extra_metadata_fields']
67 if extra_metadata_fields:
68 - if isinstance(self, NonOpenStackApisPollsterDefinition):
69 - raise declarative.NonOpenStackApisDynamicPollsterException(
70 - "Not supported the use of extra metadata gathering for "
71 - "non-openstack pollsters [%s] (yet)."
72 - % self.configurations['name'])
73 -
74 - return self._retrieve_extra_metadata(
75 - extra_metadata_fields, manager, request_sample)
76 + extra_metadata_samples = {}
77 + extra_metadata_by_name = {}
78 + if not isinstance(extra_metadata_fields, (list, tuple)):
79 + extra_metadata_fields = [extra_metadata_fields]
80 + for ext_metadata in extra_metadata_fields:
81 + ext_metadata.setdefault(
82 + 'sample_type', self.configurations['sample_type'])
83 + ext_metadata.setdefault('unit', self.configurations['unit'])
84 + ext_metadata.setdefault(
85 + 'value_attribute', ext_metadata.get(
86 + 'value', self.configurations['value_attribute']))
87 + ext_metadata['base_metadata'] = {
88 + 'extra_metadata_captured': extra_metadata_samples,
89 + 'extra_metadata_by_name': extra_metadata_by_name,
90 + 'sample': request_sample
91 + }
92 + parent_cache_ttl = self.configurations[
93 + 'extra_metadata_fields_cache_seconds']
94 + cache_ttl = ext_metadata.get(
95 + 'extra_metadata_fields_cache_seconds', parent_cache_ttl
96 + )
97 + response_cache = self.response_cache
98 + extra_metadata_pollster = DynamicPollster(
99 + ext_metadata, conf=pollster_conf, cache_ttl=cache_ttl,
100 + extra_metadata_responses_cache=response_cache,
101 + )
102 + resources = [None]
103 + if ext_metadata.get('endpoint_type'):
104 + resources = manager.discover([
105 + extra_metadata_pollster.default_discovery], {})
106 + samples = extra_metadata_pollster.get_samples(
107 + manager, None, resources)
108 + for sample in samples:
109 + self.fill_extra_metadata_samples(
110 + extra_metadata_by_name,
111 + extra_metadata_samples,
112 + sample)
113 + return extra_metadata_samples
114
115 LOG.debug("No extra metadata to be captured for pollsters [%s] and "
116 "request sample [%s].", self.definitions, request_sample)
117 return {}
118
119 - def _retrieve_extra_metadata(
120 - self, extra_metadata_fields, manager, request_sample):
121 - LOG.debug("Processing extra metadata fields [%s] for "
122 - "sample [%s].", extra_metadata_fields,
123 - request_sample)
124 -
125 - extra_metadata_captured = {}
126 - for extra_metadata in extra_metadata_fields:
127 - extra_metadata_name = extra_metadata['name']
128 -
129 - if extra_metadata_name in extra_metadata_captured.keys():
130 - LOG.warning("Duplicated extra metadata name [%s]. Therefore, "
131 - "we do not process this iteration [%s].",
132 - extra_metadata_name, extra_metadata)
133 + def fill_extra_metadata_samples(self, extra_metadata_by_name,
134 + extra_metadata_samples, sample):
135 + extra_metadata_samples[sample.name] = sample.volume
136 + LOG.debug("Merging the sample metadata [%s] of the "
137 + "extra_metadata_field [%s], with the "
138 + "extra_metadata_samples [%s].",
139 + sample.resource_metadata,
140 + sample.name,
141 + extra_metadata_samples)
142 + for key, value in sample.resource_metadata.items():
143 + if value is None and key in extra_metadata_samples:
144 + LOG.debug("Metadata [%s] for extra_metadata_field [%s] "
145 + "is None, skipping metadata override by None "
146 + "value", key, sample.name)
147 continue
148 + extra_metadata_samples[key] = value
149 + extra_metadata_by_name[sample.name] = {
150 + 'value': sample.volume,
151 + 'metadata': sample.resource_metadata
152 + }
153
154 - LOG.debug("Processing extra metadata [%s] for sample [%s].",
155 - extra_metadata_name, request_sample)
156 -
157 - endpoint_type = 'endpoint:' + extra_metadata['endpoint_type']
158 - if not endpoint_type.endswith(
159 - PollsterDefinitions.EXTERNAL_ENDPOINT_TYPE):
160 - response = self.execute_openstack_extra_metadata_gathering(
161 - endpoint_type, extra_metadata, manager, request_sample,
162 - extra_metadata_captured)
163 - else:
164 - raise declarative.NonOpenStackApisDynamicPollsterException(
165 - "Not supported the use of extra metadata gathering for "
166 - "non-openstack endpoints [%s] (yet)." % extra_metadata)
167 -
168 - extra_metadata_extractor_kwargs = {
169 - 'value_attribute': extra_metadata['value'],
170 - 'sample': request_sample}
171 -
172 - extra_metadata_value = \
173 - self.sample_extractor.retrieve_attribute_nested_value(
174 - response, **extra_metadata_extractor_kwargs)
175 -
176 - LOG.debug("Generated extra metadata [%s] with value [%s].",
177 - extra_metadata_name, extra_metadata_value)
178 - extra_metadata_captured[extra_metadata_name] = extra_metadata_value
179 -
180 - return extra_metadata_captured
181 -
182 - def execute_openstack_extra_metadata_gathering(self, endpoint_type,
183 - extra_metadata, manager,
184 - request_sample,
185 - extra_metadata_captured):
186 - url_for_endpoint_type = manager.discover(
187 - [endpoint_type], self.response_cache)
188 -
189 - LOG.debug("URL [%s] found for endpoint type [%s].",
190 - url_for_endpoint_type, endpoint_type)
191 -
192 - if url_for_endpoint_type:
193 - url_for_endpoint_type = url_for_endpoint_type[0]
194 -
195 - self.sample_gatherer.generate_url_path(
196 - extra_metadata, request_sample, extra_metadata_captured)
197 -
198 - cached_response, max_ttl_for_cache = self.response_cache.get(
199 - extra_metadata['url_path'], (None, None))
200 -
201 - extra_metadata_fields_cache_seconds = extra_metadata.get(
202 - 'extra_metadata_fields_cache_seconds',
203 - self.configurations['extra_metadata_fields_cache_seconds'])
204 -
205 - current_time = time.time()
206 - if cached_response and max_ttl_for_cache >= current_time:
207 - LOG.debug("Returning response [%s] for request [%s] as the TTL "
208 - "[max=%s, current_time=%s] has not expired yet.",
209 - cached_response, extra_metadata['url_path'],
210 - max_ttl_for_cache, current_time)
211 - return cached_response
212 -
213 - if cached_response:
214 - LOG.debug("Cleaning cached response [%s] for request [%s] "
215 - "as the TTL [max=%s, current_time=%s] has expired.",
216 - cached_response, extra_metadata['url_path'],
217 - max_ttl_for_cache, current_time)
218 -
219 - response = self.sample_gatherer.execute_request_for_definitions(
220 - extra_metadata, **{'manager': manager,
221 - 'keystone_client': manager._keystone,
222 - 'resource': url_for_endpoint_type,
223 - 'execute_id_overrides': False})
224 -
225 - max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds
226 -
227 - cache_tuple = (response, max_ttl_for_cache)
228 - self.response_cache[extra_metadata['url_path']] = cache_tuple
229 - return response
230 + LOG.debug("extra_metadata_samples after merging: [%s].",
231 + extra_metadata_samples)
232
233
234 class MultiMetricPollsterDefinitions(PollsterDefinitions):
235 @@ -739,6 +702,42 @@ class PollsterSampleGatherer(object):
236 url_path=definitions.configurations['url_path']
237 )
238
239 + def get_cache_key(self, definitions, **kwargs):
240 + return self.get_request_linked_samples_url(kwargs, definitions)
241 +
242 + def get_cached_response(self, definitions, **kwargs):
243 + if self.definitions.cache_ttl == 0:
244 + return
245 + cache_key = self.get_cache_key(definitions, **kwargs)
246 + response_cache = self.definitions.response_cache
247 + cached_response, max_ttl_for_cache = response_cache.get(
248 + cache_key, (None, None))
249 +
250 + current_time = time.time()
251 + if cached_response and max_ttl_for_cache >= current_time:
252 + LOG.debug("Returning response [%s] for request [%s] as the TTL "
253 + "[max=%s, current_time=%s] has not expired yet.",
254 + cached_response, definitions,
255 + max_ttl_for_cache, current_time)
256 + return cached_response
257 +
258 + if cached_response and max_ttl_for_cache < current_time:
259 + LOG.debug("Cleaning cached response [%s] for request [%s] "
260 + "as the TTL [max=%s, current_time=%s] has expired.",
261 + cached_response, definitions,
262 + max_ttl_for_cache, current_time)
263 + response_cache.pop(cache_key, None)
264 +
265 + def store_cached_response(self, definitions, resp, **kwargs):
266 + if self.definitions.cache_ttl == 0:
267 + return
268 + cache_key = self.get_cache_key(definitions, **kwargs)
269 + extra_metadata_fields_cache_seconds = self.definitions.cache_ttl
270 + max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds
271 +
272 + cache_tuple = (resp, max_ttl_for_cache)
273 + self.definitions.response_cache[cache_key] = cache_tuple
274 +
275 @property
276 def default_discovery(self):
277 return 'endpoint:' + self.definitions.configurations['endpoint_type']
278 @@ -748,10 +747,14 @@ class PollsterSampleGatherer(object):
279 self.definitions.configurations, **kwargs)
280
281 def execute_request_for_definitions(self, definitions, **kwargs):
282 - resp, url = self._internal_execute_request_get_samples(
283 - definitions=definitions, **kwargs)
284 + if response_dict := self.get_cached_response(definitions, **kwargs):
285 + url = 'cached'
286 + else:
287 + resp, url = self._internal_execute_request_get_samples(
288 + definitions=definitions, **kwargs)
289 + response_dict = self.response_handler_chain.handle(resp.text)
290 + self.store_cached_response(definitions, response_dict, **kwargs)
291
292 - response_dict = self.response_handler_chain.handle(resp.text)
293 entry_size = len(response_dict)
294 LOG.debug("Entries [%s] in the DICT for request [%s] "
295 "for dynamic pollster [%s].",
296 @@ -790,21 +793,6 @@ class PollsterSampleGatherer(object):
297 self.generate_new_attributes_in_sample(
298 request_sample, resource_id_attribute, 'id')
299
300 - def generate_url_path(self, extra_metadata, sample,
301 - extra_metadata_captured):
302 - if not extra_metadata.get('url_path_original'):
303 - extra_metadata[
304 - 'url_path_original'] = extra_metadata['url_path']
305 -
306 - extra_metadata['url_path'] = eval(
307 - extra_metadata['url_path_original'])
308 -
309 - LOG.debug("URL [%s] generated for pattern [%s] for sample [%s] and "
310 - "extra metadata captured [%s].",
311 - extra_metadata['url_path'],
312 - extra_metadata['url_path_original'], sample,
313 - extra_metadata_captured)
314 -
315 def generate_new_attributes_in_sample(
316 self, sample, attribute_key, new_attribute_key):
317
318 @@ -881,6 +869,15 @@ class PollsterSampleGatherer(object):
319
320 def get_request_url(self, kwargs, url_path):
321 endpoint = kwargs['resource']
322 + params = copy.deepcopy(
323 + self.definitions.configurations.get(
324 + 'base_metadata', {}))
325 + try:
326 + url_path = eval(url_path, params)
327 + except Exception:
328 + LOG.debug("Cannot eval path [%s] with params [%s],"
329 + " using [%s] instead.",
330 + url_path, params, url_path)
331 return urlparse.urljoin(endpoint, url_path)
332
333 def retrieve_entries_from_response(self, response_json, definitions):
334 @@ -919,6 +916,57 @@ class NonOpenStackApisPollsterDefinition
335 return configurations.get('module')
336
337
338 +class HostCommandPollsterDefinition(PollsterDefinitions):
339 +
340 + extra_definitions = [
341 + PollsterDefinition(name='endpoint_type', required=False),
342 + PollsterDefinition(name='url_path', required=False),
343 + PollsterDefinition(name='host_command', required=True)]
344 +
345 + def __init__(self, configurations):
346 + super(HostCommandPollsterDefinition, self).__init__(
347 + configurations)
348 + self.sample_gatherer = HostCommandSamplesGatherer(self)
349 +
350 + @staticmethod
351 + def is_field_applicable_to_definition(configurations):
352 + return configurations.get('host_command')
353 +
354 +
355 +class HostCommandSamplesGatherer(PollsterSampleGatherer):
356 +
357 + class Response(object):
358 + def __init__(self, text):
359 + self.text = text
360 +
361 + def get_cache_key(self, definitions, **kwargs):
362 + return self.get_command(definitions)
363 +
364 + def _internal_execute_request_get_samples(self, definitions, **kwargs):
365 + command = self.get_command(definitions, **kwargs)
366 + LOG.debug('Running Host command: [%s]', command)
367 + result = subprocess.getoutput(command)
368 + LOG.debug('Host command [%s] result: [%s]', command, result)
369 + return self.Response(result), command
370 +
371 + def get_command(self, definitions, next_sample_url=None, **kwargs):
372 + command = next_sample_url or definitions['host_command']
373 + params = copy.deepcopy(
374 + self.definitions.configurations.get(
375 + 'base_metadata', {}))
376 + try:
377 + command = eval(command, params)
378 + except Exception:
379 + LOG.debug("Cannot eval command [%s] with params [%s],"
380 + " using [%s] instead.",
381 + command, params, command)
382 + return command
383 +
384 + @property
385 + def default_discovery(self):
386 + return 'local_node'
387 +
388 +
389 class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
390
391 @property
392 @@ -1010,8 +1058,10 @@ class DynamicPollster(plugin_base.Pollst
393 # Mandatory name field
394 name = ""
395
396 - def __init__(self, pollster_definitions={}, conf=None,
397 - supported_definitions=[NonOpenStackApisPollsterDefinition,
398 + def __init__(self, pollster_definitions={}, conf=None, cache_ttl=0,
399 + extra_metadata_responses_cache=None,
400 + supported_definitions=[HostCommandPollsterDefinition,
401 + NonOpenStackApisPollsterDefinition,
402 MultiMetricPollsterDefinitions,
403 SingleMetricPollsterDefinitions]):
404 super(DynamicPollster, self).__init__(conf)
405 @@ -1021,6 +1071,10 @@ class DynamicPollster(plugin_base.Pollst
406
407 self.definitions = PollsterDefinitionBuilder(
408 self.supported_definitions).build_definitions(pollster_definitions)
409 + self.definitions.cache_ttl = cache_ttl
410 + self.definitions.response_cache = extra_metadata_responses_cache
411 + if extra_metadata_responses_cache is None:
412 + self.definitions.response_cache = {}
413 self.pollster_definitions = self.definitions.configurations
414 if 'metadata_fields' in self.pollster_definitions:
415 LOG.debug("Metadata fields configured to [%s].",
416 @@ -1054,9 +1108,12 @@ class DynamicPollster(plugin_base.Pollst
417 for r in resources:
418 LOG.debug("Executing get sample for resource [%s].", r)
419 samples = self.load_samples(r, manager)
420 + if not isinstance(samples, (list, tuple)):
421 + samples = [samples]
422 for pollster_sample in samples:
423 - kwargs = {'manager': manager, 'resource': r}
424 - sample = self.extract_sample(pollster_sample, **kwargs)
425 + sample = self.extract_sample(
426 + pollster_sample, manager=manager,
427 + resource=r, conf=self.conf)
428 if isinstance(sample, SkippedSample):
429 continue
430 yield from sample
431 Index: ceilometer/ceilometer/tests/unit/polling/test_dynamic_pollster.py
432 ===================================================================
433 --- ceilometer.orig/ceilometer/tests/unit/polling/test_dynamic_pollster.py
434 +++ ceilometer/ceilometer/tests/unit/polling/test_dynamic_pollster.py
435 @@ -389,6 +389,424 @@ class TestDynamicPollster(base.BaseTestC
436 self.assertEqual(4, len(samples))
437
438 @mock.patch('keystoneclient.v2_0.client.Client')
439 + def test_execute_request_extra_metadata_fields_cache_disabled(
440 + self, client_mock):
441 + definitions = copy.deepcopy(
442 + self.pollster_definition_only_required_fields)
443 + extra_metadata_fields = {
444 + 'extra_metadata_fields_cache_seconds': 0,
445 + 'name': "project_name",
446 + 'endpoint_type': "identity",
447 + 'url_path': "'/v3/projects/' + str(sample['project_id'])",
448 + 'value': "name",
449 + }
450 + definitions['value_attribute'] = 'project_id'
451 + definitions['extra_metadata_fields'] = extra_metadata_fields
452 + pollster = dynamic_pollster.DynamicPollster(definitions)
453 +
454 + return_value = self.FakeResponse()
455 + return_value.status_code = requests.codes.ok
456 + return_value._text = '''
457 + {"projects": [
458 + {"project_id": 9999, "name": "project1"},
459 + {"project_id": 8888, "name": "project2"},
460 + {"project_id": 7777, "name": "project3"},
461 + {"project_id": 9999, "name": "project1"},
462 + {"project_id": 8888, "name": "project2"},
463 + {"project_id": 7777, "name": "project3"},
464 + {"project_id": 9999, "name": "project1"},
465 + {"project_id": 8888, "name": "project2"},
466 + {"project_id": 7777, "name": "project3"}]
467 + }
468 + '''
469 +
470 + return_value9999 = self.FakeResponse()
471 + return_value9999.status_code = requests.codes.ok
472 + return_value9999._text = '''
473 + {"project":
474 + {"project_id": 9999, "name": "project1"}
475 + }
476 + '''
477 +
478 + return_value8888 = self.FakeResponse()
479 + return_value8888.status_code = requests.codes.ok
480 + return_value8888._text = '''
481 + {"project":
482 + {"project_id": 8888, "name": "project2"}
483 + }
484 + '''
485 +
486 + return_value7777 = self.FakeResponse()
487 + return_value7777.status_code = requests.codes.ok
488 + return_value7777._text = '''
489 + {"project":
490 + {"project_id": 7777, "name": "project3"}
491 + }
492 + '''
493 +
494 + def get(url, *args, **kwargs):
495 + if '9999' in url:
496 + return return_value9999
497 + if '8888' in url:
498 + return return_value8888
499 + if '7777' in url:
500 + return return_value7777
501 + return return_value
502 +
503 + client_mock.session.get.side_effect = get
504 + manager = mock.Mock
505 + manager._keystone = client_mock
506 +
507 + def discover(*args, **kwargs):
508 + return ["https://endpoint.server.name/"]
509 +
510 + manager.discover = discover
511 + samples = pollster.get_samples(
512 + manager=manager, cache=None,
513 + resources=["https://endpoint.server.name/"])
514 +
515 + samples = list(samples)
516 +
517 + n_calls = client_mock.session.get.call_count
518 + self.assertEqual(9, len(samples))
519 + self.assertEqual(10, n_calls)
520 +
521 + @mock.patch('keystoneclient.v2_0.client.Client')
522 + def test_execute_request_extra_metadata_fields_cache_enabled(
523 + self, client_mock):
524 + definitions = copy.deepcopy(
525 + self.pollster_definition_only_required_fields)
526 + extra_metadata_fields = {
527 + 'extra_metadata_fields_cache_seconds': 3600,
528 + 'name': "project_name",
529 + 'endpoint_type': "identity",
530 + 'url_path': "'/v3/projects/' + str(sample['project_id'])",
531 + 'value': "name",
532 + }
533 + definitions['value_attribute'] = 'project_id'
534 + definitions['extra_metadata_fields'] = extra_metadata_fields
535 + pollster = dynamic_pollster.DynamicPollster(definitions)
536 +
537 + return_value = self.FakeResponse()
538 + return_value.status_code = requests.codes.ok
539 + return_value._text = '''
540 + {"projects": [
541 + {"project_id": 9999, "name": "project1"},
542 + {"project_id": 8888, "name": "project2"},
543 + {"project_id": 7777, "name": "project3"},
544 + {"project_id": 9999, "name": "project4"},
545 + {"project_id": 8888, "name": "project5"},
546 + {"project_id": 7777, "name": "project6"},
547 + {"project_id": 9999, "name": "project7"},
548 + {"project_id": 8888, "name": "project8"},
549 + {"project_id": 7777, "name": "project9"}]
550 + }
551 + '''
552 +
553 + return_value9999 = self.FakeResponse()
554 + return_value9999.status_code = requests.codes.ok
555 + return_value9999._text = '''
556 + {"project":
557 + {"project_id": 9999, "name": "project1"}
558 + }
559 + '''
560 +
561 + return_value8888 = self.FakeResponse()
562 + return_value8888.status_code = requests.codes.ok
563 + return_value8888._text = '''
564 + {"project":
565 + {"project_id": 8888, "name": "project2"}
566 + }
567 + '''
568 +
569 + return_value7777 = self.FakeResponse()
570 + return_value7777.status_code = requests.codes.ok
571 + return_value7777._text = '''
572 + {"project":
573 + {"project_id": 7777, "name": "project3"}
574 + }
575 + '''
576 +
577 + def get(url, *args, **kwargs):
578 + if '9999' in url:
579 + return return_value9999
580 + if '8888' in url:
581 + return return_value8888
582 + if '7777' in url:
583 + return return_value7777
584 + return return_value
585 +
586 + client_mock.session.get.side_effect = get
587 + manager = mock.Mock
588 + manager._keystone = client_mock
589 +
590 + def discover(*args, **kwargs):
591 + return ["https://endpoint.server.name/"]
592 +
593 + manager.discover = discover
594 + samples = pollster.get_samples(
595 + manager=manager, cache=None,
596 + resources=["https://endpoint.server.name/"])
597 +
598 + samples = list(samples)
599 +
600 + n_calls = client_mock.session.get.call_count
601 + self.assertEqual(9, len(samples))
602 + self.assertEqual(4, n_calls)
603 +
604 + @mock.patch('keystoneclient.v2_0.client.Client')
605 + def test_execute_request_extra_metadata_fields(
606 + self, client_mock):
607 + definitions = copy.deepcopy(
608 + self.pollster_definition_only_required_fields)
609 + extra_metadata_fields = [{
610 + 'name': "project_name",
611 + 'endpoint_type': "identity",
612 + 'url_path': "'/v3/projects/' + str(sample['project_id'])",
613 + 'value': "name",
614 + 'metadata_fields': ['meta']
615 + }, {
616 + 'name': "project_alias",
617 + 'endpoint_type': "identity",
618 + 'url_path': "'/v3/projects/' + "
619 + "str(extra_metadata_captured['project_name'])",
620 + 'value': "name",
621 + 'metadata_fields': ['meta']
622 + }, {
623 + 'name': "project_meta",
624 + 'endpoint_type': "identity",
625 + 'url_path': "'/v3/projects/' + "
626 + "str(extra_metadata_by_name['project_name']"
627 + "['metadata']['meta'])",
628 + 'value': "project_id",
629 + 'metadata_fields': ['meta']
630 + }]
631 + definitions['value_attribute'] = 'project_id'
632 + definitions['extra_metadata_fields'] = extra_metadata_fields
633 + pollster = dynamic_pollster.DynamicPollster(definitions)
634 +
635 + return_value = self.FakeResponse()
636 + return_value.status_code = requests.codes.ok
637 + return_value._text = '''
638 + {"projects": [
639 + {"project_id": 9999, "name": "project1"},
640 + {"project_id": 8888, "name": "project2"},
641 + {"project_id": 7777, "name": "project3"}]
642 + }
643 + '''
644 +
645 + return_value9999 = self.FakeResponse()
646 + return_value9999.status_code = requests.codes.ok
647 + return_value9999._text = '''
648 + {"project":
649 + {"project_id": 9999, "name": "project1",
650 + "meta": "m1"}
651 + }
652 + '''
653 +
654 + return_value8888 = self.FakeResponse()
655 + return_value8888.status_code = requests.codes.ok
656 + return_value8888._text = '''
657 + {"project":
658 + {"project_id": 8888, "name": "project2",
659 + "meta": "m2"}
660 + }
661 + '''
662 +
663 + return_value7777 = self.FakeResponse()
664 + return_value7777.status_code = requests.codes.ok
665 + return_value7777._text = '''
666 + {"project":
667 + {"project_id": 7777, "name": "project3",
668 + "meta": "m3"}
669 + }
670 + '''
671 +
672 + return_valueP1 = self.FakeResponse()
673 + return_valueP1.status_code = requests.codes.ok
674 + return_valueP1._text = '''
675 + {"project":
676 + {"project_id": 7777, "name": "p1",
677 + "meta": null}
678 + }
679 + '''
680 +
681 + return_valueP2 = self.FakeResponse()
682 + return_valueP2.status_code = requests.codes.ok
683 + return_valueP2._text = '''
684 + {"project":
685 + {"project_id": 7777, "name": "p2",
686 + "meta": null}
687 + }
688 + '''
689 +
690 + return_valueP3 = self.FakeResponse()
691 + return_valueP3.status_code = requests.codes.ok
692 + return_valueP3._text = '''
693 + {"project":
694 + {"project_id": 7777, "name": "p3",
695 + "meta": null}
696 + }
697 + '''
698 +
699 + return_valueM1 = self.FakeResponse()
700 + return_valueM1.status_code = requests.codes.ok
701 + return_valueM1._text = '''
702 + {"project":
703 + {"project_id": "META1", "name": "p3",
704 + "meta": null}
705 + }
706 + '''
707 +
708 + return_valueM2 = self.FakeResponse()
709 + return_valueM2.status_code = requests.codes.ok
710 + return_valueM2._text = '''
711 + {"project":
712 + {"project_id": "META2", "name": "p3",
713 + "meta": null}
714 + }
715 + '''
716 +
717 + return_valueM3 = self.FakeResponse()
718 + return_valueM3.status_code = requests.codes.ok
719 + return_valueM3._text = '''
720 + {"project":
721 + {"project_id": "META3", "name": "p3",
722 + "meta": null}
723 + }
724 + '''
725 +
726 + def get(url, *args, **kwargs):
727 + if '9999' in url:
728 + return return_value9999
729 + if '8888' in url:
730 + return return_value8888
731 + if '7777' in url:
732 + return return_value7777
733 + if 'project1' in url:
734 + return return_valueP1
735 + if 'project2' in url:
736 + return return_valueP2
737 + if 'project3' in url:
738 + return return_valueP3
739 + if 'm1' in url:
740 + return return_valueM1
741 + if 'm2' in url:
742 + return return_valueM2
743 + if 'm3' in url:
744 + return return_valueM3
745 +
746 + return return_value
747 +
748 + client_mock.session.get = get
749 + manager = mock.Mock
750 + manager._keystone = client_mock
751 +
752 + def discover(*args, **kwargs):
753 + return ["https://endpoint.server.name/"]
754 +
755 + manager.discover = discover
756 + samples = pollster.get_samples(
757 + manager=manager, cache=None,
758 + resources=["https://endpoint.server.name/"])
759 +
760 + samples = list(samples)
761 + self.assertEqual(3, len(samples))
762 +
763 + self.assertEqual(samples[0].volume, 9999)
764 + self.assertEqual(samples[1].volume, 8888)
765 + self.assertEqual(samples[2].volume, 7777)
766 +
767 + self.assertEqual(samples[0].resource_metadata,
768 + {'project_name': 'project1',
769 + 'project_alias': 'p1',
770 + 'meta': 'm1',
771 + 'project_meta': 'META1'})
772 + self.assertEqual(samples[1].resource_metadata,
773 + {'project_name': 'project2',
774 + 'project_alias': 'p2',
775 + 'meta': 'm2',
776 + 'project_meta': 'META2'})
777 + self.assertEqual(samples[2].resource_metadata,
778 + {'project_name': 'project3',
779 + 'project_alias': 'p3',
780 + 'meta': 'm3',
781 + 'project_meta': 'META3'})
782 +
783 + @mock.patch('keystoneclient.v2_0.client.Client')
784 + def test_execute_request_extra_metadata_fields_different_requests(
785 + self, client_mock):
786 + definitions = copy.deepcopy(
787 + self.pollster_definition_only_required_fields)
788 +
789 + command = ''' \'\'\'echo '{"project":
790 + {"project_id": \'\'\'+ str(sample['project_id'])
791 + +\'\'\' , "name": "project1"}}' \'\'\' '''.replace('\n', '')
792 +
793 + command2 = ''' \'\'\'echo '{"project":
794 + {"project_id": \'\'\'+ str(sample['project_id'])
795 + +\'\'\' , "name": "project2"}}' \'\'\' '''.replace('\n', '')
796 +
797 + extra_metadata_fields_embedded = {
798 + 'name': "project_name2",
799 + 'host_command': command2,
800 + 'value': "name",
801 + }
802 +
803 + extra_metadata_fields = {
804 + 'name': "project_id2",
805 + 'host_command': command,
806 + 'value': "project_id",
807 + 'extra_metadata_fields': extra_metadata_fields_embedded
808 + }
809 +
810 + definitions['value_attribute'] = 'project_id'
811 + definitions['extra_metadata_fields'] = extra_metadata_fields
812 + pollster = dynamic_pollster.DynamicPollster(definitions)
813 +
814 + return_value = self.FakeResponse()
815 + return_value.status_code = requests.codes.ok
816 + return_value._text = '''
817 + {"projects": [
818 + {"project_id": 9999, "name": "project1"},
819 + {"project_id": 8888, "name": "project2"},
820 + {"project_id": 7777, "name": "project3"}]
821 + }
822 + '''
823 +
824 + def get(url, *args, **kwargs):
825 + return return_value
826 +
827 + client_mock.session.get = get
828 + manager = mock.Mock
829 + manager._keystone = client_mock
830 +
831 + def discover(*args, **kwargs):
832 + return ["https://endpoint.server.name/"]
833 +
834 + manager.discover = discover
835 + samples = pollster.get_samples(
836 + manager=manager, cache=None,
837 + resources=["https://endpoint.server.name/"])
838 +
839 + samples = list(samples)
840 + self.assertEqual(3, len(samples))
841 +
842 + self.assertEqual(samples[0].volume, 9999)
843 + self.assertEqual(samples[1].volume, 8888)
844 + self.assertEqual(samples[2].volume, 7777)
845 +
846 + self.assertEqual(samples[0].resource_metadata,
847 + {'project_id2': 9999,
848 + 'project_name2': 'project2'})
849 + self.assertEqual(samples[1].resource_metadata,
850 + {'project_id2': 8888,
851 + 'project_name2': 'project2'})
852 + self.assertEqual(samples[2].resource_metadata,
853 + {'project_id2': 7777,
854 + 'project_name2': 'project2'})
855 +
856 + @mock.patch('keystoneclient.v2_0.client.Client')
857 def test_execute_request_xml_json_response_handler_invalid_response(
858 self, client_mock):
859 definitions = copy.deepcopy(
860 @@ -410,8 +828,8 @@ class TestDynamicPollster(base.BaseTestC
861 keystone_client=client_mock,
862 resource="https://endpoint.server.name/")
863
864 - xml_handling_error = logs.output[2]
865 - json_handling_error = logs.output[3]
866 + xml_handling_error = logs.output[3]
867 + json_handling_error = logs.output[4]
868
869 self.assertIn(
870 'DEBUG:ceilometer.polling.dynamic_pollster:'
871 @@ -479,6 +897,57 @@ class TestDynamicPollster(base.BaseTestC
872 resource="https://endpoint.server.name/")
873 self.assertEqual("Mock HTTP error.", str(exception))
874
875 + def test_execute_host_command_paged_responses(self):
876 + definitions = copy.deepcopy(
877 + self.pollster_definition_only_required_fields)
878 + definitions['host_command'] = '''
879 + echo '{"server": [{"status": "ACTIVE"}], "next": ""}'
880 + '''
881 + str_json = "'{\\\"server\\\": [{\\\"status\\\": \\\"INACTIVE\\\"}]}'"
882 + definitions['next_sample_url_attribute'] = \
883 + "next|\"echo \"+value+\"" + str_json + '"'
884 + pollster = dynamic_pollster.DynamicPollster(definitions)
885 + samples = pollster.definitions.sample_gatherer. \
886 + execute_request_get_samples()
887 + resp_json = [{'status': 'ACTIVE'}, {'status': 'INACTIVE'}]
888 + self.assertEqual(resp_json, samples)
889 +
890 + def test_execute_host_command_response_handler(self):
891 + definitions = copy.deepcopy(
892 + self.pollster_definition_only_required_fields)
893 + definitions['response_handlers'] = ['xml', 'json']
894 + definitions['host_command'] = 'echo "<a><y>xml\n</y><s>xml</s></a>"'
895 + entry = 'a'
896 + definitions['response_entries_key'] = entry
897 + definitions.pop('url_path')
898 + definitions.pop('endpoint_type')
899 + pollster = dynamic_pollster.DynamicPollster(definitions)
900 +
901 + samples_xml = pollster.definitions.sample_gatherer. \
902 + execute_request_get_samples()
903 +
904 + definitions['host_command'] = 'echo \'{"a": {"y":"json",' \
905 + '\n"s":"json"}}\''
906 + samples_json = pollster.definitions.sample_gatherer. \
907 + execute_request_get_samples()
908 +
909 + resp_xml = {'a': {'y': 'xml', 's': 'xml'}}
910 + resp_json = {'a': {'y': 'json', 's': 'json'}}
911 + self.assertEqual(resp_xml[entry], samples_xml)
912 + self.assertEqual(resp_json[entry], samples_json)
913 +
914 + def test_execute_host_command_invalid_command(self):
915 + definitions = copy.deepcopy(
916 + self.pollster_definition_only_required_fields)
917 + definitions['host_command'] = 'invalid-command'
918 + definitions.pop('url_path')
919 + definitions.pop('endpoint_type')
920 + pollster = dynamic_pollster.DynamicPollster(definitions)
921 +
922 + self.assertRaises(
923 + declarative.InvalidResponseTypeException,
924 + pollster.definitions.sample_gatherer.execute_request_get_samples)
925 +
926 def test_generate_new_metadata_fields_no_metadata_mapping(self):
927 metadata = {'name': 'someName',
928 'value': 1}
929 @@ -1105,7 +1574,7 @@ class TestDynamicPollster(base.BaseTestC
930
931 sample = pollster.definitions.sample_extractor.generate_sample(
932 pollster_sample, pollster.definitions.configurations,
933 - manager=mock.Mock())
934 + manager=mock.Mock(), conf={})
935
936 self.assertEqual(1, sample.volume)
937 self.assertEqual(2, len(sample.resource_metadata))
938 @@ -1127,7 +1596,7 @@ class TestDynamicPollster(base.BaseTestC
939
940 sample = pollster.definitions.sample_extractor.generate_sample(
941 pollster_sample, pollster.definitions.configurations,
942 - manager=mock.Mock())
943 + manager=mock.Mock(), conf={})
944
945 self.assertEqual(1, sample.volume)
946 self.assertEqual(3, len(sample.resource_metadata))
947 @@ -1150,7 +1619,7 @@ class TestDynamicPollster(base.BaseTestC
948
949 sample = pollster.definitions.sample_extractor.generate_sample(
950 pollster_sample, pollster.definitions.configurations,
951 - manager=mock.Mock())
952 + manager=mock.Mock(), conf={})
953
954 self.assertEqual(1, sample.volume)
955 self.assertEqual(3, len(sample.resource_metadata))
956 Index: ceilometer/doc/source/admin/telemetry-dynamic-pollster.rst
957 ===================================================================
958 --- ceilometer.orig/doc/source/admin/telemetry-dynamic-pollster.rst
959 +++ ceilometer/doc/source/admin/telemetry-dynamic-pollster.rst
960 @@ -471,6 +471,62 @@ ones), we can use the `successful_ops`.
961 resource_id_attribute: "user"
962 response_entries_key: "summary"
963
964 +The dynamic pollsters system configuration (for local host commands)
965 +--------------------------------------------------------------------
966 +
967 +The dynamic pollster system can also be used for local host commands,
968 +these commands must be installed in the system that is running the
969 +Ceilometer compute agent.
970 +To configure local hosts commands, one can use all but two attributes of
971 +the Dynamic pollster system. The attributes that are not supported are
972 +the ``endpoint_type`` and ``url_path``. The dynamic pollster system for
973 +local host commands is activated automatically when one uses the
974 +configuration ``host_command``.
975 +
976 +The extra parameter (in addition to the original ones) that is available
977 +when using the local host commands dynamic pollster sub-subsystem is the
978 +following:
979 +
980 +* ``host_command``: required parameter. It is the host command that will
981 + be executed in the same host the Ceilometer dynamic pollster agent is
982 + running. The output of the command will be processed by the pollster and
983 + stored in the configured backend.
984 +
985 +As follows we present an example on how to use the local host command:
986 +
987 +.. code-block:: yaml
988 +
989 + ---
990 +
991 + - name: "dynamic.host.command"
992 + sample_type: "gauge"
993 + unit: "request"
994 + value_attribute: "value"
995 + response_entries_key: "test"
996 + host_command: "echo '<test><user_id>id1_u</user_id><project_id>id1_p</project_id><id>id1</id><meta>meta-data-to-store</meta><value>1</value></test>'"
997 + metadata_fields:
998 + - "meta"
999 + response_handlers:
1000 + - xml
1001 +
1002 +To execute multi page host commands, the `next_sample_url_attribute`
1003 +must generate the next sample command, like the following example:
1004 +
1005 +.. code-block:: yaml
1006 +
1007 + ---
1008 +
1009 + - name: "dynamic.s3.objects.size"
1010 + sample_type: "gauge"
1011 + unit: "request"
1012 + value_attribute: "Size"
1013 + project_id_attribute: "Owner.ID"
1014 + user_id_attribute: "Owner.ID"
1015 + resource_id_attribute: "Key"
1016 + response_entries_key: "Contents"
1017 + host_command: "aws s3api list-objects"
1018 + next_sample_url_attribute: NextToken | 'aws s3api list-objects --starting-token "' + value + '"'
1019 +
1020 Operations on extracted attributes
1021 ----------------------------------
1022
1023 @@ -876,12 +932,10 @@ we only have the `tenant_id`, which must
1024 for billing and later invoicing one might need/want the project name, domain
1025 id, and other metadata that are available in Keystone (and maybe some others
1026 that are scattered over other components). To achieve that, one can use the
1027 -OpenStack metadata enrichment option. This feature is only available
1028 -to *OpenStack pollsters*, and can only gather extra metadata from OpenStack
1029 -APIs. As follows we present an example that shows a dynamic pollster
1030 -configuration to gather virtual machine (VM) status, and to enrich the data
1031 -pushed to the storage backend (e.g. Gnocchi) with project name, domain ID,
1032 -and domain name.
1033 +OpenStack metadata enrichment option. As follows we present an example that
1034 +shows a dynamic pollster configuration to gather virtual machine (VM) status,
1035 +and to enrich the data pushed to the storage backend (e.g. Gnocchi) with
1036 +project name, domain ID, and domain name.
1037
1038 .. code-block:: yaml
1039
1040 @@ -937,26 +991,59 @@ and domain name.
1041 "Openstack-API-Version": "identity latest"
1042 value: "name"
1043 extra_metadata_fields_cache_seconds: 1800 # overriding the default cache policy
1044 + metadata_fields:
1045 + - id
1046 - name: "domain_id"
1047 endpoint_type: "identity"
1048 url_path: "'/v3/projects/' + str(sample['project_id'])"
1049 headers:
1050 "Openstack-API-Version": "identity latest"
1051 value: "domain_id"
1052 + metadata_fields:
1053 + - id
1054 - name: "domain_name"
1055 endpoint_type: "identity"
1056 url_path: "'/v3/domains/' + str(extra_metadata_captured['domain_id'])"
1057 headers:
1058 "Openstack-API-Version": "identity latest"
1059 value: "name"
1060 + metadata_fields:
1061 + - id
1062 + - name: "operating-system"
1063 + host_command: "'get-vm --vm-name ' + str(extra_metadata_by_name['project_name']['metadata']['id'])"
1064 + value: "os"
1065 +
1066
1067 The above example can be used to gather and persist in the backend the
1068 status of VMs. It will persist `1` in the backend as a measure for every
1069 collecting period if the VM's status is `ACTIVE`, and `0` otherwise. This is
1070 quite useful to create hashmap rating rules for running VMs in CloudKitty.
1071 Then, to enrich the resource in the storage backend, we are adding extra
1072 -metadata that are collected in Keystone via the `extra_metadata_fields`
1073 -options.
1074 +metadata that are collected in Keystone and in the local host via the
1075 +`extra_metadata_fields` options. If you have multiples `extra_metadata_fields`
1076 +defining the same `metadata_field`, the last not `None` metadata value will
1077 +be used.
1078 +
1079 +To operate values in the `extra_metadata_fields`, you can access 3 local
1080 +variables:
1081 +
1082 +* ``sample``: it is a dictionary which holds the current data of the root
1083 + sample. The root sample is the final sample that will be persisted in the
1084 + configured storage backend.
1085 +
1086 +* ``extra_metadata_captured``: it is a dictionary which holds the current
1087 + data of all `extra_metadata_fields` processed before this one.
1088 + If you have multiples `extra_metadata_fields` defining the same
1089 + `metadata_field`, the last not `None` metadata value will be used.
1090 +
1091 +* ``extra_metadata_by_name``: it is a dictionary which holds the data of
1092 + all `extra_metadata_fields` processed before this one. No data is
1093 + overwritten in this variable. To access an specific `extra_metadata_field`
1094 + using this variable, you can do
1095 + `extra_metadata_by_name['<extra_metadata_field_name>']['value']` to get
1096 + its value, or
1097 + `extra_metadata_by_name['<extra_metadata_field_name>']['metadata']['<metadata>']`
1098 + to get its metadata.
1099
1100 The metadata enrichment feature has the following options:
1101
1102 @@ -969,41 +1056,13 @@ The metadata enrichment feature has the
1103 value can be increased of decreased.
1104
1105 * ``extra_metadata_fields``: optional parameter. This option is a list of
1106 - objects, where each one of its elements is an extra metadata definition.
1107 - Each one of the extra metadata definition can have the options defined in
1108 - the dynamic pollsters such as to handle paged responses, operations on the
1109 - extracted values, headers and so on. The basic options that must be
1110 - defined for an extra metadata definitions are the following:
1111 -
1112 - * ``name``: This option is mandatory. The name of the extra metadata.
1113 - This is the name that is going to be used by the metadata. If there is
1114 - already any other metadata gathered via `metadata_fields` option or
1115 - transformed via `metadata_mapping` configuration, this metadata is
1116 - going to be discarded.
1117 -
1118 - * ``endpoint_type``: The endpoint type that we want to execute the
1119 - call against. This option is mandatory. It works similarly to the
1120 - `endpoint_type` option in the dynamic pollster definition.
1121 -
1122 - * ``url_path``: This option is mandatory. It works similarly to the
1123 - `url_path` option in the dynamic pollster definition. However, this
1124 - `one enables operators to execute/evaluate expressions in runtime, which
1125 - `allows one to retrieve the information from previously gathered
1126 - metadata via ``extra_metadata_captured` dictionary, or via the
1127 - `sample` itself.
1128 -
1129 - * ``value``: This configuration is mandatory. It works similarly to the
1130 - `value_attribute` option in the dynamic pollster definition. It is
1131 - the value we want to extract from the response, and assign in the
1132 - metadata being generated.
1133 -
1134 - * ``headers``: This option is optional. It works similarly to the
1135 - `headers` option in the dynamic pollster definition.
1136 -
1137 - * ``next_sample_url_attribute``: This option is optional. It works
1138 - similarly to the `next_sample_url_attribute` option in the dynamic
1139 - pollster definition.
1140 -
1141 - * ``response_entries_key``: This option is optional. It works
1142 - similarly to the `response_entries_key` option in the dynamic
1143 - pollster definition.
1144 + objects or a single one, where each one of its elements is an
1145 + dynamic pollster configuration set. Each one of the extra metadata
1146 + definition can have the same options defined in the dynamic pollsters,
1147 + including the `extra_metadata_fields` option, so this option is a
1148 + multi-level option. When defined, the result of the collected data will
1149 + be merged in the final sample resource metadata. If some of the required
1150 + dynamic pollster configuration is not set in the `extra_metadata_fields`,
1151 + will be used the parent pollster configuration, except the `name`.
1152 +
1153 +
1154 Index: ceilometer/setup.cfg
1155 ===================================================================
1156 --- ceilometer.orig/setup.cfg
1157 +++ ceilometer/setup.cfg
1158 @@ -45,6 +45,7 @@ ceilometer.sample.endpoint =
1159
1160 ceilometer.discover.compute =
1161 local_instances = ceilometer.compute.discovery:InstanceDiscovery
1162 + local_node = ceilometer.polling.discovery.localnode:LocalNodeDiscovery
1163
1164 ceilometer.discover.central =
1165 barbican = ceilometer.polling.discovery.non_openstack_credentials_discovery:NonOpenStackCredentialsDiscovery
+0
-176
debian/patches/Add_support_to_namespaces_on_dynamic_pollsters.patch less more
0 Description: Add support to namespaces on dynamic pollsters
1 Problem description
2 ===================
3 The hardcoded pollsters are defined by namespaces, so they
4 are instantied based on the namespaces provided to the
5 'AgentManager'.
6 .
7 The dynamic pollsters, on the other hand, are always instantied,
8 independent of the provided namespaces.
9 .
10 Proposal
11 ========
12 To allow operators to define in which namespaces the dynamic
13 pollster will be deployed, we propose to add a new configuration
14 'namespaces' in the dynamic pollsters yaml configuration.
15 This configuration will support a single entry or a list of the
16 namespaces that the pollster must be instantiated.
17 Author: Pedro Henrique <phpm13@gmail.com>
18 Date: Mon, 05 Sep 2022 12:09:19 -0300
19 Change-Id: I39ba0c3dd312a0601e02f8cfcab7a44e585a8a7f
20 Origin: upstream, https://review.opendev.org/c/openstack/ceilometer/+/855953
21 Last-Update: 2022-10-24
22
23 Index: ceilometer/ceilometer/polling/manager.py
24 ===================================================================
25 --- ceilometer.orig/ceilometer/polling/manager.py
26 +++ ceilometer/ceilometer/polling/manager.py
27 @@ -253,7 +253,8 @@ class AgentManager(cotyledon.Service):
28 for namespace in namespaces)
29
30 # Create dynamic pollsters
31 - extensions_dynamic_pollsters = self.create_dynamic_pollsters()
32 + extensions_dynamic_pollsters = self.create_dynamic_pollsters(
33 + namespaces)
34
35 self.extensions = list(itertools.chain(*list(extensions))) + list(
36 itertools.chain(*list(extensions_fb))) + list(
37 @@ -291,15 +292,18 @@ class AgentManager(cotyledon.Service):
38 self._keystone = None
39 self._keystone_last_exception = None
40
41 - def create_dynamic_pollsters(self):
42 + def create_dynamic_pollsters(self, namespaces):
43 """Creates dynamic pollsters
44
45 This method Creates dynamic pollsters based on configurations placed on
46 'pollsters_definitions_dirs'
47
48 + :param namespaces: The namespaces we are running on to validate if
49 + the pollster should be instantiated or not.
50 :return: a list with the dynamic pollsters defined by the operator.
51 """
52
53 + namespaces_set = set(namespaces)
54 pollsters_definitions_dirs = self.conf.pollsters_definitions_dirs
55 if not pollsters_definitions_dirs:
56 LOG.info("Variable 'pollsters_definitions_dirs' not defined.")
57 @@ -333,6 +337,21 @@ class AgentManager(cotyledon.Service):
58
59 for pollster_cfg in pollsters_cfg:
60 pollster_name = pollster_cfg['name']
61 + pollster_namespaces = pollster_cfg.get(
62 + 'namespaces', ['central'])
63 + if isinstance(pollster_namespaces, list):
64 + pollster_namespaces = set(pollster_namespaces)
65 + else:
66 + pollster_namespaces = {pollster_namespaces}
67 +
68 + if not bool(namespaces_set & pollster_namespaces):
69 + LOG.info("The pollster [%s] is not configured to run in "
70 + "these namespaces %s, the configured namespaces "
71 + "for this pollster are %s. Therefore, we are "
72 + "skipping it.", pollster_name, namespaces_set,
73 + pollster_namespaces)
74 + continue
75 +
76 if pollster_name not in pollsters_definitions:
77 LOG.info("Loading dynamic pollster [%s] from file [%s].",
78 pollster_name, pollsters_definitions_file)
79 Index: ceilometer/ceilometer/tests/unit/polling/test_manager.py
80 ===================================================================
81 --- ceilometer.orig/ceilometer/tests/unit/polling/test_manager.py
82 +++ ceilometer/ceilometer/tests/unit/polling/test_manager.py
83 @@ -422,6 +422,76 @@ class TestPollingAgent(BaseAgent):
84 self.assertIn(60, polling_tasks.keys())
85 self.assertNotIn(10, polling_tasks.keys())
86
87 + @mock.patch('glob.glob')
88 + @mock.patch('ceilometer.declarative.load_definitions')
89 + def test_setup_polling_dynamic_pollster_namespace(self, load_mock,
90 + glob_mock):
91 + glob_mock.return_value = ['test.yml']
92 + load_mock.return_value = [{
93 + 'name': "test.dynamic.pollster",
94 + 'namespaces': "dynamic",
95 + 'sample_type': 'gauge',
96 + 'unit': 'test',
97 + 'endpoint_type': 'test',
98 + 'url_path': 'test',
99 + 'value_attribute': 'test'
100 + }, {
101 + 'name': "test.compute.central.pollster",
102 + 'sample_type': 'gauge',
103 + 'namespaces': ["compute", "central"],
104 + 'unit': 'test',
105 + 'endpoint_type': 'test',
106 + 'url_path': 'test',
107 + 'value_attribute': 'test'
108 + }, {
109 + 'name': "test.compute.pollster",
110 + 'namespaces': ["compute"],
111 + 'sample_type': 'gauge',
112 + 'unit': 'test',
113 + 'endpoint_type': 'test',
114 + 'url_path': 'test',
115 + 'value_attribute': 'test'
116 + }, {
117 + 'name': "test.central.pollster",
118 + 'sample_type': 'gauge',
119 + 'unit': 'test',
120 + 'endpoint_type': 'test',
121 + 'url_path': 'test',
122 + 'value_attribute': 'test'
123 + }]
124 + mgr = manager.AgentManager(0, self.CONF, namespaces=['dynamic'])
125 + self.assertEqual(len(mgr.extensions), 1)
126 + self.assertEqual(
127 + mgr.extensions[0].definitions.configurations['name'],
128 + 'test.dynamic.pollster')
129 +
130 + mgr = manager.AgentManager(0, self.CONF)
131 + self.assertEqual(
132 + mgr.extensions[-3].definitions.configurations['name'],
133 + 'test.compute.central.pollster')
134 + self.assertEqual(
135 + mgr.extensions[-2].definitions.configurations['name'],
136 + 'test.compute.pollster')
137 + self.assertEqual(
138 + mgr.extensions[-1].definitions.configurations['name'],
139 + 'test.central.pollster')
140 +
141 + mgr = manager.AgentManager(0, self.CONF, namespaces=['compute'])
142 + self.assertEqual(
143 + mgr.extensions[-2].definitions.configurations['name'],
144 + 'test.compute.central.pollster')
145 + self.assertEqual(
146 + mgr.extensions[-1].definitions.configurations['name'],
147 + 'test.compute.pollster')
148 +
149 + mgr = manager.AgentManager(0, self.CONF, ['central'])
150 + self.assertEqual(
151 + mgr.extensions[-2].definitions.configurations['name'],
152 + 'test.compute.central.pollster')
153 + self.assertEqual(
154 + mgr.extensions[-1].definitions.configurations['name'],
155 + 'test.central.pollster')
156 +
157 def test_setup_polling_task_same_interval(self):
158 self.polling_cfg['sources'].append({
159 'name': 'test_polling_1',
160 Index: ceilometer/doc/source/admin/telemetry-dynamic-pollster.rst
161 ===================================================================
162 --- ceilometer.orig/doc/source/admin/telemetry-dynamic-pollster.rst
163 +++ ceilometer/doc/source/admin/telemetry-dynamic-pollster.rst
164 @@ -198,6 +198,11 @@ attributes to define a dynamic pollster:
165 executed serially, one after the other. Therefore, if the request hangs,
166 all pollsters (including the non-dynamic ones) will stop executing.
167
168 +* ``namespaces``: optional parameter. Defines the namespaces (running
169 + ceilometer instances) where the pollster will be instantiated. This
170 + parameter accepts a single string value or a list of strings. The
171 + default value is `central`.
172 +
173 The complete YAML configuration to gather data from Magnum (that has been used
174 as an example) is the following:
175
+0
-3
debian/patches/series less more
00 install-missing-files.patch
1 Add_response_handlers_to_support_different_response_types.patch
2 Add_support_to_namespaces_on_dynamic_pollsters.patch
3 Add_support_to_host_command_dynamic_pollster_definitions.patch