Codebase list python-osprofiler / eb6376c
Merge "OSprofiler with Jaeger Tracing as backend" Zuul authored 5 years ago Gerrit Code Review committed 5 years ago
10 changed file(s) with 297 addition(s) and 6 deletion(s). Raw diff Collapse all Expand all
+2
-0
lower-constraints.txt less more
00 coverage===4.0
11 ddt===1.0.1
22 elasticsearch===2.0.0
3 futures===3.0.0
4 jaeger-client==3.8.0
35 mock===2.0.0
46 netaddr===0.7.18
57 openstackdocstheme===1.18.1
+12
-0
osprofiler/_utils.py less more
1717 import hmac
1818 import json
1919 import os
20 import uuid
2021
2122 from oslo_utils import secretutils
23 from oslo_utils import uuidutils
2224 import six
2325
2426
146148 new_package = ".".join(root.split(os.sep)).split("....")[1]
147149 module_name = "%s.%s" % (new_package, filename[:-3])
148150 __import__(module_name)
151
152
153 def shorten_id(span_id):
154 """Convert from uuid4 to 64 bit id for OpenTracing"""
155 try:
156 short_id = uuid.UUID(span_id).int & (1 << 64) - 1
157 except ValueError:
158 # Return a new short id for this
159 short_id = shorten_id(uuidutils.generate_uuid())
160 return short_id
+1
-0
osprofiler/drivers/__init__.py less more
00 from osprofiler.drivers import base # noqa
11 from osprofiler.drivers import elasticsearch_driver # noqa
2 from osprofiler.drivers import jaeger # noqa
23 from osprofiler.drivers import loginsight # noqa
34 from osprofiler.drivers import messaging # noqa
45 from osprofiler.drivers import mongodb # noqa
+147
-0
osprofiler/drivers/jaeger.py less more
0 # Copyright 2018 Fujitsu Ltd.
1 # All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 import collections
16 import datetime
17 import time
18
19 from oslo_config import cfg
20 from oslo_serialization import jsonutils
21 import six.moves.urllib.parse as parser
22
23 from osprofiler import _utils as utils
24 from osprofiler.drivers import base
25 from osprofiler import exc
26
27
28 class Jaeger(base.Driver):
29 def __init__(self, connection_str, project=None, service=None, host=None,
30 conf=cfg.CONF, **kwargs):
31 """Jaeger driver for OSProfiler."""
32
33 super(Jaeger, self).__init__(connection_str, project=project,
34 service=service, host=host,
35 conf=conf, **kwargs)
36 try:
37 import jaeger_client
38 self.jaeger_client = jaeger_client
39 except ImportError:
40 raise exc.CommandError(
41 "To use OSProfiler with Uber Jaeger tracer, "
42 "you have to install `jaeger-client` manually. "
43 "Install with pip:\n `pip install jaeger-client`."
44 )
45
46 parsed_url = parser.urlparse(connection_str)
47 cfg = {
48 "local_agent": {
49 "reporting_host": parsed_url.hostname,
50 "reporting_port": parsed_url.port,
51 }
52 }
53
54 # Initialize tracer for each profiler
55 service_name = "{}-{}".format(project, service)
56 config = jaeger_client.Config(cfg, service_name=service_name)
57 self.tracer = config.initialize_tracer()
58
59 self.spans = collections.deque()
60
61 @classmethod
62 def get_name(cls):
63 return "jaeger"
64
65 def notify(self, payload):
66 if payload["name"].endswith("start"):
67 timestamp = datetime.datetime.strptime(payload["timestamp"],
68 "%Y-%m-%dT%H:%M:%S.%f")
69 epoch = datetime.datetime.utcfromtimestamp(0)
70 start_time = (timestamp - epoch).total_seconds()
71
72 # Create parent span
73 child_of = self.jaeger_client.SpanContext(
74 trace_id=utils.shorten_id(payload["base_id"]),
75 span_id=utils.shorten_id(payload["parent_id"]),
76 parent_id=None,
77 flags=self.jaeger_client.span.SAMPLED_FLAG
78 )
79
80 # Create Jaeger Tracing span
81 span = self.tracer.start_span(
82 operation_name=payload["name"].rstrip("-start"),
83 child_of=child_of,
84 tags=self.create_span_tags(payload),
85 start_time=start_time
86 )
87
88 # Replace Jaeger Tracing span_id (random id) to OSProfiler span_id
89 span.context.span_id = utils.shorten_id(payload["trace_id"])
90 self.spans.append(span)
91 else:
92 span = self.spans.pop()
93
94 # Store result of db call and function call
95 for call in ("db", "function"):
96 if payload.get("info", {}).get(call) is not None:
97 span.set_tag("result", payload["info"][call]["result"])
98
99 # Span error tag and log
100 if payload["info"].get("etype") is not None:
101 span.set_tag("error", True)
102 span.log_kv({"error.kind": payload["info"]["etype"]})
103 span.log_kv({"message": payload["info"]["message"]})
104
105 span.finish(finish_time=time.time())
106
107 def get_report(self, base_id):
108 """Please use Jaeger Tracing UI for this task."""
109 return self._parse_results()
110
111 def list_traces(self, fields=None):
112 """Please use Jaeger Tracing UI for this task."""
113 return []
114
115 def list_error_traces(self):
116 """Please use Jaeger Tracing UI for this task."""
117 return []
118
119 def create_span_tags(self, payload):
120 """Create tags for OpenTracing span.
121
122 :param info: Information from OSProfiler trace.
123 :returns tags: A dictionary contains standard tags
124 from OpenTracing sematic conventions,
125 and some other custom tags related to http, db calls.
126 """
127 tags = {}
128 info = payload["info"]
129
130 if info.get("db"):
131 # DB calls
132 tags["db.statement"] = info["db"]["statement"]
133 tags["db.params"] = jsonutils.dumps(info["db"]["params"])
134 elif info.get("request"):
135 # WSGI call
136 tags["http.path"] = info["request"]["path"]
137 tags["http.query"] = info["request"]["query"]
138 tags["http.method"] = info["request"]["method"]
139 tags["http.scheme"] = info["request"]["scheme"]
140 elif info.get("function"):
141 # RPC, function calls
142 tags["args"] = info["function"]["args"]
143 tags["kwargs"] = info["function"]["kwargs"]
144 tags["name"] = info["function"]["name"]
145
146 return tags
+5
-4
osprofiler/opts.py less more
8686
8787 Examples of possible values:
8888
89 * messaging://: use oslo_messaging driver for sending notifications.
90 * mongodb://127.0.0.1:27017 : use mongodb driver for sending notifications.
91 * elasticsearch://127.0.0.1:9200 : use elasticsearch driver for sending
92 notifications.
89 * messaging:// - use oslo_messaging driver for sending spans.
90 * redis://127.0.0.1:6379 - use redis driver for sending spans.
91 * mongodb://127.0.0.1:27017 - use mongodb driver for sending spans.
92 * elasticsearch://127.0.0.1:9200 - use elasticsearch driver for sending spans.
93 * jaeger://127.0.0.1:6831 - use jaeger tracing as driver for sending spans.
9394 """)
9495
9596 _es_doc_type_opt = cfg.StrOpt(
+13
-1
osprofiler/profiler.py less more
2222 from oslo_utils import reflection
2323 from oslo_utils import uuidutils
2424
25 from osprofiler import _utils as utils
2526 from osprofiler import notifier
2627
2728
342343
343344 def __exit__(self, etype, value, traceback):
344345 if etype:
345 info = {"etype": reflection.get_class_name(etype)}
346 info = {
347 "etype": reflection.get_class_name(etype),
348 "message": value.args[0] if value.args else None
349 }
346350 stop(info=info)
347351 else:
348352 stop()
357361 self._trace_stack = collections.deque([base_id, parent_id or base_id])
358362 self._name = collections.deque()
359363 self._host = socket.gethostname()
364
365 def get_shorten_id(self, uuid_id):
366 """Return shorten id of a uuid that will be used in OpenTracing drivers
367
368 :param uuid_id: A string of uuid that was generated by uuidutils
369 :returns: A shorter 64-bit long id
370 """
371 return format(utils.shorten_id(uuid_id), "x")
360372
361373 def get_base_id(self):
362374 """Return base id of a trace.
+78
-0
osprofiler/tests/unit/drivers/test_jaeger.py less more
0 # Copyright 2018 Fujitsu Ltd.
1 # All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 import mock
16
17 from osprofiler.drivers import jaeger
18 from osprofiler.tests import test
19
20
21 class JaegerTestCase(test.TestCase):
22
23 def setUp(self):
24 super(JaegerTestCase, self).setUp()
25 self.payload_start = {
26 "name": "api-start",
27 "base_id": "4e3e0ec6-2938-40b1-8504-09eb1d4b0dee",
28 "trace_id": "1c089ea8-28fe-4f3d-8c00-f6daa2bc32f1",
29 "parent_id": "e2715537-3d1c-4f0c-b3af-87355dc5fc5b",
30 "timestamp": "2018-05-03T04:31:51.781381",
31 "info": {
32 "host": "test"
33 }
34 }
35
36 self.payload_stop = {
37 "name": "api-stop",
38 "base_id": "4e3e0ec6-2938-40b1-8504-09eb1d4b0dee",
39 "trace_id": "1c089ea8-28fe-4f3d-8c00-f6daa2bc32f1",
40 "parent_id": "e2715537-3d1c-4f0c-b3af-87355dc5fc5b",
41 "timestamp": "2018-05-03T04:31:51.781381",
42 "info": {
43 "host": "test",
44 "function": {
45 "result": 1
46 }
47 }
48 }
49
50 self.driver = jaeger.Jaeger("jaeger://127.0.0.1:6831",
51 project="nova", service="api")
52
53 @mock.patch("osprofiler._utils.shorten_id")
54 def test_notify_start(self, mock_shorten_id):
55 self.driver.notify(self.payload_start)
56 calls = [
57 mock.call(self.payload_start["base_id"]),
58 mock.call(self.payload_start["parent_id"]),
59 mock.call(self.payload_start["trace_id"])
60 ]
61 mock_shorten_id.assert_has_calls(calls, any_order=True)
62
63 @mock.patch("jaeger_client.span.Span")
64 @mock.patch("time.time")
65 def test_notify_stop(self, mock_time, mock_span):
66 fake_time = 1525416065.5958152
67 mock_time.return_value = fake_time
68
69 span = mock_span()
70 self.driver.spans.append(mock_span())
71
72 self.driver.notify(self.payload_stop)
73
74 mock_time.assert_called_once()
75 mock_time.reset_mock()
76
77 span.finish.assert_called_once_with(finish_time=fake_time)
+11
-1
osprofiler/tests/unit/test_profiler.py less more
6161
6262 class ProfilerTestCase(test.TestCase):
6363
64 def test_profiler_get_shorten_id(self):
65 uuid_id = "4e3e0ec6-2938-40b1-8504-09eb1d4b0dee"
66 prof = profiler._Profiler("secret", base_id="1", parent_id="2")
67 result = prof.get_shorten_id(uuid_id)
68 expected = "850409eb1d4b0dee"
69 self.assertEqual(expected, result)
70
6471 def test_profiler_get_base_id(self):
6572 prof = profiler._Profiler("secret", base_id="1", parent_id="2")
6673 self.assertEqual(prof.get_base_id(), "1")
166173
167174 self.assertRaises(ValueError, foo)
168175 mock_start.assert_called_once_with("foo", info=None)
169 mock_stop.assert_called_once_with(info={"etype": "ValueError"})
176 mock_stop.assert_called_once_with(info={
177 "etype": "ValueError",
178 "message": "bar"
179 })
170180
171181
172182 @profiler.trace("function", info={"info": "some_info"})
+24
-0
osprofiler/tests/unit/test_utils.py less more
1515 import base64
1616 import hashlib
1717 import hmac
18 import uuid
1819
1920 import mock
2021
110111
111112 self.assertIsNone(utils.signed_unpack(data, hmac_data, hmac))
112113
114 def test_shorten_id_with_valid_uuid(self):
115 valid_id = "4e3e0ec6-2938-40b1-8504-09eb1d4b0dee"
116
117 uuid_obj = uuid.UUID(valid_id)
118
119 with mock.patch("uuid.UUID") as mock_uuid:
120 mock_uuid.return_value = uuid_obj
121
122 result = utils.shorten_id(valid_id)
123 expected = 9584796812364680686
124
125 self.assertEqual(expected, result)
126
127 @mock.patch("oslo_utils.uuidutils.generate_uuid")
128 def test_shorten_id_with_invalid_uuid(self, mock_gen_uuid):
129 invalid_id = "invalid"
130 mock_gen_uuid.return_value = "1c089ea8-28fe-4f3d-8c00-f6daa2bc32f1"
131
132 result = utils.shorten_id(invalid_id)
133 expected = 10088334584203457265
134
135 self.assertEqual(expected, result)
136
113137 def test_itersubclasses(self):
114138
115139 class A(object):
+4
-0
test-requirements.txt less more
2121
2222 # Build release notes
2323 reno # Apache-2.0
24
25 # For Jaeger Tracing
26 jaeger-client # Apache-2.0
27 futures;python_version=='2.7' or python_version=='2.6' # PSF