diff --git a/.gitignore b/.gitignore
index 6e7e085..248834e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,11 +1,12 @@
+AUTHORS
+ChangeLog
*.py[cod]
# C extensions
*.so
# Packages
-*.egg
-*.egg-info
+*.egg*
dist
build
_build
@@ -37,3 +38,7 @@
.mr.developer.cfg
.project
.pydevproject
+.idea
+
+# reno build
+releasenotes/build
diff --git a/.testr.conf b/.testr.conf
index 27df784..7c90e7d 100644
--- a/.testr.conf
+++ b/.testr.conf
@@ -1,4 +1,4 @@
[DEFAULT]
-test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./osprofiler/tests $LISTOPT $IDOPTION
+test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./osprofiler/tests/unit} $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
new file mode 100644
index 0000000..23bcb4d
--- /dev/null
+++ b/CONTRIBUTING.rst
@@ -0,0 +1,16 @@
+If you would like to contribute to the development of OpenStack,
+you must follow the steps in this page:
+
+ http://docs.openstack.org/infra/manual/developers.html
+
+Once those steps have been completed, changes to OpenStack
+should be submitted for review via the Gerrit tool, following
+the workflow documented at:
+
+ http://docs.openstack.org/infra/manual/developers.html#development-workflow
+
+Pull requests submitted through GitHub will be ignored.
+
+Bugs should be filed on Launchpad, not GitHub:
+
+ https://bugs.launchpad.net/osprofiler
\ No newline at end of file
diff --git a/README.rst b/README.rst
index 15dd701..9f1bcab 100644
--- a/README.rst
+++ b/README.rst
@@ -1,342 +1,32 @@
-OSProfiler
-==========
+========================
+Team and repository tags
+========================
-OSProfiler is an OpenStack cross-project profiling library.
+.. image:: http://governance.openstack.org/badges/osprofiler.svg
+ :target: http://governance.openstack.org/reference/tags/index.html
+.. Change things from this point on
-Background
-----------
+===========================================================
+ OSProfiler -- Library for cross-project profiling library
+===========================================================
-OpenStack consists of multiple projects. Each project, in turn, is composed of
-multiple services. To process some request, e.g. to boot a virtual machine,
-OpenStack uses multiple services from different projects. In the case something
-works too slowly, it's extremely complicated to understand what exactly goes
-wrong and to locate the bottleneck.
+.. image:: https://img.shields.io/pypi/v/osprofiler.svg
+ :target: https://pypi.python.org/pypi/osprofiler/
+ :alt: Latest Version
-To resolve this issue, we introduce a tiny but powerful library,
-**osprofiler**, that is going to be used by all OpenStack projects and their
-python clients. To be able to generate 1 trace per request, that goes through
-all involved services, and builds a tree of calls (see an
-`example `_).
+.. image:: https://img.shields.io/pypi/dm/osprofiler.svg
+ :target: https://pypi.python.org/pypi/osprofiler/
+ :alt: Downloads
+OSProfiler provides a tiny but powerful library that is used by
+most (soon to be all) OpenStack projects and their python clients. It
+provides functionality to be able to generate 1 trace per request, that goes
+through all involved services. This trace can then be extracted and used
+to build a tree of calls which can be quite handy for a variety of
+reasons (for example in isolating cross-project performance issues).
-Why not cProfile and etc?
--------------------------
-
-**The scope of this library is quite different:**
-
-* We are interested in getting one trace of points from different service,
- not tracing all python calls inside one process.
-
-* This library should be easy integratable in OpenStack. This means that:
-
- * It shouldn't require too many changes in code bases of integrating
- projects.
-
- * We should be able to turn it off fully.
-
- * We should be able to keep it turned on in lazy mode in production
- (e.g. admin should be able to "trace" on request).
-
-
-OSprofiler API version 0.3.0
-----------------------------
-
-There are a couple of things that you should know about API before using it.
-
-
-* **4 ways to add a new trace point**
-
- .. parsed-literal::
-
- from osprofiler import profiler
-
- def some_func():
- profiler.start("point_name", {"any_key": "with_any_value"})
- # your code
- profiler.stop({"any_info_about_point": "in_this_dict"})
-
-
- @profiler.trace("point_name",
- info={"any_info_about_point": "in_this_dict"},
- hide_args=False)
- def some_func2(*args, **kwargs):
- # If you need to hide args in profile info, put hide_args=True
- pass
-
- def some_func3():
- with profiler.Trace("point_name",
- info={"any_key": "with_any_value"}):
- # some code here
-
- @profiler.trace_cls("point_name", info={}, hide_args=False,
- trace_private=False)
- class TracedClass(object):
-
- def traced_method(self):
- pass
-
- def _traced_only_if_trace_private_true(self):
- pass
-
-* **How profiler works?**
-
- * **@profiler.Trace()** and **profiler.trace()** are just syntax sugar,
- that just calls **profiler.start()** & **profiler.stop()** methods.
-
- * Every call of **profiler.start()** & **profiler.stop()** sends to
- **collector** 1 message. It means that every trace point creates 2 records
- in the collector. *(more about collector & records later)*
-
- * Nested trace points are supported. The sample below produces 2 trace points:
-
- .. parsed-literal::
-
- profiler.start("parent_point")
- profiler.start("child_point")
- profiler.stop()
- profiler.stop()
-
- The implementation is quite simple. Profiler has one stack that contains
- ids of all trace points. E.g.:
-
- .. parsed-literal::
-
- profiler.start("parent_point") # trace_stack.push()
- # send to collector -> trace_stack[-2:]
-
- profiler.start("parent_point") # trace_stack.push()
- # send to collector -> trace_stack[-2:]
- profiler.stop() # send to collector -> trace_stack[-2:]
- # trace_stack.pop()
-
- profiler.stop() # send to collector -> trace_stack[-2:]
- # trace_stack.pop()
-
- It's simple to build a tree of nested trace points, having
- **(parent_id, point_id)** of all trace points.
-
-* **Process of sending to collector**
-
- Trace points contain 2 messages (start and stop). Messages like below are
- sent to a collector:
-
- .. parsed-literal::
- {
- "name": -(start|stop)
- "base_id": ,
- "parent_id": ,
- "trace_id": ,
- "info":
- }
-
- * base_id - that is equal for all trace points that belong
- to one trace, this is done to simplify the process of retrieving
- all trace points related to one trace from collector
- * parent_id - of parent trace point
- * trace_id - of current trace point
- * info - the dictionary that contains user information passed when calling
- profiler **start()** & **stop()** methods.
-
-
-
-* **Setting up the collector.**
-
- The profiler doesn't include a trace point collector. The user/developer
- should instead provide a method that sends messages to a collector. Let's
- take a look at a trivial sample, where the collector is just a file:
-
- .. parsed-literal::
-
- import json
-
- from osprofiler import notifier
-
- def send_info_to_file_collector(info, context=None):
- with open("traces", "a") as f:
- f.write(json.dumps(info))
-
- notifier.set(send_info_to_file_collector)
-
- So now on every **profiler.start()** and **profiler.stop()** call we will
- write info about the trace point to the end of the **traces** file.
-
-
-* **Initialization of profiler.**
-
- If profiler is not initialized, all calls to **profiler.start()** and
- **profiler.stop()** will be ignored.
-
- Initialization is a quite simple procedure.
-
- .. parsed-literal::
-
- from osprofiler import profiler
-
- profiler.init("SECRET_HMAC_KEY", base_id=, parent_id=)
-
- ``SECRET_HMAC_KEY`` - will be discussed later, because it's related to the
- integration of OSprofiler & OpenStack.
-
- **base_id** and **trace_id** will be used to initialize stack_trace in
- profiler, e.g. stack_trace = [base_id, trace_id].
-
-
-* **OSProfiler CLI.**
-
- To make it easier for end users to work with profiler from CLI, osprofiler
- has entry point that allows them to retrieve information about traces and
- present it in human readable from.
-
- Available commands:
-
- * Help message with all available commands and their arguments:
-
- .. parsed-literal::
-
- $ osprofiler -h/--help
-
- * OSProfiler version:
-
- .. parsed-literal::
-
- $ osprofiler -v/--version
-
- * Results of profiling can be obtained in JSON (option: ``--json``) and HTML
- (option: ``--html``) formats:
-
- .. parsed-literal::
-
- $ osprofiler trace show --json/--html
-
- hint: option ``--out`` will redirect result of ``osprofiler trace show``
- in specified file:
-
- .. parsed-literal::
-
- $ osprofiler trace show --json/--html --out /path/to/file
-
-Integration with OpenStack
---------------------------
-
-There are 4 topics related to integration OSprofiler & `OpenStack`_:
-
-* **What we should use as a centralized collector?**
-
- We decided to use `Ceilometer`_, because:
-
- * It's already integrated in OpenStack, so it's quite simple to send
- notifications to it from all projects.
-
- * There is an OpenStack API in Ceilometer that allows us to retrieve all
- messages related to one trace. Take a look at
- *osprofiler.parsers.ceilometer:get_notifications*
-
-
-* **How to setup profiler notifier?**
-
- We decided to use olso.messaging Notifier API, because:
-
- * `oslo.messaging`_ is integrated in all projects
-
- * It's the simplest way to send notification to Ceilometer, take a
- look at: *osprofiler.notifiers.messaging.Messaging:notify* method
-
- * We don't need to add any new `CONF`_ options in projects
-
-
-* **How to initialize profiler, to get one trace across all services?**
-
- To enable cross service profiling we actually need to do send from caller
- to callee (base_id & trace_id). So callee will be able to init its profiler
- with these values.
-
- In case of OpenStack there are 2 kinds of interaction between 2 services:
-
- * REST API
-
- It's well known that there are python clients for every project,
- that generate proper HTTP requests, and parse responses to objects.
-
- These python clients are used in 2 cases:
-
- * User access -> OpenStack
-
- * Service from Project 1 would like to access Service from Project 2
-
-
- So what we need is to:
-
- * Put in python clients headers with trace info (if profiler is inited)
-
- * Add `OSprofiler WSGI middleware`_ to your service, this initializes
- the profiler, if and only if there are special trace headers, that
- are signed by one of the HMAC keys from api-paste.ini (if multiple
- keys exist the signing process will continue to use the key that was
- accepted during validation).
-
- * The common items that are used to configure the middleware are the
- following (these can be provided when initializing the middleware
- object or when setting up the api-paste.ini file)::
-
- hmac_keys = KEY1, KEY2 (can be a single key as well)
-
- Actually the algorithm is a bit more complex. The Python client will
- also sign the trace info with a `HMAC`_ key (lets call that key ``A``)
- passed to profiler.init, and on reception the WSGI middleware will
- check that it's signed with *one of* the HMAC keys (the wsgi
- server should have key ``A`` as well, but may also have keys ``B``
- and ``C``) that are specified in api-paste.ini. This ensures that only
- the user that knows the HMAC key ``A`` in api-paste.ini can init a
- profiler properly and send trace info that will be actually
- processed. This ensures that trace info that is sent in that
- does **not** pass the HMAC validation will be discarded. **NOTE:** The
- application of many possible *validation* keys makes it possible to
- roll out a key upgrade in a non-impactful manner (by adding a key into
- the list and rolling out that change and then removing the older key at
- some time in the future).
-
- * RPC API
-
- RPC calls are used for interaction between services of one project.
- It's well known that projects are using `oslo.messaging`_ to deal with
- RPC. It's very good, because projects deal with RPC in similar way.
-
- So there are 2 required changes:
-
- * On callee side put in request context trace info (if profiler was
- initialized)
-
- * On caller side initialize profiler, if there is trace info in request
- context.
-
- * Trace all methods of callee API (can be done via profiler.trace_cls).
-
-
-* **What points should be tracked by default?**
-
- I think that for all projects we should include by default 5 kinds of points:
-
- * All HTTP calls - helps to get information about: what HTTP requests were
- done, duration of calls (latency of service), information about projects
- involved in request.
-
- * All RPC calls - helps to understand duration of parts of request related
- to different services in one project. This information is essential to
- understand which service produce the bottleneck.
-
- * All DB API calls - in some cases slow DB query can produce bottleneck. So
- it's quite useful to track how much time request spend in DB layer.
-
- * All driver calls - in case of nova, cinder and others we have vendor
- drivers. Duration
-
- * ALL SQL requests (turned off by default, because it produce a lot of
- traffic)
-
-.. _CONF: http://docs.openstack.org/developer/oslo.config/
-.. _HMAC: http://en.wikipedia.org/wiki/Hash-based_message_authentication_code
-.. _OpenStack: http://openstack.org/
-.. _Ceilometer: https://wiki.openstack.org/wiki/Ceilometer
-.. _oslo.messaging: https://pypi.python.org/pypi/oslo.messaging
-.. _OSprofiler WSGI middleware: https://github.com/stackforge/osprofiler/blob/master/osprofiler/web.py
+* Free software: Apache license
+* Documentation: https://docs.openstack.org/osprofiler/latest/
+* Source: https://git.openstack.org/cgit/openstack/osprofiler
+* Bugs: https://bugs.launchpad.net/osprofiler
diff --git a/devstack/README.rst b/devstack/README.rst
index 1a35e06..9e5b752 100644
--- a/devstack/README.rst
+++ b/devstack/README.rst
@@ -1,18 +1,54 @@
==================================
-Enabling OSprofiler using DevStack
+Enabling OSProfiler using DevStack
==================================
This directory contains the files necessary to run OpenStack with enabled
-OSprofiler in DevStack.
+OSProfiler in DevStack.
-To configure DevStack to enable OSprofiler edit
-``${DEVSTACK_DIR}/local.conf`` file and add::
+OSProfiler has different drivers for trace processing. The default driver uses
+Ceilometer to process and store trace events. Other drivers may connect
+to databases directly and do not require Ceilometer.
- enable_plugin ceilometer https://github.com/openstack/ceilometer master
- enable_plugin osprofiler https://github.com/openstack/osprofiler master
+To configure DevStack and enable OSProfiler edit ``${DEVSTACK_DIR}/local.conf``
+file and add the following to ``[[local|localrc]]`` section:
-to the ``[[local|localrc]]`` section.
+ * to use specified driver::
+
+ enable_plugin osprofiler https://git.openstack.org/openstack/osprofiler master
+ OSPROFILER_CONNECTION_STRING=
+
+ the driver is chosen depending on the value of
+ ``OSPROFILER_CONNECTION_STRING`` variable (refer to the next section for
+ details)
+
+ * to use default Ceilometer driver::
+
+ enable_plugin panko https://git.openstack.org/openstack/panko master
+ enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer master
+ enable_plugin osprofiler https://git.openstack.org/openstack/osprofiler master
+
+ .. note:: The order of enabling plugins matters.
Run DevStack as normal::
$ ./stack.sh
+
+
+Config variables
+----------------
+
+**OSPROFILER_HMAC_KEYS** - a set of HMAC secrets, that are used for triggering
+of profiling in OpenStack services: only the requests that specify one of these
+keys in HTTP headers will be profiled. E.g. multiple secrets are specified as
+a comma-separated list of string values::
+
+ OSPROFILER_HMAC_KEYS=swordfish,foxtrot,charlie
+
+**OSPROFILER_CONNECTION_STRING** - connection string to identify the driver.
+Default value is ``messaging://`` refers to Ceilometer driver. For a full
+list of drivers please refer to
+``http://git.openstack.org/cgit/openstack/osprofiler/tree/osprofiler/drivers``.
+Example: enable ElasticSearch driver with the server running on localhost::
+
+ OSPROFILER_CONNECTION_STRING=elasticsearch://127.0.0.1:9200
+
diff --git a/devstack/lib/osprofiler b/devstack/lib/osprofiler
index b758601..dbbebd4 100644
--- a/devstack/lib/osprofiler
+++ b/devstack/lib/osprofiler
@@ -1,5 +1,5 @@
-# lib/rally
-# Functions to control the configuration and operation of the **Rally**
+# lib/osprofiler
+# Functions to control the configuration and operation of the **osprofiler**
# Dependencies:
#
@@ -20,21 +20,21 @@
# Defaults
# --------
-OLD_STYLE_CONF_FILES=(
- /etc/cinder/cinder.conf
- /etc/heat/heat.conf
-)
-
-NEW_STYLE_CONF_FILES=(
- /etc/keystone/keystone.conf
- /etc/nova/nova.conf
- /etc/neutron/neutron.conf
- /etc/glance/glance-api.conf
- /etc/glance/glance-registry.conf
- /etc/trove/trove.conf
- /etc/trove/trove-conductor.conf
- /etc/trove/trove-guestagent.conf
- /etc/trove/trove-taskmanager.conf
+CONF_FILES=(
+ $CINDER_CONF
+ $HEAT_CONF
+ $KEYSTONE_CONF
+ $NOVA_CONF
+ $NEUTRON_CONF
+ $GLANCE_API_CONF
+ $GLANCE_REGISTRY_CONF
+ $TROVE_CONF
+ $TROVE_CONDUCTOR_CONF
+ $TROVE_GUESTAGENT_CONF
+ $TROVE_TASKMANAGER_CONF
+ $SENLIN_CONF
+ $MAGNUM_CONF
+ $ZUN_CONF
)
# This will update CEILOMETER_NOTIFICATION_TOPICS in ceilometer.conf file
@@ -47,33 +47,21 @@
# configure_osprofiler() - Nothing for now
function configure_osprofiler() {
- for conf in ${OLD_STYLE_CONF_FILES[@]}; do
- if [ -f $conf ]
- then
- iniset $conf profiler profiler_enabled True
- iniset $conf profiler trace_sqlalchemy True
- iniset $conf profiler hmac_keys SECRET_KEY
- fi
- done
-
- for conf in ${NEW_STYLE_CONF_FILES[@]}; do
+ for conf in ${CONF_FILES[@]}; do
if [ -f $conf ]
then
iniset $conf profiler enabled True
iniset $conf profiler trace_sqlalchemy True
- iniset $conf profiler hmac_keys SECRET_KEY
+ iniset $conf profiler hmac_keys $OSPROFILER_HMAC_KEYS
+ iniset $conf profiler connection_string $OSPROFILER_CONNECTION_STRING
fi
done
- CEILOMETER_CONF=/etc/ceilometer/ceilometer.conf
- iniset $CEILOMETER_CONF event store_raw info
+ if [ -f $CEILOMETER_CONF ]
+ then
+ iniset $CEILOMETER_CONF event store_raw info
+ fi
}
-
-# init_rally() - Initialize databases, etc.
-function init_osprofiler() {
-
- echo "Do nothing here for now"
-}
# Restore xtrace
$XTRACE
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index 303b605..348da47 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -1,4 +1,4 @@
-# DevStack extras script to install Rally
+# DevStack extras script to install osprofiler
# Save trace setting
XTRACE=$(set +o | grep xtrace)
@@ -6,18 +6,9 @@
source $DEST/osprofiler/devstack/lib/osprofiler
-if [[ "$1" == "source" ]]; then
- # Initial source
- source $TOP_DIR/lib/rally
-# elif [[ "$1" == "stack" && "$2" == "install" ]]; then
-# echo_summary "Installing OSprofiler"
-# install_rally
-elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
+if [[ "$1" == "stack" && "$2" == "post-config" ]]; then
echo_summary "Configuring OSprofiler"
configure_osprofiler
-elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
- echo_summary "Initializing OSprofiler"
- init_osprofiler
fi
# Restore xtrace
diff --git a/devstack/settings b/devstack/settings
index dc01705..fea08c2 100644
--- a/devstack/settings
+++ b/devstack/settings
@@ -1,3 +1,9 @@
# Devstack settings
+# A comma-separated list of secrets, that will be used for triggering
+# of profiling in OpenStack services: profiling is only performed for
+# requests that specify one of these keys in HTTP headers.
+OSPROFILER_HMAC_KEYS=${OSPROFILER_HMAC_KEYS:-"SECRET_KEY"}
+OSPROFILER_CONNECTION_STRING=${OSPROFILER_CONNECTION_STRING:-"messaging://"}
+
enable_service osprofiler
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 8854a5b..8412b05 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -11,9 +11,7 @@
# All configuration values have a default; values that are commented out
# serve to show the default.
-import datetime
import os
-import subprocess
import sys
# If extensions (or modules to document with autodoc) are in another directory,
@@ -38,7 +36,14 @@
'sphinx.ext.coverage',
'sphinx.ext.ifconfig',
'sphinx.ext.viewcode',
+ 'openstackdocstheme',
]
+
+# openstackdocstheme options
+repository_name = 'openstack/osprofiler'
+bug_project = 'osprofiler'
+bug_tag = ''
+
todo_include_todos = True
# Add any paths that contain templates here, relative to this directory.
@@ -55,16 +60,7 @@
# General information about the project.
project = u'OSprofiler'
-copyright = u'%d, Mirantis Inc.' % datetime.datetime.now().year
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = '0.2.5'
-# The full version, including alpha/beta/rc tags.
-release = '0.2.5'
+copyright = u'2016, OpenStack Foundation'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -105,7 +101,7 @@
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
-html_theme = 'default'
+html_theme = 'openstackdocs'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
@@ -138,10 +134,7 @@
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
-git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local",
- "-n1"]
-html_last_updated_fmt = subprocess.Popen(
- git_cmd, stdout=subprocess.PIPE).communicate()[0]
+html_last_updated_fmt = '%Y-%m-%d %H:%M'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
diff --git a/doc/source/index.rst b/doc/source/index.rst
deleted file mode 120000
index c768ff7..0000000
--- a/doc/source/index.rst
+++ /dev/null
@@ -1 +0,0 @@
-../../README.rst
\ No newline at end of file
diff --git a/doc/source/index.rst b/doc/source/index.rst
new file mode 100644
index 0000000..01c8b3e
--- /dev/null
+++ b/doc/source/index.rst
@@ -0,0 +1,22 @@
+=============================================
+OSProfiler -- Cross-project profiling library
+=============================================
+
+OSProfiler provides a tiny but powerful library that is used by
+most (soon to be all) OpenStack projects and their python clients. It
+provides functionality to generate 1 trace per request, that goes
+through all involved services. This trace can then be extracted and used
+to build a tree of calls which can be quite handy for a variety of
+reasons (for example in isolating cross-project performance issues).
+
+.. toctree::
+ :maxdepth: 2
+
+ user/index
+
+.. rubric:: Indices and tables
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
diff --git a/doc/source/user/api.rst b/doc/source/user/api.rst
new file mode 100644
index 0000000..78da4d7
--- /dev/null
+++ b/doc/source/user/api.rst
@@ -0,0 +1,240 @@
+===
+API
+===
+
+There are few things that you should know about API before using it.
+
+Five ways to add a new trace point.
+-----------------------------------
+
+.. code-block:: python
+
+ from osprofiler import profiler
+
+ def some_func():
+ profiler.start("point_name", {"any_key": "with_any_value"})
+ # your code
+ profiler.stop({"any_info_about_point": "in_this_dict"})
+
+
+ @profiler.trace("point_name",
+ info={"any_info_about_point": "in_this_dict"},
+ hide_args=False)
+ def some_func2(*args, **kwargs):
+ # If you need to hide args in profile info, put hide_args=True
+ pass
+
+ def some_func3():
+ with profiler.Trace("point_name",
+ info={"any_key": "with_any_value"}):
+ # some code here
+
+ @profiler.trace_cls("point_name", info={}, hide_args=False,
+ trace_private=False)
+ class TracedClass(object):
+
+ def traced_method(self):
+ pass
+
+ def _traced_only_if_trace_private_true(self):
+ pass
+
+ @six.add_metaclass(profiler.TracedMeta)
+ class RpcManagerClass(object):
+ __trace_args__ = {'name': 'rpc',
+ 'info': None,
+ 'hide_args': False,
+ 'trace_private': False}
+
+ def my_method(self, some_args):
+ pass
+
+ def my_method2(self, some_arg1, some_arg2, kw=None, kw2=None)
+ pass
+
+How profiler works?
+-------------------
+
+* **profiler.Trace()** and **@profiler.trace()** are just syntax sugar,
+ that just calls **profiler.start()** & **profiler.stop()** methods.
+
+* Every call of **profiler.start()** & **profiler.stop()** sends to
+ **collector** 1 message. It means that every trace point creates 2 records
+ in the collector. *(more about collector & records later)*
+
+* Nested trace points are supported. The sample below produces 2 trace points:
+
+ .. code-block:: python
+
+ profiler.start("parent_point")
+ profiler.start("child_point")
+ profiler.stop()
+ profiler.stop()
+
+ The implementation is quite simple. Profiler has one stack that contains
+ ids of all trace points. E.g.:
+
+ .. code-block:: python
+
+ profiler.start("parent_point") # trace_stack.push()
+ # send to collector -> trace_stack[-2:]
+
+ profiler.start("parent_point") # trace_stack.push()
+ # send to collector -> trace_stack[-2:]
+ profiler.stop() # send to collector -> trace_stack[-2:]
+ # trace_stack.pop()
+
+ profiler.stop() # send to collector -> trace_stack[-2:]
+ # trace_stack.pop()
+
+ It's simple to build a tree of nested trace points, having
+ **(parent_id, point_id)** of all trace points.
+
+Process of sending to collector.
+--------------------------------
+
+Trace points contain 2 messages (start and stop). Messages like below are
+sent to a collector:
+
+.. parsed-literal::
+
+ {
+ "name": -(start|stop)
+ "base_id": ,
+ "parent_id": ,
+ "trace_id": ,
+ "info":
+ }
+
+The fields are defined as the following:
+
+* base_id - ```` that is equal for all trace points that belong
+ to one trace, this is done to simplify the process of retrieving
+ all trace points related to one trace from collector
+* parent_id - ```` of parent trace point
+* trace_id - ```` of current trace point
+* info - the dictionary that contains user information passed when calling
+ profiler **start()** & **stop()** methods.
+
+Setting up the collector.
+-------------------------
+
+Using OSProfiler notifier.
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. note:: The following way of configuring OSProfiler is deprecated. The new
+ version description is located below - `Using OSProfiler initializer.`_.
+ Don't use OSproliler notifier directly! Its support will be removed soon
+ from OSProfiler.
+
+The profiler doesn't include a trace point collector. The user/developer
+should instead provide a method that sends messages to a collector. Let's
+take a look at a trivial sample, where the collector is just a file:
+
+.. code-block:: python
+
+ import json
+
+ from osprofiler import notifier
+
+ def send_info_to_file_collector(info, context=None):
+ with open("traces", "a") as f:
+ f.write(json.dumps(info))
+
+ notifier.set(send_info_to_file_collector)
+
+So now on every **profiler.start()** and **profiler.stop()** call we will
+write info about the trace point to the end of the **traces** file.
+
+Using OSProfiler initializer.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+OSProfiler now contains various storage drivers to collect tracing data.
+Information about what driver to use and what options to pass to OSProfiler
+are now stored in OpenStack services configuration files. Example of such
+configuration can be found below:
+
+.. code-block:: bash
+
+ [profiler]
+ enabled = True
+ trace_sqlalchemy = True
+ hmac_keys = SECRET_KEY
+ connection_string = messaging://
+
+If such configuration is provided, OSProfiler setting up can be processed in
+following way:
+
+.. code-block:: python
+
+ if CONF.profiler.enabled:
+ osprofiler_initializer.init_from_conf(
+ conf=CONF,
+ context=context.get_admin_context().to_dict(),
+ project="cinder",
+ service=binary,
+ host=host
+ )
+
+Initialization of profiler.
+---------------------------
+
+If profiler is not initialized, all calls to **profiler.start()** and
+**profiler.stop()** will be ignored.
+
+Initialization is a quite simple procedure.
+
+.. code-block:: python
+
+ from osprofiler import profiler
+
+ profiler.init("SECRET_HMAC_KEY", base_id=, parent_id=)
+
+``SECRET_HMAC_KEY`` - will be discussed later, because it's related to the
+integration of OSprofiler & OpenStack.
+
+**base_id** and **trace_id** will be used to initialize stack_trace in
+profiler, e.g. ``stack_trace = [base_id, trace_id]``.
+
+OSProfiler CLI.
+---------------
+
+To make it easier for end users to work with profiler from CLI, OSProfiler
+has entry point that allows them to retrieve information about traces and
+present it in human readable from.
+
+Available commands:
+
+* Help message with all available commands and their arguments:
+
+ .. parsed-literal::
+
+ $ osprofiler -h/--help
+
+* OSProfiler version:
+
+ .. parsed-literal::
+
+ $ osprofiler -v/--version
+
+* Results of profiling can be obtained in JSON (option: ``--json``) and HTML
+ (option: ``--html``) formats:
+
+ .. parsed-literal::
+
+ $ osprofiler trace show --json/--html
+
+ hint: option ``--out`` will redirect result of ``osprofiler trace show``
+ in specified file:
+
+ .. parsed-literal::
+
+ $ osprofiler trace show --json/--html --out /path/to/file
+
+* In latest versions of OSProfiler with storage drivers (e.g. MongoDB (URI:
+ ``mongodb://``), Messaging (URI: ``messaging://``), and Ceilometer
+ (URI: ``ceilometer://``)) ``--connection-string`` parameter should be set up:
+
+ .. parsed-literal::
+
+ $ osprofiler trace show --connection-string= --json/--html
diff --git a/doc/source/user/background.rst b/doc/source/user/background.rst
new file mode 100644
index 0000000..5845d9c
--- /dev/null
+++ b/doc/source/user/background.rst
@@ -0,0 +1,32 @@
+==========
+Background
+==========
+
+OpenStack consists of multiple projects. Each project, in turn, is composed of
+multiple services. To process some request, e.g. to boot a virtual machine,
+OpenStack uses multiple services from different projects. In the case something
+works too slow, it's extremely complicated to understand what exactly goes
+wrong and to locate the bottleneck.
+
+To resolve this issue, we introduce a tiny but powerful library,
+**osprofiler**, that is going to be used by all OpenStack projects and their
+python clients. It generates 1 trace per request, that goes through
+all involved services, and builds a tree of calls.
+
+Why not cProfile and etc?
+-------------------------
+
+**The scope of this library is quite different:**
+
+* We are interested in getting one trace of points from different services,
+ not tracing all Python calls inside one process.
+
+* This library should be easy integrable into OpenStack. This means that:
+
+ * It shouldn't require too many changes in code bases of projects it's
+ integrated with.
+
+ * We should be able to fully turn it off.
+
+ * We should be able to keep it turned on in lazy mode in production
+ (e.g. admin should be able to "trace" on request).
diff --git a/doc/source/user/collectors.rst b/doc/source/user/collectors.rst
new file mode 100644
index 0000000..359d547
--- /dev/null
+++ b/doc/source/user/collectors.rst
@@ -0,0 +1,40 @@
+==========
+Collectors
+==========
+
+There are a number of drivers to support different collector backends:
+
+Redis
+-----
+
+* Overview
+
+ The Redis driver allows profiling data to be collected into a redis
+ database instance. The traces are stored as key-value pairs where the
+ key is a string built using trace ids and timestamps and the values
+ are JSON strings containing the trace information. A second driver is
+ included to use Redis Sentinel in addition to single node Redis.
+
+* Capabilities
+
+ * Write trace data to the database.
+ * Query Traces in database: This allows for pulling trace data
+ querying on the keys used to save the data in the database.
+ * Generate a report based on the traces stored in the database.
+ * Supports use of Redis Sentinel for robustness.
+
+* Usage
+
+ The driver is used by OSProfiler when using a connection-string URL
+ of the form redis://:. To use the Sentinel version
+ use a connection-string of the form redissentinel://:
+
+* Configuration
+
+ * No config changes are required by for the base Redis driver.
+ * There are two configuration options for the Redis Sentinel driver:
+
+ * socket_timeout: specifies the sentinel connection socket timeout
+ value. Defaults to: 0.1 seconds
+ * sentinel_service_name: The name of the Sentinel service to use.
+ Defaults to: "mymaster"
diff --git a/doc/source/user/history.rst b/doc/source/user/history.rst
new file mode 100644
index 0000000..f69be70
--- /dev/null
+++ b/doc/source/user/history.rst
@@ -0,0 +1 @@
+.. include:: ../../../ChangeLog
diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst
new file mode 100644
index 0000000..fe317da
--- /dev/null
+++ b/doc/source/user/index.rst
@@ -0,0 +1,27 @@
+================
+Using OSProfiler
+================
+
+OSProfiler provides a tiny but powerful library that is used by
+most (soon to be all) OpenStack projects and their python clients. It
+provides functionality to generate 1 trace per request, that goes
+through all involved services. This trace can then be extracted and used
+to build a tree of calls which can be quite handy for a variety of
+reasons (for example in isolating cross-project performance issues).
+
+.. toctree::
+ :maxdepth: 2
+
+ background
+ api
+ integration
+ collectors
+ similar_projects
+
+Release Notes
+=============
+
+.. toctree::
+ :maxdepth: 1
+
+ history
diff --git a/doc/source/user/integration.rst b/doc/source/user/integration.rst
new file mode 100644
index 0000000..dfb0de2
--- /dev/null
+++ b/doc/source/user/integration.rst
@@ -0,0 +1,134 @@
+===========
+Integration
+===========
+
+There are 4 topics related to integration OSprofiler & `OpenStack`_:
+
+What we should use as a centralized collector?
+----------------------------------------------
+
+We primarily decided to use `Ceilometer`_, because:
+
+* It's already integrated in OpenStack, so it's quite simple to send
+ notifications to it from all projects.
+
+* There is an OpenStack API in Ceilometer that allows us to retrieve all
+ messages related to one trace. Take a look at
+ *osprofiler.drivers.ceilometer.Ceilometer:get_report*
+
+In OSProfiler starting with 1.4.0 version other options (MongoDB driver in
+1.4.0 release, Elasticsearch driver added later, etc.) are also available.
+
+
+How to setup profiler notifier?
+-------------------------------
+
+We primarily decided to use oslo.messaging Notifier API, because:
+
+* `oslo.messaging`_ is integrated in all projects
+
+* It's the simplest way to send notification to Ceilometer, take a
+ look at: *osprofiler.drivers.messaging.Messaging:notify* method
+
+* We don't need to add any new `CONF`_ options in projects
+
+In OSProfiler starting with 1.4.0 version other options (MongoDB driver in
+1.4.0 release, Elasticsearch driver added later, etc.) are also available.
+
+How to initialize profiler, to get one trace across all services?
+-----------------------------------------------------------------
+
+To enable cross service profiling we actually need to do send from caller
+to callee (base_id & trace_id). So callee will be able to init its profiler
+with these values.
+
+In case of OpenStack there are 2 kinds of interaction between 2 services:
+
+* REST API
+
+ It's well known that there are python clients for every project,
+ that generate proper HTTP requests, and parse responses to objects.
+
+ These python clients are used in 2 cases:
+
+ * User access -> OpenStack
+
+ * Service from Project 1 would like to access Service from Project 2
+
+
+ So what we need is to:
+
+ * Put in python clients headers with trace info (if profiler is inited)
+
+ * Add `OSprofiler WSGI middleware`_ to your service, this initializes
+ the profiler, if and only if there are special trace headers, that
+ are signed by one of the HMAC keys from api-paste.ini (if multiple
+ keys exist the signing process will continue to use the key that was
+ accepted during validation).
+
+ * The common items that are used to configure the middleware are the
+ following (these can be provided when initializing the middleware
+ object or when setting up the api-paste.ini file)::
+
+ hmac_keys = KEY1, KEY2 (can be a single key as well)
+
+ Actually the algorithm is a bit more complex. The Python client will
+ also sign the trace info with a `HMAC`_ key (lets call that key ``A``)
+ passed to profiler.init, and on reception the WSGI middleware will
+ check that it's signed with *one of* the HMAC keys (the wsgi
+ server should have key ``A`` as well, but may also have keys ``B``
+ and ``C``) that are specified in api-paste.ini. This ensures that only
+ the user that knows the HMAC key ``A`` in api-paste.ini can init a
+ profiler properly and send trace info that will be actually
+ processed. This ensures that trace info that is sent in that
+ does **not** pass the HMAC validation will be discarded. **NOTE:** The
+ application of many possible *validation* keys makes it possible to
+ roll out a key upgrade in a non-impactful manner (by adding a key into
+ the list and rolling out that change and then removing the older key at
+ some time in the future).
+
+* RPC API
+
+ RPC calls are used for interaction between services of one project.
+ It's well known that projects are using `oslo.messaging`_ to deal with
+ RPC. It's very good, because projects deal with RPC in similar way.
+
+ So there are 2 required changes:
+
+ * On callee side put in request context trace info (if profiler was
+ initialized)
+
+ * On caller side initialize profiler, if there is trace info in request
+ context.
+
+ * Trace all methods of callee API (can be done via profiler.trace_cls).
+
+
+What points should be tracked by default?
+-----------------------------------------
+
+I think that for all projects we should include by default 5 kinds of points:
+
+* All HTTP calls - helps to get information about: what HTTP requests were
+ done, duration of calls (latency of service), information about projects
+ involved in request.
+
+* All RPC calls - helps to understand duration of parts of request related
+ to different services in one project. This information is essential to
+ understand which service produce the bottleneck.
+
+* All DB API calls - in some cases slow DB query can produce bottleneck. So
+ it's quite useful to track how much time request spend in DB layer.
+
+* All driver calls - in case of nova, cinder and others we have vendor
+ drivers. Duration
+
+* ALL SQL requests (turned off by default, because it produce a lot of
+ traffic)
+
+.. _CONF: http://docs.openstack.org/developer/oslo.config/
+.. _HMAC: http://en.wikipedia.org/wiki/Hash-based_message_authentication_code
+.. _OpenStack: http://openstack.org/
+.. _Ceilometer: https://wiki.openstack.org/wiki/Ceilometer
+.. _oslo.messaging: https://pypi.python.org/pypi/oslo.messaging
+.. _OSprofiler WSGI middleware: https://github.com/openstack/osprofiler/blob/master/osprofiler/web.py
diff --git a/doc/source/user/similar_projects.rst b/doc/source/user/similar_projects.rst
new file mode 100644
index 0000000..8099e0c
--- /dev/null
+++ b/doc/source/user/similar_projects.rst
@@ -0,0 +1,20 @@
+================
+Similar projects
+================
+
+Other projects (some alive, some abandoned, some research prototypes)
+that are similar (in idea and ideal to OSprofiler).
+
+* `Zipkin`_
+* `Dapper`_
+* `Tomograph`_
+* `HTrace`_
+* `Jaeger`_
+* `OpenTracing`_
+
+.. _Zipkin: http://zipkin.io/
+.. _Dapper: http://research.google.com/pubs/pub36356.html
+.. _Tomograph: https://github.com/stackforge/tomograph
+.. _HTrace: https://htrace.incubator.apache.org/
+.. _Jaeger: https://uber.github.io/jaeger/
+.. _OpenTracing: http://opentracing.io/
diff --git a/doc/specs/implemented/make_paste_ini_config_optional.rst b/doc/specs/implemented/make_paste_ini_config_optional.rst
new file mode 100644
index 0000000..a797924
--- /dev/null
+++ b/doc/specs/implemented/make_paste_ini_config_optional.rst
@@ -0,0 +1,82 @@
+..
+ This work is licensed under a Creative Commons Attribution 3.0 Unported
+ License.
+
+ http://creativecommons.org/licenses/by/3.0/legalcode
+
+..
+ This template should be in ReSTructured text. The filename in the git
+ repository should match the launchpad URL, for example a URL of
+ https://blueprints.launchpad.net/heat/+spec/awesome-thing should be named
+ awesome-thing.rst . Please do not delete any of the sections in this
+ template. If you have nothing to say for a whole section, just write: None
+ For help with syntax, see http://sphinx-doc.org/rest.html
+ To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html
+
+======================================
+ Make api-paste.ini Arguments Optional
+======================================
+
+Problem description
+===================
+
+Integration of OSprofiler with OpenStack projects is harder than it should be,
+it requires keeping part of arguments inside api-paste.ini files and part in
+projects.conf file.
+
+We should make all configuration options from api-paste.ini file optional
+and add alternative way to configure osprofiler.web.WsgiMiddleware
+
+
+Proposed change
+===============
+
+Integration of OSprofiler requires 2 changes in api-paste.ini file:
+
+- One is adding osprofiler.web.WsgiMiddleware to pipelines:
+ https://github.com/openstack/cinder/blob/master/etc/cinder/api-paste.ini#L13
+
+- Another is to add it's arguments:
+ https://github.com/openstack/cinder/blob/master/etc/cinder/api-paste.ini#L31-L32
+
+ so WsgiMiddleware will be correctly initialized here:
+ https://github.com/openstack/osprofiler/blob/51761f375189bdc03b7e72a266ad0950777f32b1/osprofiler/web.py#L64
+
+We should make ``hmac_keys`` and ``enabled`` variable optional, create
+separated method from initialization of wsgi middleware and cut new release.
+After that remove
+
+
+Alternatives
+------------
+
+None.
+
+
+Implementation
+==============
+
+Assignee(s)
+-----------
+
+Primary assignee:
+ dbelova
+
+Work Items
+----------
+
+- Modify osprofiler.web.WsgiMiddleware to make ``hmac_keys`` optional (done)
+
+- Add alternative way to setup osprofiler.web.WsgiMiddleware, e.g. extra
+ argument hmac_keys to enable() method (done)
+
+- Cut new release 0.3.1 (tbd)
+
+- Fix the code in all projects: remove api-paste.ini arguments and use
+ osprofiler.web.enable with extra argument (tbd)
+
+
+Dependencies
+============
+
+- Cinder, Glance, Trove - projects should be fixed
diff --git a/doc/specs/implemented/multi_backend_support.rst b/doc/specs/implemented/multi_backend_support.rst
new file mode 100644
index 0000000..1d4a6d0
--- /dev/null
+++ b/doc/specs/implemented/multi_backend_support.rst
@@ -0,0 +1,91 @@
+..
+ This work is licensed under a Creative Commons Attribution 3.0 Unported
+ License.
+
+ http://creativecommons.org/licenses/by/3.0/legalcode
+
+..
+ This template should be in ReSTructured text. The filename in the git
+ repository should match the launchpad URL, for example a URL of
+ https://blueprints.launchpad.net/heat/+spec/awesome-thing should be named
+ awesome-thing.rst . Please do not delete any of the sections in this
+ template. If you have nothing to say for a whole section, just write: None
+ For help with syntax, see http://sphinx-doc.org/rest.html
+ To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html
+
+=====================
+Multi backend support
+=====================
+
+Make OSProfiler more flexible and production ready.
+
+Problem description
+===================
+
+Currently OSprofiler works only with one backend Ceilometer which actually
+doesn't work well and adds huge overhead. More over often Ceilometer is not
+installed/used at all. To resolve this we should add support for different
+backends like: MongoDB, InfluxDB, ElasticSearch, ...
+
+
+Proposed change
+===============
+
+And new osprofiler.drivers mechanism, each driver will do 2 things:
+send notifications and parse all notification in unified tree structure
+that can be processed by the REST lib.
+
+Deprecate osprofiler.notifiers and osprofiler.parsers
+
+Change all projects that are using OSprofiler to new model
+
+Alternatives
+------------
+
+I don't know any good alternative.
+
+Implementation
+==============
+
+Assignee(s)
+-----------
+
+Primary assignees:
+ dbelova
+ ayelistratov
+
+
+Work Items
+----------
+
+To add support of multi backends we should change few places in osprofiler
+that are hardcoded on Ceilometer:
+
+- CLI command ``show``:
+
+ I believe we should add extra argument "connection_string" which will allow
+ people to specify where is backend. So it will look like:
+ ://[[user[:password]]@[address][:port][/database]]
+
+- Merge osprofiler.notifiers and osprofiler.parsers to osprofiler.drivers
+
+ Notifiers and Parsers are tightly related. Like for MongoDB notifier you
+ should use MongoDB parsers, so there is better solution to keep both
+ in the same place.
+
+ This change should be done with keeping backward compatibility,
+ in other words
+ we should create separated directory osprofiler.drivers and put first
+ Ceilometer and then start working on other backends.
+
+ These drivers will be chosen based on connection string
+
+- Deprecate osprofiler.notifiers and osprofiler.parsers
+
+- Switch all projects to new model with connection string
+
+
+Dependencies
+============
+
+- Cinder, Glance, Trove, Heat should be changed
diff --git a/doc/specs/in-progress/better_devstack_integration.rst b/doc/specs/in-progress/better_devstack_integration.rst
index 199436d..b4624f2 100644
--- a/doc/specs/in-progress/better_devstack_integration.rst
+++ b/doc/specs/in-progress/better_devstack_integration.rst
@@ -52,7 +52,7 @@
- Make DevStack plugin for OSprofiler
-- Configure Celiometer
+- Configure Ceilometer
- Configure services that support OSprofiler
diff --git a/doc/specs/in-progress/make_paste_ini_config_optional.rst b/doc/specs/in-progress/make_paste_ini_config_optional.rst
deleted file mode 100644
index a797924..0000000
--- a/doc/specs/in-progress/make_paste_ini_config_optional.rst
+++ /dev/null
@@ -1,82 +0,0 @@
-..
- This work is licensed under a Creative Commons Attribution 3.0 Unported
- License.
-
- http://creativecommons.org/licenses/by/3.0/legalcode
-
-..
- This template should be in ReSTructured text. The filename in the git
- repository should match the launchpad URL, for example a URL of
- https://blueprints.launchpad.net/heat/+spec/awesome-thing should be named
- awesome-thing.rst . Please do not delete any of the sections in this
- template. If you have nothing to say for a whole section, just write: None
- For help with syntax, see http://sphinx-doc.org/rest.html
- To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html
-
-======================================
- Make api-paste.ini Arguments Optional
-======================================
-
-Problem description
-===================
-
-Integration of OSprofiler with OpenStack projects is harder than it should be,
-it requires keeping part of arguments inside api-paste.ini files and part in
-projects.conf file.
-
-We should make all configuration options from api-paste.ini file optional
-and add alternative way to configure osprofiler.web.WsgiMiddleware
-
-
-Proposed change
-===============
-
-Integration of OSprofiler requires 2 changes in api-paste.ini file:
-
-- One is adding osprofiler.web.WsgiMiddleware to pipelines:
- https://github.com/openstack/cinder/blob/master/etc/cinder/api-paste.ini#L13
-
-- Another is to add it's arguments:
- https://github.com/openstack/cinder/blob/master/etc/cinder/api-paste.ini#L31-L32
-
- so WsgiMiddleware will be correctly initialized here:
- https://github.com/openstack/osprofiler/blob/51761f375189bdc03b7e72a266ad0950777f32b1/osprofiler/web.py#L64
-
-We should make ``hmac_keys`` and ``enabled`` variable optional, create
-separated method from initialization of wsgi middleware and cut new release.
-After that remove
-
-
-Alternatives
-------------
-
-None.
-
-
-Implementation
-==============
-
-Assignee(s)
------------
-
-Primary assignee:
- dbelova
-
-Work Items
-----------
-
-- Modify osprofiler.web.WsgiMiddleware to make ``hmac_keys`` optional (done)
-
-- Add alternative way to setup osprofiler.web.WsgiMiddleware, e.g. extra
- argument hmac_keys to enable() method (done)
-
-- Cut new release 0.3.1 (tbd)
-
-- Fix the code in all projects: remove api-paste.ini arguments and use
- osprofiler.web.enable with extra argument (tbd)
-
-
-Dependencies
-============
-
-- Cinder, Glance, Trove - projects should be fixed
diff --git a/doc/specs/in-progress/multi_backend_support.rst b/doc/specs/in-progress/multi_backend_support.rst
deleted file mode 100644
index 965e589..0000000
--- a/doc/specs/in-progress/multi_backend_support.rst
+++ /dev/null
@@ -1,91 +0,0 @@
-..
- This work is licensed under a Creative Commons Attribution 3.0 Unported
- License.
-
- http://creativecommons.org/licenses/by/3.0/legalcode
-
-..
- This template should be in ReSTructured text. The filename in the git
- repository should match the launchpad URL, for example a URL of
- https://blueprints.launchpad.net/heat/+spec/awesome-thing should be named
- awesome-thing.rst . Please do not delete any of the sections in this
- template. If you have nothing to say for a whole section, just write: None
- For help with syntax, see http://sphinx-doc.org/rest.html
- To test out your formatting, see http://www.tele3.cz/jbar/rest/rest.html
-
-======================
- Multi backend support
-======================
-
-Make OSProfiler more flexible and production ready.
-
-Problem description
-===================
-
-Currently OSprofiler works only with one backend Celiometer which actually
-doesn't work well and adds huge overhead. More over often Ceilometer is not
-installed/used at all. To resolve this we should add support for different
-backends like: MongoDB, InfluxDB, ElasticSearch, ...
-
-
-Proposed change
-===============
-
-And new osprofiler.drivers mechanism, each driver will do 2 things:
-send notifications and parse all notification in unififed tree strcture
-that can be processed by the REST lib.
-
-Deprecate osprofiler.notifiers and osprofiler.parsers
-
-Change all projects that are using OSprofiler to new model
-
-Alternatives
-------------
-
-I don't know any good alternative.
-
-Implementation
-==============
-
-Assignee(s)
------------
-
-Primary assignee:
-
-
-
-Work Items
-----------
-
-To add support of multi backends we should change few places in osprofiler
-that are hardcoded on Ceilometer:
-
-- CLI command ``show``:
-
- I believe we should add extra argument "connection_string" which will allow
- people to specify where is backend. So it will look like:
- ://[[user[:password]]@[address][:port][/database]]
-
-- Merge osprofiler.notifiers and osprofiler.parsers to osprofiler.drivers
-
- Notifiers and Parsers are tightly related. Like for MongoDB notifier you
- should use MongoDB parsers, so there is better solution to keep both
- in the same place.
-
- This change should be done with keeping backward compatiblity, in other words
- we should create separated direcotory osprofier.drivers and put first
- Ceilometer and then start working on other backends.
-
- These drivers will be chosen based on connection string
-
-- Deprecate osprofiler.notifiers and osprofier.parsers
-
-- Cut new release 0.4.2
-
-- Switch all projects to new model with connection string
-
-
-Dependencies
-============
-
-- Cinder, Glance, Trove should be changed
diff --git a/osprofiler/__init__.py b/osprofiler/__init__.py
index f64a0e6..22bedd3 100644
--- a/osprofiler/__init__.py
+++ b/osprofiler/__init__.py
@@ -13,19 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
-import os
+import pkg_resources
-from six.moves import configparser
-
-from osprofiler import _utils as utils
-
-
-utils.import_modules_from_package("osprofiler._notifiers")
-
-_conf = configparser.ConfigParser()
-_conf.read(os.path.join(
- os.path.dirname(os.path.dirname(__file__)), "setup.cfg"))
-try:
- __version__ = _conf.get("metadata", "version")
-except (configparser.NoOptionError, configparser.NoSectionError):
- __version__ = None
+__version__ = pkg_resources.get_distribution("osprofiler").version
diff --git a/osprofiler/_notifiers/__init__.py b/osprofiler/_notifiers/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/osprofiler/_notifiers/base.py b/osprofiler/_notifiers/base.py
deleted file mode 100644
index cf0fa64..0000000
--- a/osprofiler/_notifiers/base.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from osprofiler import _utils as utils
-
-
-class Notifier(object):
-
- def notify(self, info, context=None):
- """This method will be called on each notifier.notify() call.
-
- To add new drivers you should, create new subclass of this class and
- implement notify method.
-
- :param info: Contains information about trace element.
- In payload dict there are always 3 ids:
- "base_id" - uuid that is common for all notifications
- related to one trace. Used to simplify
- retrieving of all trace elements from
- Ceilometer.
- "parent_id" - uuid of parent element in trace
- "trace_id" - uuid of current element in trace
-
- With parent_id and trace_id it's quite simple to build
- tree of trace elements, which simplify analyze of trace.
-
- :param context: request context that is mostly used to specify
- current active user and tenant.
- """
-
- @staticmethod
- def factory(name, *args, **kwargs):
- for driver in utils.itersubclasses(Notifier):
- if name == driver.__name__:
- return driver(*args, **kwargs).notify
-
- raise TypeError("There is no driver, with name: %s" % name)
diff --git a/osprofiler/_notifiers/messaging.py b/osprofiler/_notifiers/messaging.py
deleted file mode 100644
index ee19c83..0000000
--- a/osprofiler/_notifiers/messaging.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from osprofiler._notifiers import base
-
-
-class Messaging(base.Notifier):
-
- def __init__(self, messaging, context, transport, project, service, host):
- """Init Messaging notify driver.
-
- """
- super(Messaging, self).__init__()
- self.messaging = messaging
- self.context = context
- self.project = project
- self.service = service
-
- self.notifier = messaging.Notifier(
- transport, publisher_id=host, driver="messaging",
- topic="profiler", retry=0)
-
- def notify(self, info, context=None):
- """Send notifications to Ceilometer via oslo.messaging notifier API.
-
- :param info: Contains information about trace element.
- In payload dict there are always 3 ids:
- "base_id" - uuid that is common for all notifications
- related to one trace. Used to simplify
- retrieving of all trace elements from
- Ceilometer.
- "parent_id" - uuid of parent element in trace
- "trace_id" - uuid of current element in trace
-
- With parent_id and trace_id it's quite simple to build
- tree of trace elements, which simplify analyze of trace.
-
- :param context: request context that is mostly used to specify
- current active user and tenant.
- """
-
- info["project"] = self.project
- info["service"] = self.service
- self.notifier.info(context or self.context,
- "profiler.%s" % self.service, info)
diff --git a/osprofiler/_utils.py b/osprofiler/_utils.py
index 4f839b6..3d5c1cc 100644
--- a/osprofiler/_utils.py
+++ b/osprofiler/_utils.py
@@ -19,33 +19,8 @@
import json
import os
+from oslo_utils import secretutils
import six
-
-try:
- # Only in python 2.7.7+ (and python 3.3+)
- # https://docs.python.org/2/library/hmac.html#hmac.compare_digest
- from hmac import compare_digest # noqa
-except (AttributeError, ImportError):
- # Taken/slightly modified from:
- # https://mail.python.org/pipermail/python-checkins/2012-June/114532.html
- def compare_digest(a, b):
- """Returns the equivalent of 'a == b'.
-
- This method avoids content based short circuiting to reduce the
- vulnerability to timing attacks.
- """
- # We assume the length of the expected digest is public knowledge,
- # thus this early return isn't leaking anything an attacker wouldn't
- # already know
- if len(a) != len(b):
- return False
-
- # We assume that integers in the bytes range are all cached,
- # thus timing shouldn't vary much due to integer object creation
- result = 0
- for x, y in zip(a, b):
- result |= ord(x) ^ ord(y)
- return result == 0
def split(text, strip=True):
@@ -128,10 +103,10 @@
for hmac_key in hmac_keys:
try:
user_hmac_data = generate_hmac(data, hmac_key)
- except Exception:
+ except Exception: # nosec
pass
else:
- if compare_digest(hmac_data, user_hmac_data):
+ if secretutils.constant_time_compare(hmac_data, user_hmac_data):
try:
contents = json.loads(
binary_decode(base64.urlsafe_b64decode(data)))
diff --git a/osprofiler/cmd/commands.py b/osprofiler/cmd/commands.py
index b2700b6..6181a64 100644
--- a/osprofiler/cmd/commands.py
+++ b/osprofiler/cmd/commands.py
@@ -16,9 +16,11 @@
import json
import os
+from oslo_utils import uuidutils
+
from osprofiler.cmd import cliutils
-from osprofiler.cmd import exc
-from osprofiler.parsers import ceilometer as ceiloparser
+from osprofiler.drivers import base
+from osprofiler import exc
class BaseCommand(object):
@@ -29,65 +31,113 @@
group_name = "trace"
@cliutils.arg("trace", help="File with trace or trace id")
+ @cliutils.arg("--connection-string", dest="conn_str",
+ default=(cliutils.env("OSPROFILER_CONNECTION_STRING") or
+ "ceilometer://"),
+ help="Storage driver's connection string. Defaults to "
+ "env[OSPROFILER_CONNECTION_STRING] if set, else "
+ "ceilometer://")
@cliutils.arg("--json", dest="use_json", action="store_true",
help="show trace in JSON")
@cliutils.arg("--html", dest="use_html", action="store_true",
help="show trace in HTML")
+ @cliutils.arg("--dot", dest="use_dot", action="store_true",
+ help="show trace in DOT language")
+ @cliutils.arg("--render-dot", dest="render_dot_filename",
+ help="filename for rendering the dot graph in pdf format")
@cliutils.arg("--out", dest="file_name", help="save output in file")
def show(self, args):
- """Displays trace-results by given trace id in HTML or JSON format."""
+ """Display trace results in HTML, JSON or DOT format."""
trace = None
- if os.path.exists(args.trace):
+ if not uuidutils.is_uuid_like(args.trace):
trace = json.load(open(args.trace))
else:
try:
- import ceilometerclient.client
- import ceilometerclient.exc
- import ceilometerclient.shell
- except ImportError:
- raise ImportError(
- "To use this command, you should install "
- "'ceilometerclient' manually. Use command:\n "
- "'pip install ceilometerclient'.")
- try:
- client = ceilometerclient.client.get_client(
- args.ceilometer_api_version, **args.__dict__)
- notifications = ceiloparser.get_notifications(
- client, args.trace)
+ engine = base.get_driver(args.conn_str, **args.__dict__)
except Exception as e:
- if hasattr(e, "http_status") and e.http_status == 401:
- msg = "Invalid OpenStack Identity credentials."
- else:
- msg = "Something has gone wrong. See logs for more details"
- raise exc.CommandError(msg)
+ raise exc.CommandError(e.message)
- if notifications:
- trace = ceiloparser.parse_notifications(notifications)
+ trace = engine.get_report(args.trace)
- if not trace:
- msg = ("Trace with UUID %s not found. "
- "There are 3 possible reasons: \n"
- " 1) You are using not admin credentials\n"
- " 2) You specified wrong trace id\n"
- " 3) You specified wrong HMAC Key in original calling"
- % args.trace)
+ if not trace or not trace.get("children"):
+ msg = ("Trace with UUID %s not found. Please check the HMAC key "
+ "used in the command." % args.trace)
raise exc.CommandError(msg)
+ # NOTE(ayelistratov): Ceilometer translates datetime objects to
+ # strings, other drivers store this data in ISO Date format.
+ # Since datetime.datetime is not JSON serializable by default,
+ # this method will handle that.
+ def datetime_json_serialize(obj):
+ if hasattr(obj, "isoformat"):
+ return obj.isoformat()
+ else:
+ return obj
+
if args.use_json:
- output = json.dumps(trace)
+ output = json.dumps(trace, default=datetime_json_serialize,
+ separators=(",", ": "),
+ indent=2)
elif args.use_html:
with open(os.path.join(os.path.dirname(__file__),
"template.html")) as html_template:
output = html_template.read().replace(
- "$DATA", json.dumps(trace, indent=2))
+ "$DATA", json.dumps(trace, indent=4,
+ separators=(",", ": "),
+ default=datetime_json_serialize))
+ elif args.use_dot:
+ dot_graph = self._create_dot_graph(trace)
+ output = dot_graph.source
+ if args.render_dot_filename:
+ dot_graph.render(args.render_dot_filename, cleanup=True)
else:
raise exc.CommandError("You should choose one of the following "
- "output-formats: --json or --html.")
+ "output formats: json, html or dot.")
if args.file_name:
with open(args.file_name, "w+") as output_file:
output_file.write(output)
else:
- print (output)
+ print(output)
+
+ def _create_dot_graph(self, trace):
+ try:
+ import graphviz
+ except ImportError:
+ raise exc.CommandError(
+ "graphviz library is required to use this option.")
+
+ dot = graphviz.Digraph(format="pdf")
+ next_id = [0]
+
+ def _create_node(info):
+ time_taken = info["finished"] - info["started"]
+ service = info["service"] + ":" if "service" in info else ""
+ name = info["name"]
+ label = "%s%s - %d ms" % (service, name, time_taken)
+
+ if name == "wsgi":
+ req = info["meta.raw_payload.wsgi-start"]["info"]["request"]
+ label = "%s\\n%s %s.." % (label, req["method"],
+ req["path"][:30])
+ elif name == "rpc" or name == "driver":
+ raw = info["meta.raw_payload.%s-start" % name]
+ fn_name = raw["info"]["function"]["name"]
+ label = "%s\\n%s" % (label, fn_name.split(".")[-1])
+
+ node_id = str(next_id[0])
+ next_id[0] += 1
+ dot.node(node_id, label)
+ return node_id
+
+ def _create_sub_graph(root):
+ rid = _create_node(root["info"])
+ for child in root["children"]:
+ cid = _create_sub_graph(child)
+ dot.edge(rid, cid)
+ return rid
+
+ _create_sub_graph(trace)
+ return dot
diff --git a/osprofiler/cmd/exc.py b/osprofiler/cmd/exc.py
deleted file mode 100644
index 0ffc9c9..0000000
--- a/osprofiler/cmd/exc.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-class CommandError(Exception):
- """Invalid usage of CLI."""
-
- def __init__(self, message=None):
- self.message = message
-
- def __str__(self):
- return self.message or self.__class__.__doc__
diff --git a/osprofiler/cmd/shell.py b/osprofiler/cmd/shell.py
index e613a90..f48f23b 100644
--- a/osprofiler/cmd/shell.py
+++ b/osprofiler/cmd/shell.py
@@ -18,21 +18,24 @@
Command-line interface to the OpenStack Profiler.
"""
+import argparse
import inspect
import sys
-import argparse
+from oslo_config import cfg
import osprofiler
from osprofiler.cmd import cliutils
from osprofiler.cmd import commands
-from osprofiler.cmd import exc
+from osprofiler import exc
+from osprofiler import opts
class OSProfilerShell(object):
def __init__(self, argv):
args = self._get_base_parser().parse_args(argv)
+ opts.set_defaults(cfg.CONF)
if not (args.os_auth_token and args.ceilometer_url):
if not args.os_username:
@@ -235,7 +238,7 @@
try:
OSProfilerShell(args)
except exc.CommandError as e:
- print (e.message)
+ print(e.message)
return 1
diff --git a/osprofiler/cmd/template.html b/osprofiler/cmd/template.html
index 2006e0c..ac31fc7 100644
--- a/osprofiler/cmd/template.html
+++ b/osprofiler/cmd/template.html
@@ -1,191 +1,290 @@
-
-
-
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
Levels
+
Duration
+
Type
+
Project
+
Service
+
Host
+
Details
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Levels
-
Duration
-
Type
-
Project
-
Service
-
Host
-
Details
-
-
-
+
+
-
-
+
diff --git a/osprofiler/drivers/__init__.py b/osprofiler/drivers/__init__.py
new file mode 100644
index 0000000..bb287da
--- /dev/null
+++ b/osprofiler/drivers/__init__.py
@@ -0,0 +1,7 @@
+from osprofiler.drivers import base # noqa
+from osprofiler.drivers import ceilometer # noqa
+from osprofiler.drivers import elasticsearch_driver # noqa
+from osprofiler.drivers import loginsight # noqa
+from osprofiler.drivers import messaging # noqa
+from osprofiler.drivers import mongodb # noqa
+from osprofiler.drivers import redis_driver # noqa
diff --git a/osprofiler/drivers/base.py b/osprofiler/drivers/base.py
new file mode 100644
index 0000000..3373be2
--- /dev/null
+++ b/osprofiler/drivers/base.py
@@ -0,0 +1,249 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import datetime
+
+from oslo_log import log
+import six.moves.urllib.parse as urlparse
+
+from osprofiler import _utils
+
+LOG = log.getLogger(__name__)
+
+
+def get_driver(connection_string, *args, **kwargs):
+ """Create driver's instance according to specified connection string"""
+ # NOTE(ayelistratov) Backward compatibility with old Messaging notation
+ # Remove after patching all OS services
+ # NOTE(ishakhat) Raise exception when ParsedResult.scheme is empty
+ if "://" not in connection_string:
+ connection_string += "://"
+
+ parsed_connection = urlparse.urlparse(connection_string)
+ LOG.debug("String %s looks like a connection string, trying it.",
+ connection_string)
+
+ backend = parsed_connection.scheme
+ for driver in _utils.itersubclasses(Driver):
+ if backend == driver.get_name():
+ return driver(connection_string, *args, **kwargs)
+
+ raise ValueError("Driver not found for connection string: "
+ "%s" % connection_string)
+
+
+class Driver(object):
+ """Base Driver class.
+
+ This class provides protected common methods that
+ do not rely on a specific storage backend. Public methods notify() and/or
+ get_report(), which require using storage backend API, must be overridden
+ and implemented by any class derived from this class.
+ """
+
+ def __init__(self, connection_str, project=None, service=None, host=None):
+ self.connection_str = connection_str
+ self.project = project
+ self.service = service
+ self.host = host
+ self.result = {}
+ self.started_at = None
+ self.finished_at = None
+ # Last trace started time
+ self.last_started_at = None
+
+ def notify(self, info, **kwargs):
+ """This method will be called on each notifier.notify() call.
+
+ To add new drivers you should, create new subclass of this class and
+ implement notify method.
+
+ :param info: Contains information about trace element.
+ In payload dict there are always 3 ids:
+ "base_id" - uuid that is common for all notifications
+ related to one trace. Used to simplify
+ retrieving of all trace elements from
+ the backend.
+ "parent_id" - uuid of parent element in trace
+ "trace_id" - uuid of current element in trace
+
+ With parent_id and trace_id it's quite simple to build
+ tree of trace elements, which simplify analyze of trace.
+
+ """
+ raise NotImplementedError("{0}: This method is either not supported "
+ "or has to be overridden".format(
+ self.get_name()))
+
+ def get_report(self, base_id):
+ """Forms and returns report composed from the stored notifications.
+
+ :param base_id: Base id of trace elements.
+ """
+ raise NotImplementedError("{0}: This method is either not supported "
+ "or has to be overridden".format(
+ self.get_name()))
+
+ @classmethod
+ def get_name(cls):
+ """Returns backend specific name for the driver."""
+ return cls.__name__
+
+ def list_traces(self, query, fields):
+ """Returns array of all base_id fields that match the given criteria
+
+ :param query: dict that specifies the query criteria
+ :param fields: iterable of strings that specifies the output fields
+ """
+ raise NotImplementedError("{0}: This method is either not supported "
+ "or has to be overridden".format(
+ self.get_name()))
+
+ @staticmethod
+ def _build_tree(nodes):
+ """Builds the tree (forest) data structure based on the list of nodes.
+
+ Tree building works in O(n*log(n)).
+
+ :param nodes: dict of nodes, where each node is a dictionary with fields
+ "parent_id", "trace_id", "info"
+ :returns: list of top level ("root") nodes in form of dictionaries,
+ each containing the "info" and "children" fields, where
+ "children" is the list of child nodes ("children" will be
+ empty for leafs)
+ """
+
+ tree = []
+
+ for trace_id in nodes:
+ node = nodes[trace_id]
+ node.setdefault("children", [])
+ parent_id = node["parent_id"]
+ if parent_id in nodes:
+ nodes[parent_id].setdefault("children", [])
+ nodes[parent_id]["children"].append(node)
+ else:
+ tree.append(node) # no parent => top-level node
+
+ for trace_id in nodes:
+ nodes[trace_id]["children"].sort(
+ key=lambda x: x["info"]["started"])
+
+ return sorted(tree, key=lambda x: x["info"]["started"])
+
+ def _append_results(self, trace_id, parent_id, name, project, service,
+ host, timestamp, raw_payload=None):
+ """Appends the notification to the dictionary of notifications.
+
+ :param trace_id: UUID of current trace point
+ :param parent_id: UUID of parent trace point
+ :param name: name of operation
+ :param project: project name
+ :param service: service name
+ :param host: host name or FQDN
+ :param timestamp: Unicode-style timestamp matching the pattern
+ "%Y-%m-%dT%H:%M:%S.%f" , e.g. 2016-04-18T17:42:10.77
+ :param raw_payload: raw notification without any filtering, with all
+ fields included
+ """
+ timestamp = datetime.datetime.strptime(timestamp,
+ "%Y-%m-%dT%H:%M:%S.%f")
+ if trace_id not in self.result:
+ self.result[trace_id] = {
+ "info": {
+ "name": name.split("-")[0],
+ "project": project,
+ "service": service,
+ "host": host,
+ },
+ "trace_id": trace_id,
+ "parent_id": parent_id,
+ }
+
+ self.result[trace_id]["info"]["meta.raw_payload.%s"
+ % name] = raw_payload
+
+ if name.endswith("stop"):
+ self.result[trace_id]["info"]["finished"] = timestamp
+ self.result[trace_id]["info"]["exception"] = "None"
+ if raw_payload and "info" in raw_payload:
+ exc = raw_payload["info"].get("etype", "None")
+ self.result[trace_id]["info"]["exception"] = exc
+ else:
+ self.result[trace_id]["info"]["started"] = timestamp
+ if not self.last_started_at or self.last_started_at < timestamp:
+ self.last_started_at = timestamp
+
+ if not self.started_at or self.started_at > timestamp:
+ self.started_at = timestamp
+
+ if not self.finished_at or self.finished_at < timestamp:
+ self.finished_at = timestamp
+
+ def _parse_results(self):
+ """Parses Driver's notifications placed by _append_results() .
+
+ :returns: full profiling report
+ """
+
+ def msec(dt):
+ # NOTE(boris-42): Unfortunately this is the simplest way that works
+ # in py26 and py27
+ microsec = (dt.microseconds + (dt.seconds + dt.days * 24 * 3600) *
+ 1e6)
+ return int(microsec / 1000.0)
+
+ stats = {}
+
+ for r in self.result.values():
+ # NOTE(boris-42): We are not able to guarantee that the backend
+ # consumed all messages => so we should at make duration 0ms.
+
+ if "started" not in r["info"]:
+ r["info"]["started"] = r["info"]["finished"]
+ if "finished" not in r["info"]:
+ r["info"]["finished"] = r["info"]["started"]
+
+ op_type = r["info"]["name"]
+ op_started = msec(r["info"]["started"] - self.started_at)
+ op_finished = msec(r["info"]["finished"] -
+ self.started_at)
+ duration = op_finished - op_started
+
+ r["info"]["started"] = op_started
+ r["info"]["finished"] = op_finished
+
+ if op_type not in stats:
+ stats[op_type] = {
+ "count": 1,
+ "duration": duration
+ }
+ else:
+ stats[op_type]["count"] += 1
+ stats[op_type]["duration"] += duration
+
+ return {
+ "info": {
+ "name": "total",
+ "started": 0,
+ "finished": msec(self.finished_at -
+ self.started_at) if self.started_at else None,
+ "last_trace_started": msec(
+ self.last_started_at - self.started_at
+ ) if self.started_at else None
+ },
+ "children": self._build_tree(self.result),
+ "stats": stats
+ }
diff --git a/osprofiler/drivers/ceilometer.py b/osprofiler/drivers/ceilometer.py
new file mode 100644
index 0000000..fb6bd95
--- /dev/null
+++ b/osprofiler/drivers/ceilometer.py
@@ -0,0 +1,79 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from osprofiler.drivers import base
+from osprofiler import exc
+
+
+class Ceilometer(base.Driver):
+ def __init__(self, connection_str, **kwargs):
+ """Driver receiving profiled information from ceilometer."""
+ super(Ceilometer, self).__init__(connection_str)
+ try:
+ import ceilometerclient.client
+ except ImportError:
+ raise exc.CommandError(
+ "To use this command, you should install "
+ "'ceilometerclient' manually. Use command:\n "
+ "'pip install python-ceilometerclient'.")
+
+ try:
+ self.client = ceilometerclient.client.get_client(
+ kwargs["ceilometer_api_version"], **kwargs)
+ except Exception as e:
+ if hasattr(e, "http_status") and e.http_status == 401:
+ msg = "Invalid OpenStack Identity credentials."
+ else:
+ msg = "Error occurred while connecting to Ceilometer: %s." % e
+ raise exc.CommandError(msg)
+
+ @classmethod
+ def get_name(cls):
+ return "ceilometer"
+
+ def get_report(self, base_id):
+ """Retrieves and parses notification from ceilometer.
+
+ :param base_id: Base id of trace elements.
+ """
+
+ _filter = [{"field": "base_id", "op": "eq", "value": base_id}]
+
+ # limit is hardcoded in this code state. Later that will be changed via
+ # connection string usage
+ notifications = [n.to_dict()
+ for n in self.client.events.list(_filter,
+ limit=100000)]
+
+ for n in notifications:
+ traits = n["traits"]
+
+ def find_field(f_name):
+ return [t["value"] for t in traits if t["name"] == f_name][0]
+
+ trace_id = find_field("trace_id")
+ parent_id = find_field("parent_id")
+ name = find_field("name")
+ project = find_field("project")
+ service = find_field("service")
+ host = find_field("host")
+ timestamp = find_field("timestamp")
+
+ payload = n.get("raw", {}).get("payload", {})
+
+ self._append_results(trace_id, parent_id, name, project, service,
+ host, timestamp, payload)
+
+ return self._parse_results()
diff --git a/osprofiler/drivers/elasticsearch_driver.py b/osprofiler/drivers/elasticsearch_driver.py
new file mode 100644
index 0000000..6b7fc97
--- /dev/null
+++ b/osprofiler/drivers/elasticsearch_driver.py
@@ -0,0 +1,136 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import six.moves.urllib.parse as parser
+
+from oslo_config import cfg
+from osprofiler.drivers import base
+from osprofiler import exc
+
+
+class ElasticsearchDriver(base.Driver):
+ def __init__(self, connection_str, index_name="osprofiler-notifications",
+ project=None, service=None, host=None, conf=cfg.CONF,
+ **kwargs):
+ """Elasticsearch driver for OSProfiler."""
+
+ super(ElasticsearchDriver, self).__init__(connection_str,
+ project=project,
+ service=service, host=host)
+ try:
+ from elasticsearch import Elasticsearch
+ except ImportError:
+ raise exc.CommandError(
+ "To use this command, you should install "
+ "'elasticsearch' manually. Use command:\n "
+ "'pip install elasticsearch'.")
+
+ client_url = parser.urlunparse(parser.urlparse(self.connection_str)
+ ._replace(scheme="http"))
+ self.conf = conf
+ self.client = Elasticsearch(client_url)
+ self.index_name = index_name
+
+ @classmethod
+ def get_name(cls):
+ return "elasticsearch"
+
+ def notify(self, info):
+ """Send notifications to Elasticsearch.
+
+ :param info: Contains information about trace element.
+ In payload dict there are always 3 ids:
+ "base_id" - uuid that is common for all notifications
+ related to one trace. Used to simplify
+ retrieving of all trace elements from
+ Elasticsearch.
+ "parent_id" - uuid of parent element in trace
+ "trace_id" - uuid of current element in trace
+
+ With parent_id and trace_id it's quite simple to build
+ tree of trace elements, which simplify analyze of trace.
+
+ """
+
+ info = info.copy()
+ info["project"] = self.project
+ info["service"] = self.service
+ self.client.index(index=self.index_name,
+ doc_type=self.conf.profiler.es_doc_type, body=info)
+
+ def _hits(self, response):
+ """Returns all hits of search query using scrolling
+
+ :param response: ElasticSearch query response
+ """
+ scroll_id = response["_scroll_id"]
+ scroll_size = len(response["hits"]["hits"])
+ result = []
+
+ while scroll_size > 0:
+ for hit in response["hits"]["hits"]:
+ result.append(hit["_source"])
+ response = self.client.scroll(scroll_id=scroll_id,
+ scroll=self.conf.profiler.
+ es_scroll_time)
+ scroll_id = response["_scroll_id"]
+ scroll_size = len(response["hits"]["hits"])
+
+ return result
+
+ def list_traces(self, query={"match_all": {}}, fields=[]):
+ """Returns array of all base_id fields that match the given criteria
+
+ :param query: dict that specifies the query criteria
+ :param fields: iterable of strings that specifies the output fields
+ """
+ for base_field in ["base_id", "timestamp"]:
+ if base_field not in fields:
+ fields.append(base_field)
+
+ response = self.client.search(index=self.index_name,
+ doc_type=self.conf.profiler.es_doc_type,
+ size=self.conf.profiler.es_scroll_size,
+ scroll=self.conf.profiler.es_scroll_time,
+ body={"_source": fields, "query": query,
+ "sort": [{"timestamp": "asc"}]})
+
+ return self._hits(response)
+
+ def get_report(self, base_id):
+ """Retrieves and parses notification from Elasticsearch.
+
+ :param base_id: Base id of trace elements.
+ """
+ response = self.client.search(index=self.index_name,
+ doc_type=self.conf.profiler.es_doc_type,
+ size=self.conf.profiler.es_scroll_size,
+ scroll=self.conf.profiler.es_scroll_time,
+ body={"query": {
+ "match": {"base_id": base_id}}})
+
+ for n in self._hits(response):
+ trace_id = n["trace_id"]
+ parent_id = n["parent_id"]
+ name = n["name"]
+ project = n["project"]
+ service = n["service"]
+ host = n["info"]["host"]
+ timestamp = n["timestamp"]
+
+ self._append_results(trace_id, parent_id, name, project, service,
+ host, timestamp, n)
+
+ return self._parse_results()
diff --git a/osprofiler/drivers/loginsight.py b/osprofiler/drivers/loginsight.py
new file mode 100644
index 0000000..4e875ae
--- /dev/null
+++ b/osprofiler/drivers/loginsight.py
@@ -0,0 +1,263 @@
+# Copyright (c) 2016 VMware, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Classes to use VMware vRealize Log Insight as the trace data store.
+"""
+
+import json
+import logging as log
+
+import netaddr
+from oslo_concurrency.lockutils import synchronized
+import requests
+import six.moves.urllib.parse as urlparse
+
+from osprofiler.drivers import base
+from osprofiler import exc
+
+LOG = log.getLogger(__name__)
+
+
+class LogInsightDriver(base.Driver):
+ """Driver for storing trace data in VMware vRealize Log Insight.
+
+ The driver uses Log Insight ingest service to store trace data and uses
+ the query service to retrieve it. The minimum required Log Insight version
+ is 3.3.
+
+ The connection string to initialize the driver should be of the format:
+ loginsight://:@
+
+ If the username or password contains the character ':' or '@', it must be
+ escaped using URL encoding. For example, the connection string to connect
+ to Log Insight server at 10.1.2.3 using username "osprofiler" and password
+ "p@ssword" is:
+ loginsight://osprofiler:p%40ssword@10.1.2.3
+ """
+ def __init__(
+ self, connection_str, project=None, service=None, host=None,
+ **kwargs):
+ super(LogInsightDriver, self).__init__(connection_str,
+ project=project,
+ service=service,
+ host=host)
+
+ parsed_connection = urlparse.urlparse(connection_str)
+ try:
+ creds, host = parsed_connection.netloc.split("@")
+ username, password = creds.split(":")
+ except ValueError:
+ raise ValueError("Connection string format is: loginsight://"
+ ":@. If the "
+ "username or password contains the character '@' "
+ "or ':', it must be escaped using URL encoding.")
+
+ username = urlparse.unquote(username)
+ password = urlparse.unquote(password)
+ self._client = LogInsightClient(host, username, password)
+
+ self._client.login()
+
+ @classmethod
+ def get_name(cls):
+ return "loginsight"
+
+ def notify(self, info):
+ """Send trace to Log Insight server."""
+
+ trace = info.copy()
+ trace["project"] = self.project
+ trace["service"] = self.service
+
+ event = {"text": "OSProfiler trace"}
+
+ def _create_field(name, content):
+ return {"name": name, "content": content}
+
+ event["fields"] = [_create_field("base_id", trace["base_id"]),
+ _create_field("trace_id", trace["trace_id"]),
+ _create_field("project", trace["project"]),
+ _create_field("service", trace["service"]),
+ _create_field("name", trace["name"]),
+ _create_field("trace", json.dumps(trace))]
+
+ self._client.send_event(event)
+
+ def get_report(self, base_id):
+ """Retrieves and parses trace data from Log Insight.
+
+ :param base_id: Trace base ID
+ """
+ response = self._client.query_events({"base_id": base_id})
+
+ if "events" in response:
+ for event in response["events"]:
+ if "fields" not in event:
+ continue
+
+ for field in event["fields"]:
+ if field["name"] == "trace":
+ trace = json.loads(field["content"])
+ trace_id = trace["trace_id"]
+ parent_id = trace["parent_id"]
+ name = trace["name"]
+ project = trace["project"]
+ service = trace["service"]
+ host = trace["info"]["host"]
+ timestamp = trace["timestamp"]
+
+ self._append_results(
+ trace_id, parent_id, name, project, service, host,
+ timestamp, trace)
+ break
+
+ return self._parse_results()
+
+
+class LogInsightClient(object):
+ """A minimal Log Insight client."""
+
+ LI_OSPROFILER_AGENT_ID = "F52D775B-6017-4787-8C8A-F21AE0AEC057"
+
+ # API paths
+ SESSIONS_PATH = "api/v1/sessions"
+ CURRENT_SESSIONS_PATH = "api/v1/sessions/current"
+ EVENTS_INGEST_PATH = "api/v1/events/ingest/%s" % LI_OSPROFILER_AGENT_ID
+ QUERY_EVENTS_BASE_PATH = "api/v1/events"
+
+ def __init__(self, host, username, password, api_port=9000,
+ api_ssl_port=9543, query_timeout=60000):
+ self._host = host
+ self._username = username
+ self._password = password
+ self._api_port = api_port
+ self._api_ssl_port = api_ssl_port
+ self._query_timeout = query_timeout
+ self._session = requests.Session()
+ self._session_id = None
+
+ def _build_base_url(self, scheme):
+ proto_str = "%s://" % scheme
+ host_str = ("[%s]" % self._host if netaddr.valid_ipv6(self._host)
+ else self._host)
+ port_str = ":%d" % (self._api_ssl_port if scheme == "https"
+ else self._api_port)
+ return proto_str + host_str + port_str
+
+ def _check_response(self, resp):
+ if resp.status_code == 440:
+ raise exc.LogInsightLoginTimeout()
+
+ if not resp.ok:
+ msg = "n/a"
+ if resp.text:
+ try:
+ body = json.loads(resp.text)
+ msg = body.get("errorMessage", msg)
+ except ValueError:
+ pass
+ else:
+ msg = resp.reason
+ raise exc.LogInsightAPIError(msg)
+
+ def _send_request(
+ self, method, scheme, path, headers=None, body=None, params=None):
+ url = "%s/%s" % (self._build_base_url(scheme), path)
+
+ headers = headers or {}
+ headers["content-type"] = "application/json"
+ body = body or {}
+ params = params or {}
+
+ req = requests.Request(
+ method, url, headers=headers, data=json.dumps(body), params=params)
+ req = req.prepare()
+ resp = self._session.send(req, verify=False)
+
+ self._check_response(resp)
+ return resp.json()
+
+ def _get_auth_header(self):
+ return {"X-LI-Session-Id": self._session_id}
+
+ def _trunc_session_id(self):
+ if self._session_id:
+ return self._session_id[-5:]
+
+ def _is_current_session_active(self):
+ try:
+ self._send_request("get",
+ "https",
+ self.CURRENT_SESSIONS_PATH,
+ headers=self._get_auth_header())
+ LOG.debug("Current session %s is active.",
+ self._trunc_session_id())
+ return True
+ except (exc.LogInsightLoginTimeout, exc.LogInsightAPIError):
+ LOG.debug("Current session %s is not active.",
+ self._trunc_session_id())
+ return False
+
+ @synchronized("li_login_lock")
+ def login(self):
+ # Another thread might have created the session while the current
+ # thread was waiting for the lock.
+ if self._session_id and self._is_current_session_active():
+ return
+
+ LOG.info("Logging into Log Insight server: %s.", self._host)
+ resp = self._send_request("post",
+ "https",
+ self.SESSIONS_PATH,
+ body={"username": self._username,
+ "password": self._password})
+
+ self._session_id = resp["sessionId"]
+ LOG.debug("Established session %s.", self._trunc_session_id())
+
+ def send_event(self, event):
+ events = {"events": [event]}
+ self._send_request("post",
+ "http",
+ self.EVENTS_INGEST_PATH,
+ body=events)
+
+ def query_events(self, params):
+ # Assumes that the keys and values in the params are strings and
+ # the operator is "CONTAINS".
+ constraints = []
+ for field, value in params.items():
+ constraints.append("%s/CONTAINS+%s" % (field, value))
+ constraints.append("timestamp/GT+0")
+
+ path = "%s/%s" % (self.QUERY_EVENTS_BASE_PATH, "/".join(constraints))
+
+ def _query_events():
+ return self._send_request("get",
+ "https",
+ path,
+ headers=self._get_auth_header(),
+ params={"limit": 20000,
+ "timeout": self._query_timeout})
+ try:
+ resp = _query_events()
+ except exc.LogInsightLoginTimeout:
+ # Login again and re-try.
+ LOG.debug("Current session timed out.")
+ self.login()
+ resp = _query_events()
+
+ return resp
diff --git a/osprofiler/drivers/messaging.py b/osprofiler/drivers/messaging.py
new file mode 100644
index 0000000..47a8a81
--- /dev/null
+++ b/osprofiler/drivers/messaging.py
@@ -0,0 +1,62 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from osprofiler.drivers import base
+
+
+class Messaging(base.Driver):
+ def __init__(self, connection_str, messaging=None, context=None,
+ transport=None, project=None, service=None,
+ host=None, **kwargs):
+ """Driver sending notifications via message queues."""
+
+ super(Messaging, self).__init__(connection_str, project=project,
+ service=service, host=host)
+
+ self.messaging = messaging
+ self.context = context
+
+ self.client = messaging.Notifier(
+ transport, publisher_id=self.host, driver="messaging",
+ topics=["profiler"], retry=0)
+
+ @classmethod
+ def get_name(cls):
+ return "messaging"
+
+ def notify(self, info, context=None):
+ """Send notifications to backend via oslo.messaging notifier API.
+
+ :param info: Contains information about trace element.
+ In payload dict there are always 3 ids:
+ "base_id" - uuid that is common for all notifications
+ related to one trace. Used to simplify
+ retrieving of all trace elements from
+ Ceilometer.
+ "parent_id" - uuid of parent element in trace
+ "trace_id" - uuid of current element in trace
+
+ With parent_id and trace_id it's quite simple to build
+ tree of trace elements, which simplify analyze of trace.
+
+ :param context: request context that is mostly used to specify
+ current active user and tenant.
+ """
+
+ info["project"] = self.project
+ info["service"] = self.service
+ self.client.info(context or self.context,
+ "profiler.%s" % info["service"],
+ info)
diff --git a/osprofiler/drivers/mongodb.py b/osprofiler/drivers/mongodb.py
new file mode 100644
index 0000000..60d7b2c
--- /dev/null
+++ b/osprofiler/drivers/mongodb.py
@@ -0,0 +1,92 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from osprofiler.drivers import base
+from osprofiler import exc
+
+
+class MongoDB(base.Driver):
+ def __init__(self, connection_str, db_name="osprofiler", project=None,
+ service=None, host=None, **kwargs):
+ """MongoDB driver for OSProfiler."""
+
+ super(MongoDB, self).__init__(connection_str, project=project,
+ service=service, host=host)
+ try:
+ from pymongo import MongoClient
+ except ImportError:
+ raise exc.CommandError(
+ "To use this command, you should install "
+ "'pymongo' manually. Use command:\n "
+ "'pip install pymongo'.")
+
+ client = MongoClient(self.connection_str, connect=False)
+ self.db = client[db_name]
+
+ @classmethod
+ def get_name(cls):
+ return "mongodb"
+
+ def notify(self, info):
+ """Send notifications to MongoDB.
+
+ :param info: Contains information about trace element.
+ In payload dict there are always 3 ids:
+ "base_id" - uuid that is common for all notifications
+ related to one trace. Used to simplify
+ retrieving of all trace elements from
+ MongoDB.
+ "parent_id" - uuid of parent element in trace
+ "trace_id" - uuid of current element in trace
+
+ With parent_id and trace_id it's quite simple to build
+ tree of trace elements, which simplify analyze of trace.
+
+ """
+ data = info.copy()
+ data["project"] = self.project
+ data["service"] = self.service
+ self.db.profiler.insert_one(data)
+
+ def list_traces(self, query, fields=[]):
+ """Returns array of all base_id fields that match the given criteria
+
+ :param query: dict that specifies the query criteria
+ :param fields: iterable of strings that specifies the output fields
+ """
+ ids = self.db.profiler.find(query).distinct("base_id")
+ out_format = {"base_id": 1, "timestamp": 1, "_id": 0}
+ out_format.update({i: 1 for i in fields})
+ return [self.db.profiler.find(
+ {"base_id": i}, out_format).sort("timestamp")[0] for i in ids]
+
+ def get_report(self, base_id):
+ """Retrieves and parses notification from MongoDB.
+
+ :param base_id: Base id of trace elements.
+ """
+ for n in self.db.profiler.find({"base_id": base_id}, {"_id": 0}):
+ trace_id = n["trace_id"]
+ parent_id = n["parent_id"]
+ name = n["name"]
+ project = n["project"]
+ service = n["service"]
+ host = n["info"]["host"]
+ timestamp = n["timestamp"]
+
+ self._append_results(trace_id, parent_id, name, project, service,
+ host, timestamp, n)
+
+ return self._parse_results()
diff --git a/osprofiler/drivers/redis_driver.py b/osprofiler/drivers/redis_driver.py
new file mode 100644
index 0000000..f3f5812
--- /dev/null
+++ b/osprofiler/drivers/redis_driver.py
@@ -0,0 +1,137 @@
+# Copyright 2016 Mirantis Inc.
+# Copyright 2016 IBM Corporation.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_serialization import jsonutils
+import six.moves.urllib.parse as parser
+
+from osprofiler.drivers import base
+from osprofiler import exc
+
+
+class Redis(base.Driver):
+ def __init__(self, connection_str, db=0, project=None,
+ service=None, host=None, **kwargs):
+ """Redis driver for OSProfiler."""
+
+ super(Redis, self).__init__(connection_str, project=project,
+ service=service, host=host)
+ try:
+ from redis import StrictRedis
+ except ImportError:
+ raise exc.CommandError(
+ "To use this command, you should install "
+ "'redis' manually. Use command:\n "
+ "'pip install redis'.")
+
+ parsed_url = parser.urlparse(self.connection_str)
+ self.db = StrictRedis(host=parsed_url.hostname,
+ port=parsed_url.port,
+ db=db)
+ self.namespace = "osprofiler:"
+
+ @classmethod
+ def get_name(cls):
+ return "redis"
+
+ def notify(self, info):
+ """Send notifications to Redis.
+
+ :param info: Contains information about trace element.
+ In payload dict there are always 3 ids:
+ "base_id" - uuid that is common for all notifications
+ related to one trace. Used to simplify
+ retrieving of all trace elements from
+ Redis.
+ "parent_id" - uuid of parent element in trace
+ "trace_id" - uuid of current element in trace
+
+ With parent_id and trace_id it's quite simple to build
+ tree of trace elements, which simplify analyze of trace.
+
+ """
+ data = info.copy()
+ data["project"] = self.project
+ data["service"] = self.service
+ key = self.namespace + data["base_id"] + "_" + data["trace_id"] + "_" + \
+ data["timestamp"]
+ self.db.set(key, jsonutils.dumps(data))
+
+ def list_traces(self, query="*", fields=[]):
+ """Returns array of all base_id fields that match the given criteria
+
+ :param query: string that specifies the query criteria
+ :param fields: iterable of strings that specifies the output fields
+ """
+ for base_field in ["base_id", "timestamp"]:
+ if base_field not in fields:
+ fields.append(base_field)
+ ids = self.db.scan_iter(match=self.namespace + query)
+ traces = [jsonutils.loads(self.db.get(i)) for i in ids]
+ result = []
+ for trace in traces:
+ result.append({key: value for key, value in trace.iteritems()
+ if key in fields})
+ return result
+
+ def get_report(self, base_id):
+ """Retrieves and parses notification from Redis.
+
+ :param base_id: Base id of trace elements.
+ """
+ for key in self.db.scan_iter(match=self.namespace + base_id + "*"):
+ data = self.db.get(key)
+ n = jsonutils.loads(data)
+ trace_id = n["trace_id"]
+ parent_id = n["parent_id"]
+ name = n["name"]
+ project = n["project"]
+ service = n["service"]
+ host = n["info"]["host"]
+ timestamp = n["timestamp"]
+
+ self._append_results(trace_id, parent_id, name, project, service,
+ host, timestamp, n)
+
+ return self._parse_results()
+
+
+class RedisSentinel(Redis, base.Driver):
+ def __init__(self, connection_str, db=0, project=None,
+ service=None, host=None, conf=cfg.CONF, **kwargs):
+ """Redis driver for OSProfiler."""
+
+ super(RedisSentinel, self).__init__(connection_str, project=project,
+ service=service, host=host)
+ try:
+ from redis.sentinel import Sentinel
+ except ImportError:
+ raise exc.CommandError(
+ "To use this command, you should install "
+ "'redis' manually. Use command:\n "
+ "'pip install redis'.")
+
+ self.conf = conf
+ socket_timeout = self.conf.profiler.socket_timeout
+ parsed_url = parser.urlparse(self.connection_str)
+ sentinel = Sentinel([(parsed_url.hostname, int(parsed_url.port))],
+ socket_timeout=socket_timeout)
+ self.db = sentinel.master_for(self.conf.profiler.sentinel_service_name,
+ socket_timeout=socket_timeout)
+
+ @classmethod
+ def get_name(cls):
+ return "redissentinel"
diff --git a/osprofiler/exc.py b/osprofiler/exc.py
new file mode 100644
index 0000000..0f2fa33
--- /dev/null
+++ b/osprofiler/exc.py
@@ -0,0 +1,32 @@
+# Copyright 2014 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+class CommandError(Exception):
+ """Invalid usage of CLI."""
+
+ def __init__(self, message=None):
+ self.message = message
+
+ def __str__(self):
+ return self.message or self.__class__.__doc__
+
+
+class LogInsightAPIError(Exception):
+ pass
+
+
+class LogInsightLoginTimeout(Exception):
+ pass
diff --git a/osprofiler/hacking/__init__.py b/osprofiler/hacking/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/osprofiler/hacking/checks.py b/osprofiler/hacking/checks.py
new file mode 100644
index 0000000..5b6e45d
--- /dev/null
+++ b/osprofiler/hacking/checks.py
@@ -0,0 +1,378 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Guidelines for writing new hacking checks
+
+ - Use only for OSProfiler specific tests. OpenStack general tests
+ should be submitted to the common 'hacking' module.
+ - Pick numbers in the range N3xx. Find the current test with
+ the highest allocated number and then pick the next value.
+ - Keep the test method code in the source file ordered based
+ on the N3xx value.
+ - List the new rule in the top level HACKING.rst file
+ - Add test cases for each new rule to tests/unit/test_hacking.py
+
+"""
+
+import functools
+import re
+import tokenize
+
+re_assert_true_instance = re.compile(
+ r"(.)*assertTrue\(isinstance\((\w|\.|\'|\"|\[|\])+, "
+ r"(\w|\.|\'|\"|\[|\])+\)\)")
+re_assert_equal_type = re.compile(
+ r"(.)*assertEqual\(type\((\w|\.|\'|\"|\[|\])+\), "
+ r"(\w|\.|\'|\"|\[|\])+\)")
+re_assert_equal_end_with_none = re.compile(r"assertEqual\(.*?,\s+None\)$")
+re_assert_equal_start_with_none = re.compile(r"assertEqual\(None,")
+re_assert_true_false_with_in_or_not_in = re.compile(
+ r"assert(True|False)\("
+ r"(\w|[][.'\"])+( not)? in (\w|[][.'\",])+(, .*)?\)")
+re_assert_true_false_with_in_or_not_in_spaces = re.compile(
+ r"assert(True|False)\((\w|[][.'\"])+( not)? in [\[|'|\"](\w|[][.'\", ])+"
+ r"[\[|'|\"](, .*)?\)")
+re_assert_equal_in_end_with_true_or_false = re.compile(
+ r"assertEqual\((\w|[][.'\"])+( not)? in (\w|[][.'\", ])+, (True|False)\)")
+re_assert_equal_in_start_with_true_or_false = re.compile(
+ r"assertEqual\((True|False), (\w|[][.'\"])+( not)? in (\w|[][.'\", ])+\)")
+re_no_construct_dict = re.compile(
+ r"\sdict\(\)")
+re_no_construct_list = re.compile(
+ r"\slist\(\)")
+re_str_format = re.compile(r"""
+% # start of specifier
+\(([^)]+)\) # mapping key, in group 1
+[#0 +\-]? # optional conversion flag
+(?:-?\d*)? # optional minimum field width
+(?:\.\d*)? # optional precision
+[hLl]? # optional length modifier
+[A-z%] # conversion modifier
+""", re.X)
+re_raises = re.compile(
+ r"\s:raise[^s] *.*$|\s:raises *:.*$|\s:raises *[^:]+$")
+
+
+def skip_ignored_lines(func):
+
+ @functools.wraps(func)
+ def wrapper(logical_line, filename):
+ line = logical_line.strip()
+ if not line or line.startswith("#") or line.endswith("# noqa"):
+ return
+ yield next(func(logical_line, filename))
+
+ return wrapper
+
+
+def _parse_assert_mock_str(line):
+ point = line.find(".assert_")
+
+ if point != -1:
+ end_pos = line[point:].find("(") + point
+ return point, line[point + 1: end_pos], line[: point]
+ else:
+ return None, None, None
+
+
+@skip_ignored_lines
+def check_assert_methods_from_mock(logical_line, filename):
+ """Ensure that ``assert_*`` methods from ``mock`` library is used correctly
+
+ N301 - base error number
+ N302 - related to nonexistent "assert_called"
+ N303 - related to nonexistent "assert_called_once"
+ """
+
+ correct_names = ["assert_any_call", "assert_called_once_with",
+ "assert_called_with", "assert_has_calls"]
+ ignored_files = ["./tests/unit/test_hacking.py"]
+
+ if filename.startswith("./tests") and filename not in ignored_files:
+ pos, method_name, obj_name = _parse_assert_mock_str(logical_line)
+
+ if pos:
+ if method_name not in correct_names:
+ error_number = "N301"
+ msg = ("%(error_number)s:'%(method)s' is not present in `mock`"
+ " library. %(custom_msg)s For more details, visit "
+ "http://www.voidspace.org.uk/python/mock/ .")
+
+ if method_name == "assert_called":
+ error_number = "N302"
+ custom_msg = ("Maybe, you should try to use "
+ "'assertTrue(%s.called)' instead." %
+ obj_name)
+ elif method_name == "assert_called_once":
+ # For more details, see a bug in Rally:
+ # https://bugs.launchpad.net/rally/+bug/1305991
+ error_number = "N303"
+ custom_msg = ("Maybe, you should try to use "
+ "'assertEqual(1, %s.call_count)' "
+ "or '%s.assert_called_once_with()'"
+ " instead." % (obj_name, obj_name))
+ else:
+ custom_msg = ("Correct 'assert_*' methods: '%s'."
+ % "', '".join(correct_names))
+
+ yield (pos, msg % {
+ "error_number": error_number,
+ "method": method_name,
+ "custom_msg": custom_msg})
+
+
+@skip_ignored_lines
+def assert_true_instance(logical_line, filename):
+ """Check for assertTrue(isinstance(a, b)) sentences
+
+ N320
+ """
+ if re_assert_true_instance.match(logical_line):
+ yield (0, "N320 assertTrue(isinstance(a, b)) sentences not allowed, "
+ "you should use assertIsInstance(a, b) instead.")
+
+
+@skip_ignored_lines
+def assert_equal_type(logical_line, filename):
+ """Check for assertEqual(type(A), B) sentences
+
+ N321
+ """
+ if re_assert_equal_type.match(logical_line):
+ yield (0, "N321 assertEqual(type(A), B) sentences not allowed, "
+ "you should use assertIsInstance(a, b) instead.")
+
+
+@skip_ignored_lines
+def assert_equal_none(logical_line, filename):
+ """Check for assertEqual(A, None) or assertEqual(None, A) sentences
+
+ N322
+ """
+ res = (re_assert_equal_start_with_none.search(logical_line) or
+ re_assert_equal_end_with_none.search(logical_line))
+ if res:
+ yield (0, "N322 assertEqual(A, None) or assertEqual(None, A) "
+ "sentences not allowed, you should use assertIsNone(A) "
+ "instead.")
+
+
+@skip_ignored_lines
+def assert_true_or_false_with_in(logical_line, filename):
+ """Check assertTrue/False(A in/not in B) with collection contents
+
+ Check for assertTrue/False(A in B), assertTrue/False(A not in B),
+ assertTrue/False(A in B, message) or assertTrue/False(A not in B, message)
+ sentences.
+
+ N323
+ """
+ res = (re_assert_true_false_with_in_or_not_in.search(logical_line) or
+ re_assert_true_false_with_in_or_not_in_spaces.search(logical_line))
+ if res:
+ yield (0, "N323 assertTrue/assertFalse(A in/not in B)sentences not "
+ "allowed, you should use assertIn(A, B) or assertNotIn(A, B)"
+ " instead.")
+
+
+@skip_ignored_lines
+def assert_equal_in(logical_line, filename):
+ """Check assertEqual(A in/not in B, True/False) with collection contents
+
+ Check for assertEqual(A in B, True/False), assertEqual(True/False, A in B),
+ assertEqual(A not in B, True/False) or assertEqual(True/False, A not in B)
+ sentences.
+
+ N324
+ """
+ res = (re_assert_equal_in_end_with_true_or_false.search(logical_line) or
+ re_assert_equal_in_start_with_true_or_false.search(logical_line))
+ if res:
+ yield (0, "N324: Use assertIn/NotIn(A, B) rather than "
+ "assertEqual(A in/not in B, True/False) when checking "
+ "collection contents.")
+
+
+@skip_ignored_lines
+def check_quotes(logical_line, filename):
+ """Check that single quotation marks are not used
+
+ N350
+ """
+
+ in_string = False
+ in_multiline_string = False
+ single_quotas_are_used = False
+
+ check_tripple = (
+ lambda line, i, char: (
+ i + 2 < len(line) and
+ (char == line[i] == line[i + 1] == line[i + 2])
+ )
+ )
+
+ i = 0
+ while i < len(logical_line):
+ char = logical_line[i]
+
+ if in_string:
+ if char == "\"":
+ in_string = False
+ if char == "\\":
+ i += 1 # ignore next char
+
+ elif in_multiline_string:
+ if check_tripple(logical_line, i, "\""):
+ i += 2 # skip next 2 chars
+ in_multiline_string = False
+
+ elif char == "#":
+ break
+
+ elif char == "'":
+ single_quotas_are_used = True
+ break
+
+ elif char == "\"":
+ if check_tripple(logical_line, i, "\""):
+ in_multiline_string = True
+ i += 3
+ continue
+ in_string = True
+
+ i += 1
+
+ if single_quotas_are_used:
+ yield (i, "N350 Remove Single quotes")
+
+
+@skip_ignored_lines
+def check_no_constructor_data_struct(logical_line, filename):
+ """Check that data structs (lists, dicts) are declared using literals
+
+ N351
+ """
+
+ match = re_no_construct_dict.search(logical_line)
+ if match:
+ yield (0, "N351 Remove dict() construct and use literal {}")
+ match = re_no_construct_list.search(logical_line)
+ if match:
+ yield (0, "N351 Remove list() construct and use literal []")
+
+
+def check_dict_formatting_in_string(logical_line, tokens):
+ """Check that strings do not use dict-formatting with a single replacement
+
+ N352
+ """
+ # NOTE(stpierre): Can't use @skip_ignored_lines here because it's
+ # a stupid decorator that only works on functions that take
+ # (logical_line, filename) as arguments.
+ if (not logical_line or
+ logical_line.startswith("#") or
+ logical_line.endswith("# noqa")):
+ return
+
+ current_string = ""
+ in_string = False
+ for token_type, text, start, end, line in tokens:
+ if token_type == tokenize.STRING:
+ if not in_string:
+ current_string = ""
+ in_string = True
+ current_string += text.strip("\"")
+ elif token_type == tokenize.OP:
+ if not current_string:
+ continue
+ # NOTE(stpierre): The string formatting operator % has
+ # lower precedence than +, so we assume that the logical
+ # string has concluded whenever we hit an operator of any
+ # sort. (Most operators don't work for strings anyway.)
+ # Some string operators do have higher precedence than %,
+ # though, so you can technically trick this check by doing
+ # things like:
+ #
+ # "%(foo)s" * 1 % {"foo": 1}
+ # "%(foo)s"[:] % {"foo": 1}
+ #
+ # It also will produce false positives if you use explicit
+ # parenthesized addition for two strings instead of
+ # concatenation by juxtaposition, e.g.:
+ #
+ # ("%(foo)s" + "%(bar)s") % vals
+ #
+ # But if you do any of those things, then you deserve all
+ # of the horrible things that happen to you, and probably
+ # many more.
+ in_string = False
+ if text == "%":
+ format_keys = set()
+ for match in re_str_format.finditer(current_string):
+ format_keys.add(match.group(1))
+ if len(format_keys) == 1:
+ yield (0,
+ "N353 Do not use mapping key string formatting "
+ "with a single key")
+ if text != ")":
+ # NOTE(stpierre): You can have a parenthesized string
+ # followed by %, so a closing paren doesn't obviate
+ # the possibility for a substitution operator like
+ # every other operator does.
+ current_string = ""
+ elif token_type in (tokenize.NL, tokenize.COMMENT):
+ continue
+ else:
+ in_string = False
+ if token_type == tokenize.NEWLINE:
+ current_string = ""
+
+
+@skip_ignored_lines
+def check_using_unicode(logical_line, filename):
+ """Check crosspython unicode usage
+
+ N353
+ """
+
+ if re.search(r"\bunicode\(", logical_line):
+ yield (0, "N353 'unicode' function is absent in python3. Please "
+ "use 'six.text_type' instead.")
+
+
+def check_raises(physical_line, filename):
+ """Check raises usage
+
+ N354
+ """
+
+ ignored_files = ["./tests/unit/test_hacking.py",
+ "./tests/hacking/checks.py"]
+ if filename not in ignored_files:
+ if re_raises.search(physical_line):
+ return (0, "N354 ':Please use ':raises Exception: conditions' "
+ "in docstrings.")
+
+
+def factory(register):
+ register(check_assert_methods_from_mock)
+ register(assert_true_instance)
+ register(assert_equal_type)
+ register(assert_equal_none)
+ register(assert_true_or_false_with_in)
+ register(assert_equal_in)
+ register(check_quotes)
+ register(check_no_constructor_data_struct)
+ register(check_dict_formatting_in_string)
+ register(check_using_unicode)
+ register(check_raises)
diff --git a/osprofiler/initializer.py b/osprofiler/initializer.py
new file mode 100644
index 0000000..84e56d8
--- /dev/null
+++ b/osprofiler/initializer.py
@@ -0,0 +1,46 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import oslo_messaging
+
+from osprofiler import notifier
+from osprofiler import web
+
+
+def init_from_conf(conf, context, project, service, host):
+ """Initialize notifier from service configuration
+
+ :param conf: service configuration
+ :param context: request context
+ :param project: project name (keystone, cinder etc.)
+ :param service: service name that will be profiled
+ :param host: hostname or host IP address that the service will be
+ running on.
+ """
+ connection_str = conf.profiler.connection_string
+ kwargs = {}
+ if connection_str.startswith("messaging"):
+ kwargs = {"messaging": oslo_messaging,
+ "transport": oslo_messaging.get_notification_transport(conf)}
+ _notifier = notifier.create(
+ connection_str,
+ context=context,
+ project=project,
+ service=service,
+ host=host,
+ conf=conf,
+ **kwargs)
+ notifier.set(_notifier)
+ web.enable(conf.profiler.hmac_keys)
diff --git a/osprofiler/notifier.py b/osprofiler/notifier.py
index d27dd9f..3bd4412 100644
--- a/osprofiler/notifier.py
+++ b/osprofiler/notifier.py
@@ -13,7 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-from osprofiler._notifiers import base
+from osprofiler.drivers import base
def _noop_notifier(info, context=None):
@@ -22,6 +22,7 @@
# NOTE(boris-42): By default we are using noop notifier.
__notifier = _noop_notifier
+__driver_cache = {}
def notify(info):
@@ -48,13 +49,19 @@
__notifier = notifier
-def create(plugin_name, *args, **kwargs):
+def create(connection_string, *args, **kwargs):
"""Create notifier based on specified plugin_name
- :param plugin_name: Name of plugin that creates notifier
- :param *args: args that will be passed to plugin init method
- :param **kwargs: kwargs that will be passed to plugin init method
+ :param connection_string: connection string which specifies the storage
+ driver for notifier
+ :param *args: args that will be passed to the driver's __init__ method
+ :param **kwargs: kwargs that will be passed to the driver's __init__ method
:returns: Callable notifier method
:raises TypeError: In case of invalid name of plugin raises TypeError
"""
- return base.Notifier.factory(plugin_name, *args, **kwargs)
+ global __driver_cache
+ if connection_string not in __driver_cache:
+ __driver_cache[connection_string] = base.get_driver(connection_string,
+ *args,
+ **kwargs).notify
+ return __driver_cache[connection_string]
diff --git a/osprofiler/opts.py b/osprofiler/opts.py
index c7ecf2d..36b942f 100644
--- a/osprofiler/opts.py
+++ b/osprofiler/opts.py
@@ -33,7 +33,6 @@
_enabled_opt = cfg.BoolOpt(
"enabled",
default=False,
- deprecated_group="profiler",
deprecated_name="profiler_enabled",
help="""
Enables the profiling for all services on this node. Default value is False
@@ -79,14 +78,82 @@
ensures it can be used from client side to generate the trace, containing
information from all possible resources.""")
+_connection_string_opt = cfg.StrOpt(
+ "connection_string",
+ default="messaging://",
+ help="""
+Connection string for a notifier backend. Default value is messaging:// which
+sets the notifier to oslo_messaging.
+
+Examples of possible values:
+
+* messaging://: use oslo_messaging driver for sending notifications.
+* mongodb://127.0.0.1:27017 : use mongodb driver for sending notifications.
+* elasticsearch://127.0.0.1:9200 : use elasticsearch driver for sending
+notifications.
+""")
+
+_es_doc_type_opt = cfg.StrOpt(
+ "es_doc_type",
+ default="notification",
+ help="""
+Document type for notification indexing in elasticsearch.
+""")
+
+_es_scroll_time_opt = cfg.StrOpt(
+ "es_scroll_time",
+ default="2m",
+ help="""
+This parameter is a time value parameter (for example: es_scroll_time=2m),
+indicating for how long the nodes that participate in the search will maintain
+relevant resources in order to continue and support it.
+""")
+
+_es_scroll_size_opt = cfg.IntOpt(
+ "es_scroll_size",
+ default=10000,
+ help="""
+Elasticsearch splits large requests in batches. This parameter defines
+maximum size of each batch (for example: es_scroll_size=10000).
+""")
+
+_socket_timeout_opt = cfg.FloatOpt(
+ "socket_timeout",
+ default=0.1,
+ help="""
+Redissentinel provides a timeout option on the connections.
+This parameter defines that timeout (for example: socket_timeout=0.1).
+""")
+
+_sentinel_service_name_opt = cfg.StrOpt(
+ "sentinel_service_name",
+ default="mymaster",
+ help="""
+Redissentinel uses a service name to identify a master redis service.
+This parameter defines the name (for example:
+sentinal_service_name=mymaster).
+""")
+
+
_PROFILER_OPTS = [
_enabled_opt,
_trace_sqlalchemy_opt,
_hmac_keys_opt,
+ _connection_string_opt,
+ _es_doc_type_opt,
+ _es_scroll_time_opt,
+ _es_scroll_size_opt,
+ _socket_timeout_opt,
+ _sentinel_service_name_opt
]
-
-def set_defaults(conf, enabled=None, trace_sqlalchemy=None, hmac_keys=None):
+cfg.CONF.register_opts(_PROFILER_OPTS, group=_profiler_opt_group)
+
+
+def set_defaults(conf, enabled=None, trace_sqlalchemy=None, hmac_keys=None,
+ connection_string=None, es_doc_type=None,
+ es_scroll_time=None, es_scroll_size=None,
+ socket_timeout=None, sentinel_service_name=None):
conf.register_opts(_PROFILER_OPTS, group=_profiler_opt_group)
if enabled is not None:
@@ -99,6 +166,30 @@
conf.set_default("hmac_keys", hmac_keys,
group=_profiler_opt_group.name)
+ if connection_string is not None:
+ conf.set_default("connection_string", connection_string,
+ group=_profiler_opt_group.name)
+
+ if es_doc_type is not None:
+ conf.set_default("es_doc_type", es_doc_type,
+ group=_profiler_opt_group.name)
+
+ if es_scroll_time is not None:
+ conf.set_default("es_scroll_time", es_scroll_time,
+ group=_profiler_opt_group.name)
+
+ if es_scroll_size is not None:
+ conf.set_default("es_scroll_size", es_scroll_size,
+ group=_profiler_opt_group.name)
+
+ if socket_timeout is not None:
+ conf.set_default("socket_timeout", socket_timeout,
+ group=_profiler_opt_group.name)
+
+ if sentinel_service_name is not None:
+ conf.set_default("sentinel_service_name", sentinel_service_name,
+ group=_profiler_opt_group.name)
+
def is_trace_enabled(conf=None):
if conf is None:
diff --git a/osprofiler/parsers/__init__.py b/osprofiler/parsers/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/osprofiler/parsers/ceilometer.py b/osprofiler/parsers/ceilometer.py
deleted file mode 100644
index b267979..0000000
--- a/osprofiler/parsers/ceilometer.py
+++ /dev/null
@@ -1,138 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import datetime
-
-
-def _build_tree(nodes):
- """Builds the tree (forest) data structure based on the list of nodes.
-
- Works in O(n).
-
- :param nodes: list of nodes, where each node is a dictionary with fields
- "parent_id", "trace_id", "info"
- :returns: list of top level ("root") nodes in form of dictionaries,
- each containing the "info" and "children" fields, where
- "children" is the list of child nodes ("children" will be
- empty for leafs)
- """
-
- tree = []
-
- for trace_id in nodes:
- node = nodes[trace_id]
- node.setdefault("children", [])
- parent_id = node["parent_id"]
- if parent_id in nodes:
- nodes[parent_id].setdefault("children", [])
- nodes[parent_id]["children"].append(node)
- else:
- tree.append(node) # no parent => top-level node
-
- for node in nodes:
- nodes[node]["children"].sort(key=lambda x: x["info"]["started"])
-
- return sorted(tree, key=lambda x: x["info"]["started"])
-
-
-def parse_notifications(notifications):
- """Parse & builds tree structure from list of ceilometer notifications."""
-
- result = {}
- started_at = 0
- finished_at = 0
-
- for n in notifications:
- traits = n["traits"]
-
- def find_field(f_name):
- return [t["value"] for t in traits if t["name"] == f_name][0]
-
- trace_id = find_field("trace_id")
- parent_id = find_field("parent_id")
- name = find_field("name")
- project = find_field("project")
- service = find_field("service")
- host = find_field("host")
- timestamp = find_field("timestamp")
-
- timestamp = datetime.datetime.strptime(timestamp,
- "%Y-%m-%dT%H:%M:%S.%f")
-
- if trace_id not in result:
- result[trace_id] = {
- "info": {
- "name": name.split("-")[0],
- "project": project,
- "service": service,
- "host": host,
- },
- "trace_id": trace_id,
- "parent_id": parent_id,
- }
-
- result[trace_id]["info"]["meta.raw_payload.%s" % name] = n.get(
- "raw", {}).get("payload", {})
-
- if name.endswith("stop"):
- result[trace_id]["info"]["finished"] = timestamp
- else:
- result[trace_id]["info"]["started"] = timestamp
-
- if not started_at or started_at > timestamp:
- started_at = timestamp
-
- if not finished_at or finished_at < timestamp:
- finished_at = timestamp
-
- def msec(dt):
- # NOTE(boris-42): Unfortunately this is the simplest way that works in
- # py26 and py27
- microsec = (dt.microseconds + (dt.seconds + dt.days * 24 * 3600) * 1e6)
- return int(microsec / 1000.0)
-
- for r in result.values():
- # NOTE(boris-42): We are not able to guarantee that ceilometer consumed
- # all messages => so we should at make duration 0ms.
- if "started" not in r["info"]:
- r["info"]["started"] = r["info"]["finished"]
- if "finished" not in r["info"]:
- r["info"]["finished"] = r["info"]["started"]
-
- r["info"]["started"] = msec(r["info"]["started"] - started_at)
- r["info"]["finished"] = msec(r["info"]["finished"] - started_at)
-
- return {
- "info": {
- "name": "total",
- "started": 0,
- "finished": msec(finished_at - started_at) if started_at else 0
- },
- "children": _build_tree(result)
- }
-
-
-def get_notifications(ceilometer, base_id):
- """Retrieves and parses notification from ceilometer.
-
- :param ceilometer: Initialized ceilometer client.
- :param base_id: Base id of trace elements.
- """
-
- _filter = [{"field": "base_id", "op": "eq", "value": base_id}]
- # limit is hardcoded in this code state. Later that will be changed via
- # connection string usage
- return [n.to_dict()
- for n in ceilometer.events.list(_filter, limit=100000)]
diff --git a/osprofiler/profiler.py b/osprofiler/profiler.py
index 7e2c9b1..4e501a6 100644
--- a/osprofiler/profiler.py
+++ b/osprofiler/profiler.py
@@ -19,10 +19,9 @@
import inspect
import socket
import threading
-import uuid
from oslo_utils import reflection
-import six
+from oslo_utils import uuidutils
from osprofiler import notifier
@@ -35,7 +34,18 @@
__local_ctx.profiler = None
-def init(hmac_key, base_id=None, parent_id=None):
+def _ensure_no_multiple_traced(traceable_attrs):
+ for attr_name, attr in traceable_attrs:
+ traced_times = getattr(attr, "__traced__", 0)
+ if traced_times:
+ raise ValueError("Can not apply new trace on top of"
+ " previously traced attribute '%s' since"
+ " it has been traced %s times previously"
+ % (attr_name, traced_times))
+
+
+def init(hmac_key, base_id=None, parent_id=None, connection_str=None,
+ project=None, service=None):
"""Init profiler instance for current thread.
You should call profiler.init() before using osprofiler.
@@ -44,10 +54,16 @@
:param hmac_key: secret key to sign trace information.
:param base_id: Used to bind all related traces.
:param parent_id: Used to build tree of traces.
+ :param connection_str: Connection string to the backend to use for
+ notifications.
+ :param project: Project name that is under profiling
+ :param service: Service name that is under profiling
:returns: Profiler instance
"""
__local_ctx.profiler = _Profiler(hmac_key, base_id=base_id,
- parent_id=parent_id)
+ parent_id=parent_id,
+ connection_str=connection_str,
+ project=project, service=service)
return __local_ctx.profiler
@@ -78,7 +94,7 @@
profiler.stop(info=info)
-def trace(name, info=None, hide_args=False):
+def trace(name, info=None, hide_args=False, allow_multiple_trace=True):
"""Trace decorator for functions.
Very useful if you would like to add trace point on existing function:
@@ -93,6 +109,10 @@
:param hide_args: Don't push to trace info args and kwargs. Quite useful
if you have some info in args that you wont to share,
e.g. passwords.
+ :param allow_multiple_trace: If the wrapped function has already been
+ traced either allow the new trace to occur
+ or raise a value error denoting that multiple
+ tracing is not allowed (by default allow).
"""
if not info:
info = {}
@@ -101,6 +121,22 @@
info["function"] = {}
def decorator(f):
+ trace_times = getattr(f, "__traced__", 0)
+ if not allow_multiple_trace and trace_times:
+ raise ValueError("Function '%s' has already"
+ " been traced %s times" % (f, trace_times))
+
+ try:
+ f.__traced__ = trace_times + 1
+ except AttributeError:
+ # Tries to work around the following:
+ #
+ # AttributeError: 'instancemethod' object has no
+ # attribute '__traced__'
+ try:
+ f.im_func.__traced__ = trace_times + 1
+ except AttributeError: # nosec
+ pass
@functools.wraps(f)
def wrapper(*args, **kwargs):
@@ -121,7 +157,9 @@
return decorator
-def trace_cls(name, info=None, hide_args=False, trace_private=False):
+def trace_cls(name, info=None, hide_args=False,
+ trace_private=False, allow_multiple_trace=True,
+ trace_class_methods=False, trace_static_methods=False):
"""Trace decorator for instances of class .
Very useful if you would like to add trace point on existing method:
@@ -142,36 +180,63 @@
:param hide_args: Don't push to trace info args and kwargs. Quite useful
if you have some info in args that you wont to share,
e.g. passwords.
-
:param trace_private: Trace methods that starts with "_". It wont trace
methods that starts "__" even if it is turned on.
- """
+ :param trace_static_methods: Trace staticmethods. This may be prone to
+ issues so careful usage is recommended (this
+ is also why this defaults to false).
+ :param trace_class_methods: Trace classmethods. This may be prone to
+ issues so careful usage is recommended (this
+ is also why this defaults to false).
+ :param allow_multiple_trace: If wrapped attributes have already been
+ traced either allow the new trace to occur
+ or raise a value error denoting that multiple
+ tracing is not allowed (by default allow).
+ """
+
+ def trace_checker(attr_name, to_be_wrapped):
+ if attr_name.startswith("__"):
+ # Never trace really private methods.
+ return (False, None)
+ if not trace_private and attr_name.startswith("_"):
+ return (False, None)
+ if isinstance(to_be_wrapped, staticmethod):
+ if not trace_static_methods:
+ return (False, None)
+ return (True, staticmethod)
+ if isinstance(to_be_wrapped, classmethod):
+ if not trace_class_methods:
+ return (False, None)
+ return (True, classmethod)
+ return (True, None)
def decorator(cls):
clss = cls if inspect.isclass(cls) else cls.__class__
mro_dicts = [c.__dict__ for c in inspect.getmro(clss)]
+ traceable_attrs = []
+ traceable_wrappers = []
for attr_name, attr in inspect.getmembers(cls):
if not (inspect.ismethod(attr) or inspect.isfunction(attr)):
continue
- if attr_name.startswith("__"):
- continue
- if not trace_private and attr_name.startswith("_"):
- continue
-
wrapped_obj = None
for cls_dict in mro_dicts:
if attr_name in cls_dict:
wrapped_obj = cls_dict[attr_name]
break
-
+ should_wrap, wrapper = trace_checker(attr_name, wrapped_obj)
+ if not should_wrap:
+ continue
+ traceable_attrs.append((attr_name, attr))
+ traceable_wrappers.append(wrapper)
+ if not allow_multiple_trace:
+ # Check before doing any other further work (so we don't
+ # halfway trace this class).
+ _ensure_no_multiple_traced(traceable_attrs)
+ for i, (attr_name, attr) in enumerate(traceable_attrs):
wrapped_method = trace(name, info=info, hide_args=hide_args)(attr)
- if isinstance(wrapped_obj, staticmethod):
- # FIXME(dbelova): tracing staticmethod is prone to issues,
- # there are lots of edge cases, so let's figure that out later.
- continue
- # wrapped_method = staticmethod(wrapped_method)
- elif isinstance(wrapped_obj, classmethod):
- wrapped_method = classmethod(wrapped_method)
+ wrapper = traceable_wrappers[i]
+ if wrapper is not None:
+ wrapped_method = wrapper(wrapped_method)
setattr(cls, attr_name, wrapped_method)
return cls
@@ -206,12 +271,14 @@
trace_args = dict(getattr(cls, "__trace_args__", {}))
trace_private = trace_args.pop("trace_private", False)
+ allow_multiple_trace = trace_args.pop("allow_multiple_trace", True)
if "name" not in trace_args:
raise TypeError("Please specify __trace_args__ class level "
"dictionary attribute with mandatory 'name' key - "
"e.g. __trace_args__ = {'name': 'rpc'}")
- for attr_name, attr_value in six.iteritems(attrs):
+ traceable_attrs = []
+ for attr_name, attr_value in attrs.items():
if not (inspect.ismethod(attr_value) or
inspect.isfunction(attr_value)):
continue
@@ -219,7 +286,12 @@
continue
if not trace_private and attr_name.startswith("_"):
continue
-
+ traceable_attrs.append((attr_name, attr_value))
+ if not allow_multiple_trace:
+ # Check before doing any other further work (so we don't
+ # halfway trace this class).
+ _ensure_no_multiple_traced(traceable_attrs)
+ for attr_name, attr_value in traceable_attrs:
setattr(cls, attr_name, trace(**trace_args)(getattr(cls,
attr_name)))
@@ -248,18 +320,26 @@
start(self._name, info=self._info)
def __exit__(self, etype, value, traceback):
- stop()
+ if etype:
+ info = {"etype": reflection.get_class_name(etype)}
+ stop(info=info)
+ else:
+ stop()
class _Profiler(object):
- def __init__(self, hmac_key, base_id=None, parent_id=None):
+ def __init__(self, hmac_key, base_id=None, parent_id=None,
+ connection_str=None, project=None, service=None):
self.hmac_key = hmac_key
if not base_id:
- base_id = str(uuid.uuid4())
+ base_id = str(uuidutils.generate_uuid())
self._trace_stack = collections.deque([base_id, parent_id or base_id])
self._name = collections.deque()
self._host = socket.gethostname()
+ self._connection_str = connection_str
+ self._project = project
+ self._service = service
def get_base_id(self):
"""Return base id of a trace.
@@ -286,10 +366,6 @@
parent_id - to build tree of events (not just a list)
trace_id - current event id.
- As we are writing this code special for OpenStack, and there will be
- only one implementation of notifier based on ceilometer notifier api.
- That already contains timestamps, so we don't measure time by hand.
-
:param name: name of trace element (db, wsgi, rpc, etc..)
:param info: Dictionary with any useful information related to this
trace element. (sql request, rpc message or url...)
@@ -297,12 +373,14 @@
info = info or {}
info["host"] = self._host
+ info["project"] = self._project
+ info["service"] = self._service
self._name.append(name)
- self._trace_stack.append(str(uuid.uuid4()))
+ self._trace_stack.append(str(uuidutils.generate_uuid()))
self._notify("%s-start" % name, info)
def stop(self, info=None):
- """Finish latests event.
+ """Finish latest event.
Same as a start, but instead of pushing trace_id to stack it pops it.
@@ -310,6 +388,8 @@
"""
info = info or {}
info["host"] = self._host
+ info["project"] = self._project
+ info["service"] = self._service
self._notify("%s-stop" % self._name.pop(), info)
self._trace_stack.pop()
diff --git a/osprofiler/sqlalchemy.py b/osprofiler/sqlalchemy.py
index e98c59c..c593684 100644
--- a/osprofiler/sqlalchemy.py
+++ b/osprofiler/sqlalchemy.py
@@ -12,6 +12,8 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
+
+import contextlib
from osprofiler import profiler
@@ -42,6 +44,15 @@
_after_cursor_execute())
+@contextlib.contextmanager
+def wrap_session(sqlalchemy, sess):
+ with sess as s:
+ if not getattr(s.bind, "traced", False):
+ add_tracing(sqlalchemy, s.bind, "db")
+ s.bind.traced = True
+ yield s
+
+
def _before_cursor_execute(name):
"""Add listener that will send trace info before query is executed."""
diff --git a/osprofiler/tests/cmd/__init__.py b/osprofiler/tests/cmd/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/osprofiler/tests/cmd/test_shell.py b/osprofiler/tests/cmd/test_shell.py
deleted file mode 100644
index a146189..0000000
--- a/osprofiler/tests/cmd/test_shell.py
+++ /dev/null
@@ -1,228 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import json
-import os
-import sys
-
-import mock
-import six
-
-from osprofiler.cmd import exc
-from osprofiler.cmd import shell
-from osprofiler.tests import test
-
-
-class ShellTestCase(test.TestCase):
- def setUp(self):
- super(ShellTestCase, self).setUp()
- self.old_environment = os.environ.copy()
- os.environ = {
- "OS_USERNAME": "username",
- "OS_USER_ID": "user_id",
- "OS_PASSWORD": "password",
- "OS_USER_DOMAIN_ID": "user_domain_id",
- "OS_USER_DOMAIN_NAME": "user_domain_name",
- "OS_PROJECT_DOMAIN_ID": "project_domain_id",
- "OS_PROJECT_DOMAIN_NAME": "project_domain_name",
- "OS_PROJECT_ID": "project_id",
- "OS_PROJECT_NAME": "project_name",
- "OS_TENANT_ID": "tenant_id",
- "OS_TENANT_NAME": "tenant_name",
- "OS_AUTH_URL": "http://127.0.0.1:5000/v3/",
- "OS_AUTH_TOKEN": "pass",
- "OS_CACERT": "/path/to/cacert",
- "OS_SERVICE_TYPE": "service_type",
- "OS_ENDPOINT_TYPE": "public",
- "OS_REGION_NAME": "test"
- }
-
- self.ceiloclient = mock.MagicMock()
- sys.modules["ceilometerclient"] = self.ceiloclient
- self.addCleanup(sys.modules.pop, "ceilometerclient", None)
- ceilo_modules = ["client", "exc", "shell"]
- for module in ceilo_modules:
- sys.modules["ceilometerclient.%s" % module] = getattr(
- self.ceiloclient, module)
- self.addCleanup(
- sys.modules.pop, "ceilometerclient.%s" % module, None)
-
- def tearDown(self):
- super(ShellTestCase, self).tearDown()
- os.environ = self.old_environment
-
- @mock.patch("sys.stdout", six.StringIO())
- @mock.patch("osprofiler.cmd.shell.OSProfilerShell")
- def test_shell_main(self, mock_shell):
- mock_shell.side_effect = exc.CommandError("some_message")
- shell.main()
- self.assertEqual("some_message\n", sys.stdout.getvalue())
-
- def run_command(self, cmd):
- shell.OSProfilerShell(cmd.split())
-
- def _test_with_command_error(self, cmd, expected_message):
- try:
- self.run_command(cmd)
- except exc.CommandError as actual_error:
- self.assertEqual(str(actual_error), expected_message)
- else:
- raise ValueError(
- "Expected: `osprofiler.cmd.exc.CommandError` is raised with "
- "message: '%s'." % expected_message)
-
- def test_username_is_not_presented(self):
- os.environ.pop("OS_USERNAME")
- msg = ("You must provide a username via either --os-username or "
- "via env[OS_USERNAME]")
- self._test_with_command_error("trace show fake-uuid", msg)
-
- def test_password_is_not_presented(self):
- os.environ.pop("OS_PASSWORD")
- msg = ("You must provide a password via either --os-password or "
- "via env[OS_PASSWORD]")
- self._test_with_command_error("trace show fake-uuid", msg)
-
- def test_auth_url(self):
- os.environ.pop("OS_AUTH_URL")
- msg = ("You must provide an auth url via either --os-auth-url or "
- "via env[OS_AUTH_URL]")
- self._test_with_command_error("trace show fake-uuid", msg)
-
- def test_no_project_and_domain_set(self):
- os.environ.pop("OS_PROJECT_ID")
- os.environ.pop("OS_PROJECT_NAME")
- os.environ.pop("OS_TENANT_ID")
- os.environ.pop("OS_TENANT_NAME")
- os.environ.pop("OS_USER_DOMAIN_ID")
- os.environ.pop("OS_USER_DOMAIN_NAME")
-
- msg = ("You must provide a project_id via either --os-project-id or "
- "via env[OS_PROJECT_ID] and a domain_name via either "
- "--os-user-domain-name or via env[OS_USER_DOMAIN_NAME] or a "
- "domain_id via either --os-user-domain-id or via "
- "env[OS_USER_DOMAIN_ID]")
- self._test_with_command_error("trace show fake-uuid", msg)
-
- def test_trace_show_ceilometrclient_is_missed(self):
- sys.modules["ceilometerclient"] = None
- sys.modules["ceilometerclient.client"] = None
- sys.modules["ceilometerclient.exc"] = None
- sys.modules["ceilometerclient.shell"] = None
-
- self.assertRaises(ImportError, shell.main,
- "trace show fake_uuid".split())
-
- def test_trace_show_unauthorized(self):
- class FakeHTTPUnauthorized(Exception):
- http_status = 401
-
- self.ceiloclient.client.get_client.side_effect = FakeHTTPUnauthorized
-
- msg = "Invalid OpenStack Identity credentials."
- self._test_with_command_error("trace show fake_id", msg)
-
- def test_trace_show_unknown_error(self):
- class FakeException(Exception):
- pass
-
- self.ceiloclient.client.get_client.side_effect = FakeException
- msg = "Something has gone wrong. See logs for more details"
- self._test_with_command_error("trace show fake_id", msg)
-
- @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
- @mock.patch("osprofiler.parsers.ceilometer.parse_notifications")
- def test_trace_show_no_selected_format(self, mock_notifications, mock_get):
- mock_get.return_value = "some_notificatios"
- msg = ("You should choose one of the following output-formats: "
- "--json or --html.")
- self._test_with_command_error("trace show fake_id", msg)
-
- @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
- def test_trace_show_trace_id_not_found(self, mock_get):
- mock_get.return_value = None
-
- fake_trace_id = "fake_id"
- msg = ("Trace with UUID %s not found. There are 3 possible reasons: \n"
- " 1) You are using not admin credentials\n"
- " 2) You specified wrong trace id\n"
- " 3) You specified wrong HMAC Key in original calling"
- % fake_trace_id)
-
- self._test_with_command_error("trace show %s" % fake_trace_id, msg)
-
- @mock.patch("sys.stdout", six.StringIO())
- @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
- @mock.patch("osprofiler.parsers.ceilometer.parse_notifications")
- def test_trace_show_in_json(self, mock_notifications, mock_get):
- mock_get.return_value = "some notification"
- notifications = {
- "info": {
- "started": 0, "finished": 0, "name": "total"}, "children": []}
- mock_notifications.return_value = notifications
-
- self.run_command("trace show fake_id --json")
- self.assertEqual("%s\n" % json.dumps(notifications),
- sys.stdout.getvalue())
-
- @mock.patch("sys.stdout", six.StringIO())
- @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
- @mock.patch("osprofiler.parsers.ceilometer.parse_notifications")
- def test_trace_show_in_html(self, mock_notifications, mock_get):
- mock_get.return_value = "some notification"
-
- notifications = {
- "info": {
- "started": 0, "finished": 0, "name": "total"}, "children": []}
- mock_notifications.return_value = notifications
-
- # NOTE(akurilin): to simplify assert statement, html-template should be
- # replaced.
- html_template = (
- "A long time ago in a galaxy far, far away..."
- " some_data = $DATA"
- "It is a period of civil war. Rebel"
- "spaceships, striking from a hidden"
- "base, have won their first victory"
- "against the evil Galactic Empire.")
-
- with mock.patch("osprofiler.cmd.commands.open",
- mock.mock_open(read_data=html_template), create=True):
- self.run_command("trace show fake_id --html")
- self.assertEqual("A long time ago in a galaxy far, far away..."
- " some_data = %s"
- "It is a period of civil war. Rebel"
- "spaceships, striking from a hidden"
- "base, have won their first victory"
- "against the evil Galactic Empire."
- "\n" % json.dumps(notifications, indent=2),
- sys.stdout.getvalue())
-
- @mock.patch("sys.stdout", six.StringIO())
- @mock.patch("osprofiler.parsers.ceilometer.get_notifications")
- @mock.patch("osprofiler.parsers.ceilometer.parse_notifications")
- def test_trace_show_write_to_file(self, mock_notifications, mock_get):
- mock_get.return_value = "some notification"
- notifications = {
- "info": {
- "started": 0, "finished": 0, "name": "total"}, "children": []}
- mock_notifications.return_value = notifications
-
- with mock.patch("osprofiler.cmd.commands.open",
- mock.mock_open(), create=True) as mock_open:
- self.run_command("trace show fake_id --json --out='/file'")
-
- output = mock_open.return_value.__enter__.return_value
- output.write.assert_called_once_with(json.dumps(notifications))
diff --git a/osprofiler/tests/doc/__init__.py b/osprofiler/tests/doc/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/osprofiler/tests/doc/test_specs.py b/osprofiler/tests/doc/test_specs.py
deleted file mode 100644
index 39cabb5..0000000
--- a/osprofiler/tests/doc/test_specs.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import glob
-import os
-import re
-
-import docutils.core
-
-from osprofiler.tests import test
-
-
-class TitlesTestCase(test.TestCase):
-
- specs_path = os.path.join(
- os.path.dirname(__file__),
- os.pardir, os.pardir, os.pardir,
- "doc", "specs")
-
- def _get_title(self, section_tree):
- section = {"subtitles": []}
- for node in section_tree:
- if node.tagname == "title":
- section["name"] = node.rawsource
- elif node.tagname == "section":
- subsection = self._get_title(node)
- section["subtitles"].append(subsection["name"])
- return section
-
- def _get_titles(self, spec):
- titles = {}
- for node in spec:
- if node.tagname == "section":
- # Note subsection subtitles are thrown away
- section = self._get_title(node)
- titles[section["name"]] = section["subtitles"]
- return titles
-
- def _check_titles(self, filename, expect, actual):
- missing_sections = [x for x in expect.keys() if x not in actual.keys()]
- extra_sections = [x for x in actual.keys() if x not in expect.keys()]
-
- msgs = []
- if len(missing_sections) > 0:
- msgs.append("Missing sections: %s" % missing_sections)
- if len(extra_sections) > 0:
- msgs.append("Extra sections: %s" % extra_sections)
-
- for section in expect.keys():
- missing_subsections = [x for x in expect[section]
- if x not in actual.get(section, {})]
- # extra subsections are allowed
- if len(missing_subsections) > 0:
- msgs.append("Section '%s' is missing subsections: %s"
- % (section, missing_subsections))
-
- if len(msgs) > 0:
- self.fail("While checking '%s':\n %s"
- % (filename, "\n ".join(msgs)))
-
- def _check_lines_wrapping(self, tpl, raw):
- for i, line in enumerate(raw.split("\n")):
- if "http://" in line or "https://" in line:
- continue
- self.assertTrue(
- len(line) < 80,
- msg="%s:%d: Line limited to a maximum of 79 characters." %
- (tpl, i+1))
-
- def _check_no_cr(self, tpl, raw):
- matches = re.findall("\r", raw)
- self.assertEqual(
- len(matches), 0,
- "Found %s literal carriage returns in file %s" %
- (len(matches), tpl))
-
- def _check_trailing_spaces(self, tpl, raw):
- for i, line in enumerate(raw.split("\n")):
- trailing_spaces = re.findall(" +$", line)
- self.assertEqual(
- len(trailing_spaces), 0,
- "Found trailing spaces on line %s of %s" % (i+1, tpl))
-
- def test_template(self):
- with open(os.path.join(self.specs_path, "template.rst")) as f:
- template = f.read()
-
- spec = docutils.core.publish_doctree(template)
- template_titles = self._get_titles(spec)
-
- for d in ["implemented", "in-progress"]:
- spec_dir = "%s/%s" % (self.specs_path, d)
-
- self.assertTrue(os.path.isdir(spec_dir),
- "%s is not a directory" % spec_dir)
- for filename in glob.glob(spec_dir + "/*"):
- if filename.endswith("README.rst"):
- continue
-
- self.assertTrue(
- filename.endswith(".rst"),
- "spec's file must have .rst ext. Found: %s" % filename)
- with open(filename) as f:
- data = f.read()
-
- titles = self._get_titles(docutils.core.publish_doctree(data))
- self._check_titles(filename, template_titles, titles)
- self._check_lines_wrapping(filename, data)
- self._check_no_cr(filename, data)
- self._check_trailing_spaces(filename, data)
diff --git a/osprofiler/tests/functional/__init__.py b/osprofiler/tests/functional/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/osprofiler/tests/functional/config.cfg b/osprofiler/tests/functional/config.cfg
new file mode 100644
index 0000000..d1d1e49
--- /dev/null
+++ b/osprofiler/tests/functional/config.cfg
@@ -0,0 +1,3 @@
+
+[profiler]
+connection_string="messaging://"
diff --git a/osprofiler/tests/functional/test_driver.py b/osprofiler/tests/functional/test_driver.py
new file mode 100644
index 0000000..6a1b020
--- /dev/null
+++ b/osprofiler/tests/functional/test_driver.py
@@ -0,0 +1,110 @@
+# Copyright (c) 2016 VMware, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from oslo_config import cfg
+
+from osprofiler.drivers import base
+from osprofiler import initializer
+from osprofiler import opts
+from osprofiler import profiler
+from osprofiler.tests import test
+
+
+CONF = cfg.CONF
+
+
+class DriverTestCase(test.TestCase):
+
+ SERVICE = "service"
+ PROJECT = "project"
+
+ def setUp(self):
+ super(DriverTestCase, self).setUp()
+ CONF(["--config-file", os.path.dirname(__file__) + "/config.cfg"])
+ opts.set_defaults(CONF,
+ enabled=True,
+ trace_sqlalchemy=False,
+ hmac_keys="SECRET_KEY")
+
+ @profiler.trace_cls("rpc", hide_args=True)
+ class Foo(object):
+
+ def bar(self, x):
+ return self.baz(x, x)
+
+ def baz(self, x, y):
+ return x * y
+
+ def _assert_dict(self, info, **kwargs):
+ for key in kwargs:
+ self.assertEqual(kwargs[key], info[key])
+
+ def _assert_child_dict(self, child, base_id, parent_id, name, fn_name):
+ self.assertEqual(parent_id, child["parent_id"])
+
+ exp_info = {"name": "rpc",
+ "service": self.SERVICE,
+ "project": self.PROJECT}
+ self._assert_dict(child["info"], **exp_info)
+
+ exp_raw_info = {"project": self.PROJECT,
+ "service": self.SERVICE}
+ raw_start = child["info"]["meta.raw_payload.%s-start" % name]
+ self._assert_dict(raw_start["info"], **exp_raw_info)
+ self.assertEqual(fn_name, raw_start["info"]["function"]["name"])
+ exp_raw = {"name": "%s-start" % name,
+ "service": self.SERVICE,
+ "trace_id": child["trace_id"],
+ "project": self.PROJECT,
+ "base_id": base_id}
+ self._assert_dict(raw_start, **exp_raw)
+
+ raw_stop = child["info"]["meta.raw_payload.%s-stop" % name]
+ self._assert_dict(raw_stop["info"], **exp_raw_info)
+ exp_raw["name"] = "%s-stop" % name
+ self._assert_dict(raw_stop, **exp_raw)
+
+ def test_get_report(self):
+ initializer.init_from_conf(
+ CONF, None, self.PROJECT, self.SERVICE, "host")
+ profiler.init("SECRET_KEY", project=self.PROJECT, service=self.SERVICE)
+
+ foo = DriverTestCase.Foo()
+ foo.bar(1)
+
+ engine = base.get_driver(CONF.profiler.connection_string,
+ project=self.PROJECT,
+ service=self.SERVICE,
+ host="host",
+ conf=CONF)
+ base_id = profiler.get().get_base_id()
+ res = engine.get_report(base_id)
+
+ self.assertEqual("total", res["info"]["name"])
+ self.assertEqual(2, res["stats"]["rpc"]["count"])
+ self.assertEqual(1, len(res["children"]))
+
+ cbar = res["children"][0]
+ self._assert_child_dict(
+ cbar, base_id, base_id, "rpc",
+ "osprofiler.tests.functional.test_driver.Foo.bar")
+
+ self.assertEqual(1, len(cbar["children"]))
+ cbaz = cbar["children"][0]
+ self._assert_child_dict(
+ cbaz, base_id, cbar["trace_id"], "rpc",
+ "osprofiler.tests.functional.test_driver.Foo.baz")
diff --git a/osprofiler/tests/hacking/__init__.py b/osprofiler/tests/hacking/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/osprofiler/tests/hacking/checks.py b/osprofiler/tests/hacking/checks.py
deleted file mode 100644
index 5b6e45d..0000000
--- a/osprofiler/tests/hacking/checks.py
+++ /dev/null
@@ -1,378 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""
-Guidelines for writing new hacking checks
-
- - Use only for OSProfiler specific tests. OpenStack general tests
- should be submitted to the common 'hacking' module.
- - Pick numbers in the range N3xx. Find the current test with
- the highest allocated number and then pick the next value.
- - Keep the test method code in the source file ordered based
- on the N3xx value.
- - List the new rule in the top level HACKING.rst file
- - Add test cases for each new rule to tests/unit/test_hacking.py
-
-"""
-
-import functools
-import re
-import tokenize
-
-re_assert_true_instance = re.compile(
- r"(.)*assertTrue\(isinstance\((\w|\.|\'|\"|\[|\])+, "
- r"(\w|\.|\'|\"|\[|\])+\)\)")
-re_assert_equal_type = re.compile(
- r"(.)*assertEqual\(type\((\w|\.|\'|\"|\[|\])+\), "
- r"(\w|\.|\'|\"|\[|\])+\)")
-re_assert_equal_end_with_none = re.compile(r"assertEqual\(.*?,\s+None\)$")
-re_assert_equal_start_with_none = re.compile(r"assertEqual\(None,")
-re_assert_true_false_with_in_or_not_in = re.compile(
- r"assert(True|False)\("
- r"(\w|[][.'\"])+( not)? in (\w|[][.'\",])+(, .*)?\)")
-re_assert_true_false_with_in_or_not_in_spaces = re.compile(
- r"assert(True|False)\((\w|[][.'\"])+( not)? in [\[|'|\"](\w|[][.'\", ])+"
- r"[\[|'|\"](, .*)?\)")
-re_assert_equal_in_end_with_true_or_false = re.compile(
- r"assertEqual\((\w|[][.'\"])+( not)? in (\w|[][.'\", ])+, (True|False)\)")
-re_assert_equal_in_start_with_true_or_false = re.compile(
- r"assertEqual\((True|False), (\w|[][.'\"])+( not)? in (\w|[][.'\", ])+\)")
-re_no_construct_dict = re.compile(
- r"\sdict\(\)")
-re_no_construct_list = re.compile(
- r"\slist\(\)")
-re_str_format = re.compile(r"""
-% # start of specifier
-\(([^)]+)\) # mapping key, in group 1
-[#0 +\-]? # optional conversion flag
-(?:-?\d*)? # optional minimum field width
-(?:\.\d*)? # optional precision
-[hLl]? # optional length modifier
-[A-z%] # conversion modifier
-""", re.X)
-re_raises = re.compile(
- r"\s:raise[^s] *.*$|\s:raises *:.*$|\s:raises *[^:]+$")
-
-
-def skip_ignored_lines(func):
-
- @functools.wraps(func)
- def wrapper(logical_line, filename):
- line = logical_line.strip()
- if not line or line.startswith("#") or line.endswith("# noqa"):
- return
- yield next(func(logical_line, filename))
-
- return wrapper
-
-
-def _parse_assert_mock_str(line):
- point = line.find(".assert_")
-
- if point != -1:
- end_pos = line[point:].find("(") + point
- return point, line[point + 1: end_pos], line[: point]
- else:
- return None, None, None
-
-
-@skip_ignored_lines
-def check_assert_methods_from_mock(logical_line, filename):
- """Ensure that ``assert_*`` methods from ``mock`` library is used correctly
-
- N301 - base error number
- N302 - related to nonexistent "assert_called"
- N303 - related to nonexistent "assert_called_once"
- """
-
- correct_names = ["assert_any_call", "assert_called_once_with",
- "assert_called_with", "assert_has_calls"]
- ignored_files = ["./tests/unit/test_hacking.py"]
-
- if filename.startswith("./tests") and filename not in ignored_files:
- pos, method_name, obj_name = _parse_assert_mock_str(logical_line)
-
- if pos:
- if method_name not in correct_names:
- error_number = "N301"
- msg = ("%(error_number)s:'%(method)s' is not present in `mock`"
- " library. %(custom_msg)s For more details, visit "
- "http://www.voidspace.org.uk/python/mock/ .")
-
- if method_name == "assert_called":
- error_number = "N302"
- custom_msg = ("Maybe, you should try to use "
- "'assertTrue(%s.called)' instead." %
- obj_name)
- elif method_name == "assert_called_once":
- # For more details, see a bug in Rally:
- # https://bugs.launchpad.net/rally/+bug/1305991
- error_number = "N303"
- custom_msg = ("Maybe, you should try to use "
- "'assertEqual(1, %s.call_count)' "
- "or '%s.assert_called_once_with()'"
- " instead." % (obj_name, obj_name))
- else:
- custom_msg = ("Correct 'assert_*' methods: '%s'."
- % "', '".join(correct_names))
-
- yield (pos, msg % {
- "error_number": error_number,
- "method": method_name,
- "custom_msg": custom_msg})
-
-
-@skip_ignored_lines
-def assert_true_instance(logical_line, filename):
- """Check for assertTrue(isinstance(a, b)) sentences
-
- N320
- """
- if re_assert_true_instance.match(logical_line):
- yield (0, "N320 assertTrue(isinstance(a, b)) sentences not allowed, "
- "you should use assertIsInstance(a, b) instead.")
-
-
-@skip_ignored_lines
-def assert_equal_type(logical_line, filename):
- """Check for assertEqual(type(A), B) sentences
-
- N321
- """
- if re_assert_equal_type.match(logical_line):
- yield (0, "N321 assertEqual(type(A), B) sentences not allowed, "
- "you should use assertIsInstance(a, b) instead.")
-
-
-@skip_ignored_lines
-def assert_equal_none(logical_line, filename):
- """Check for assertEqual(A, None) or assertEqual(None, A) sentences
-
- N322
- """
- res = (re_assert_equal_start_with_none.search(logical_line) or
- re_assert_equal_end_with_none.search(logical_line))
- if res:
- yield (0, "N322 assertEqual(A, None) or assertEqual(None, A) "
- "sentences not allowed, you should use assertIsNone(A) "
- "instead.")
-
-
-@skip_ignored_lines
-def assert_true_or_false_with_in(logical_line, filename):
- """Check assertTrue/False(A in/not in B) with collection contents
-
- Check for assertTrue/False(A in B), assertTrue/False(A not in B),
- assertTrue/False(A in B, message) or assertTrue/False(A not in B, message)
- sentences.
-
- N323
- """
- res = (re_assert_true_false_with_in_or_not_in.search(logical_line) or
- re_assert_true_false_with_in_or_not_in_spaces.search(logical_line))
- if res:
- yield (0, "N323 assertTrue/assertFalse(A in/not in B)sentences not "
- "allowed, you should use assertIn(A, B) or assertNotIn(A, B)"
- " instead.")
-
-
-@skip_ignored_lines
-def assert_equal_in(logical_line, filename):
- """Check assertEqual(A in/not in B, True/False) with collection contents
-
- Check for assertEqual(A in B, True/False), assertEqual(True/False, A in B),
- assertEqual(A not in B, True/False) or assertEqual(True/False, A not in B)
- sentences.
-
- N324
- """
- res = (re_assert_equal_in_end_with_true_or_false.search(logical_line) or
- re_assert_equal_in_start_with_true_or_false.search(logical_line))
- if res:
- yield (0, "N324: Use assertIn/NotIn(A, B) rather than "
- "assertEqual(A in/not in B, True/False) when checking "
- "collection contents.")
-
-
-@skip_ignored_lines
-def check_quotes(logical_line, filename):
- """Check that single quotation marks are not used
-
- N350
- """
-
- in_string = False
- in_multiline_string = False
- single_quotas_are_used = False
-
- check_tripple = (
- lambda line, i, char: (
- i + 2 < len(line) and
- (char == line[i] == line[i + 1] == line[i + 2])
- )
- )
-
- i = 0
- while i < len(logical_line):
- char = logical_line[i]
-
- if in_string:
- if char == "\"":
- in_string = False
- if char == "\\":
- i += 1 # ignore next char
-
- elif in_multiline_string:
- if check_tripple(logical_line, i, "\""):
- i += 2 # skip next 2 chars
- in_multiline_string = False
-
- elif char == "#":
- break
-
- elif char == "'":
- single_quotas_are_used = True
- break
-
- elif char == "\"":
- if check_tripple(logical_line, i, "\""):
- in_multiline_string = True
- i += 3
- continue
- in_string = True
-
- i += 1
-
- if single_quotas_are_used:
- yield (i, "N350 Remove Single quotes")
-
-
-@skip_ignored_lines
-def check_no_constructor_data_struct(logical_line, filename):
- """Check that data structs (lists, dicts) are declared using literals
-
- N351
- """
-
- match = re_no_construct_dict.search(logical_line)
- if match:
- yield (0, "N351 Remove dict() construct and use literal {}")
- match = re_no_construct_list.search(logical_line)
- if match:
- yield (0, "N351 Remove list() construct and use literal []")
-
-
-def check_dict_formatting_in_string(logical_line, tokens):
- """Check that strings do not use dict-formatting with a single replacement
-
- N352
- """
- # NOTE(stpierre): Can't use @skip_ignored_lines here because it's
- # a stupid decorator that only works on functions that take
- # (logical_line, filename) as arguments.
- if (not logical_line or
- logical_line.startswith("#") or
- logical_line.endswith("# noqa")):
- return
-
- current_string = ""
- in_string = False
- for token_type, text, start, end, line in tokens:
- if token_type == tokenize.STRING:
- if not in_string:
- current_string = ""
- in_string = True
- current_string += text.strip("\"")
- elif token_type == tokenize.OP:
- if not current_string:
- continue
- # NOTE(stpierre): The string formatting operator % has
- # lower precedence than +, so we assume that the logical
- # string has concluded whenever we hit an operator of any
- # sort. (Most operators don't work for strings anyway.)
- # Some string operators do have higher precedence than %,
- # though, so you can technically trick this check by doing
- # things like:
- #
- # "%(foo)s" * 1 % {"foo": 1}
- # "%(foo)s"[:] % {"foo": 1}
- #
- # It also will produce false positives if you use explicit
- # parenthesized addition for two strings instead of
- # concatenation by juxtaposition, e.g.:
- #
- # ("%(foo)s" + "%(bar)s") % vals
- #
- # But if you do any of those things, then you deserve all
- # of the horrible things that happen to you, and probably
- # many more.
- in_string = False
- if text == "%":
- format_keys = set()
- for match in re_str_format.finditer(current_string):
- format_keys.add(match.group(1))
- if len(format_keys) == 1:
- yield (0,
- "N353 Do not use mapping key string formatting "
- "with a single key")
- if text != ")":
- # NOTE(stpierre): You can have a parenthesized string
- # followed by %, so a closing paren doesn't obviate
- # the possibility for a substitution operator like
- # every other operator does.
- current_string = ""
- elif token_type in (tokenize.NL, tokenize.COMMENT):
- continue
- else:
- in_string = False
- if token_type == tokenize.NEWLINE:
- current_string = ""
-
-
-@skip_ignored_lines
-def check_using_unicode(logical_line, filename):
- """Check crosspython unicode usage
-
- N353
- """
-
- if re.search(r"\bunicode\(", logical_line):
- yield (0, "N353 'unicode' function is absent in python3. Please "
- "use 'six.text_type' instead.")
-
-
-def check_raises(physical_line, filename):
- """Check raises usage
-
- N354
- """
-
- ignored_files = ["./tests/unit/test_hacking.py",
- "./tests/hacking/checks.py"]
- if filename not in ignored_files:
- if re_raises.search(physical_line):
- return (0, "N354 ':Please use ':raises Exception: conditions' "
- "in docstrings.")
-
-
-def factory(register):
- register(check_assert_methods_from_mock)
- register(assert_true_instance)
- register(assert_equal_type)
- register(assert_equal_none)
- register(assert_true_or_false_with_in)
- register(assert_equal_in)
- register(check_quotes)
- register(check_no_constructor_data_struct)
- register(check_dict_formatting_in_string)
- register(check_using_unicode)
- register(check_raises)
diff --git a/osprofiler/tests/notifiers/__init__.py b/osprofiler/tests/notifiers/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/osprofiler/tests/notifiers/test_base.py b/osprofiler/tests/notifiers/test_base.py
deleted file mode 100644
index 6cd1a9e..0000000
--- a/osprofiler/tests/notifiers/test_base.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import mock
-
-from osprofiler._notifiers import base
-from osprofiler.tests import test
-
-
-class NotifierBaseTestCase(test.TestCase):
-
- def test_factory(self):
-
- class A(base.Notifier):
-
- def notify(self, a):
- return a
-
- self.assertEqual(base.Notifier.factory("A")(10), 10)
-
- def test_factory_with_args(self):
-
- class B(base.Notifier):
-
- def __init__(self, a, b=10):
- self.a = a
- self.b = b
-
- def notify(self, c):
- return self.a + self.b + c
-
- self.assertEqual(base.Notifier.factory("B", 5, b=7)(10), 22)
-
- def test_factory_not_found(self):
- self.assertRaises(TypeError, base.Notifier.factory, "non existing")
-
- def test_notify(self):
- base.Notifier().notify("")
-
- def test_plugins_are_imported(self):
- base.Notifier.factory("Messaging", mock.MagicMock(), "context",
- "transport", "project", "service", "host")
diff --git a/osprofiler/tests/notifiers/test_messaging.py b/osprofiler/tests/notifiers/test_messaging.py
deleted file mode 100644
index 46e11e7..0000000
--- a/osprofiler/tests/notifiers/test_messaging.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import mock
-
-from osprofiler._notifiers import base
-from osprofiler.tests import test
-
-
-class MessagingTestCase(test.TestCase):
-
- def test_init_and_notify(self):
-
- messaging = mock.MagicMock()
- context = "context"
- transport = "transport"
- project = "project"
- service = "service"
- host = "host"
-
- notify_func = base.Notifier.factory("Messaging", messaging, context,
- transport, project, service, host)
-
- messaging.Notifier.assert_called_once_with(
- transport, publisher_id=host, driver="messaging",
- topic="profiler", retry=0)
-
- info = {
- "a": 10
- }
- notify_func(info)
-
- expected_data = {"project": project, "service": service}
- expected_data.update(info)
- messaging.Notifier().info.assert_called_once_with(
- context, "profiler.%s" % service, expected_data)
-
- messaging.reset_mock()
- notify_func(info, context="my_context")
- messaging.Notifier().info.assert_called_once_with(
- "my_context", "profiler.%s" % service, expected_data)
diff --git a/osprofiler/tests/parsers/__init__.py b/osprofiler/tests/parsers/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/osprofiler/tests/parsers/test_ceilometer.py b/osprofiler/tests/parsers/test_ceilometer.py
deleted file mode 100644
index 103f244..0000000
--- a/osprofiler/tests/parsers/test_ceilometer.py
+++ /dev/null
@@ -1,402 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import mock
-
-from osprofiler.parsers import ceilometer
-from osprofiler.tests import test
-
-
-class CeilometerParserTestCase(test.TestCase):
- def test_build_empty_tree(self):
- self.assertEqual(ceilometer._build_tree({}), [])
-
- def test_build_complex_tree(self):
- test_input = {
- "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}},
- "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}},
- "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}},
- "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}},
- "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}},
- "113": {"parent_id": "11", "trace_id": "113",
- "info": {"started": 3}},
- "112": {"parent_id": "11", "trace_id": "112",
- "info": {"started": 2}},
- "114": {"parent_id": "11", "trace_id": "114",
- "info": {"started": 5}}
- }
-
- expected_output = [
- {
- "parent_id": "0",
- "trace_id": "1",
- "info": {"started": 0},
- "children": [
- {
- "parent_id": "1",
- "trace_id": "11",
- "info": {"started": 1},
- "children": [
- {"parent_id": "11", "trace_id": "112",
- "info": {"started": 2}, "children": []},
- {"parent_id": "11", "trace_id": "113",
- "info": {"started": 3}, "children": []},
- {"parent_id": "11", "trace_id": "114",
- "info": {"started": 5}, "children": []}
- ]
- }
- ]
- },
- {
- "parent_id": "0",
- "trace_id": "2",
- "info": {"started": 1},
- "children": [
- {"parent_id": "2", "trace_id": "21",
- "info": {"started": 6}, "children": []},
- {"parent_id": "2", "trace_id": "22",
- "info": {"started": 7}, "children": []}
- ]
- }
- ]
-
- self.assertEqual(ceilometer._build_tree(test_input), expected_output)
-
- def test_parse_notifications_empty(self):
- expected = {
- "info": {
- "name": "total",
- "started": 0,
- "finished": 0
- },
- "children": []
- }
- self.assertEqual(ceilometer.parse_notifications([]), expected)
-
- def test_parse_notifications(self):
- events = [
- {
- "traits": [
- {
- "type": "string",
- "name": "base_id",
- "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
- },
- {
- "type": "string",
- "name": "host",
- "value": "ubuntu"
- },
- {
- "type": "string",
- "name": "method",
- "value": "POST"
- },
- {
- "type": "string",
- "name": "name",
- "value": "wsgi-start"
- },
- {
- "type": "string",
- "name": "parent_id",
- "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
- },
- {
- "type": "string",
- "name": "project",
- "value": "keystone"
- },
- {
- "type": "string",
- "name": "service",
- "value": "main"
- },
- {
- "type": "string",
- "name": "timestamp",
- "value": "2015-12-23T14:02:22.338776"
- },
- {
- "type": "string",
- "name": "trace_id",
- "value": "06320327-2c2c-45ae-923a-515de890276a"
- }
- ],
- "raw": {},
- "generated": "2015-12-23T10:41:38.415793",
- "event_type": "profiler.main",
- "message_id": "65fc1553-3082-4a6f-9d1e-0e3183f57a47"},
- {
- "traits":
- [
- {
- "type": "string",
- "name": "base_id",
- "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
- },
- {
- "type": "string",
- "name": "host",
- "value": "ubuntu"
- },
- {
- "type": "string",
- "name": "name",
- "value": "wsgi-stop"
- },
- {
- "type": "string",
- "name": "parent_id",
- "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
- },
- {
- "type": "string",
- "name": "project",
- "value": "keystone"
- },
- {
- "type": "string",
- "name": "service",
- "value": "main"
- },
- {
- "type": "string",
- "name": "timestamp",
- "value": "2015-12-23T14:02:22.380405"
- },
- {
- "type": "string",
- "name": "trace_id",
- "value": "016c97fd-87f3-40b2-9b55-e431156b694b"
- }
- ],
- "raw": {},
- "generated": "2015-12-23T10:41:38.406052",
- "event_type": "profiler.main",
- "message_id": "3256d9f1-48ba-4ac5-a50b-64fa42c6e264"},
- {
- "traits":
- [
- {
- "type": "string",
- "name": "base_id",
- "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
- },
- {
- "type": "string",
- "name": "db.params",
- "value": "[]"
- },
- {
- "type": "string",
- "name": "db.statement",
- "value": "SELECT 1"
- },
- {
- "type": "string",
- "name": "host",
- "value": "ubuntu"
- },
- {
- "type": "string",
- "name": "name",
- "value": "db-start"
- },
- {
- "type": "string",
- "name": "parent_id",
- "value": "06320327-2c2c-45ae-923a-515de890276a"
- },
- {
- "type": "string",
- "name": "project",
- "value": "keystone"
- },
- {
- "type": "string",
- "name": "service",
- "value": "main"
- },
- {
- "type": "string",
- "name": "timestamp",
- "value": "2015-12-23T14:02:22.395365"
- },
- {
- "type": "string",
- "name": "trace_id",
- "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"
- }
- ],
- "raw": {},
- "generated": "2015-12-23T10:41:38.984161",
- "event_type": "profiler.main",
- "message_id": "60368aa4-16f0-4f37-a8fb-89e92fdf36ff"
- },
- {
- "traits":
- [
- {
- "type": "string",
- "name": "base_id",
- "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
- },
- {
- "type": "string",
- "name": "host",
- "value": "ubuntu"
- },
- {
- "type": "string",
- "name": "name",
- "value": "db-stop"
- },
- {
- "type": "string",
- "name": "parent_id",
- "value": "06320327-2c2c-45ae-923a-515de890276a"
- },
- {
- "type": "string",
- "name": "project",
- "value": "keystone"
- },
- {
- "type": "string",
- "name": "service",
- "value": "main"
- },
- {
- "type": "string",
- "name": "timestamp",
- "value": "2015-12-23T14:02:22.415486"
- },
- {
- "type": "string",
- "name": "trace_id",
- "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"
- }
- ],
- "raw": {},
- "generated": "2015-12-23T10:41:39.019378",
- "event_type": "profiler.main",
- "message_id": "3fbeb339-55c5-4f28-88e4-15bee251dd3d"
- },
- {
- "traits":
- [
- {
- "type": "string",
- "name": "base_id",
- "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
- },
- {
- "type": "string",
- "name": "host",
- "value": "ubuntu"
- },
- {
- "type": "string",
- "name": "method",
- "value": "GET"
- },
- {
- "type": "string",
- "name": "name",
- "value": "wsgi-start"
- },
- {
- "type": "string",
- "name": "parent_id",
- "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
- },
- {
- "type": "string",
- "name": "project",
- "value": "keystone"
- },
- {
- "type": "string",
- "name": "service",
- "value": "main"
- },
- {
- "type": "string",
- "name": "timestamp",
- "value": "2015-12-23T14:02:22.427444"
- },
- {
- "type": "string",
- "name": "trace_id",
- "value": "016c97fd-87f3-40b2-9b55-e431156b694b"
- }
- ],
- "raw": {},
- "generated": "2015-12-23T10:41:38.360409",
- "event_type": "profiler.main",
- "message_id": "57b971a9-572f-4f29-9838-3ed2564c6b5b"
- }
- ]
-
- expected = {"children": [
- {"children": [{"children": [],
- "info": {"finished": 76,
- "host": "ubuntu",
- "meta.raw_payload.db-start": {},
- "meta.raw_payload.db-stop": {},
- "name": "db",
- "project": "keystone",
- "service": "main",
- "started": 56},
- "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
- "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}
- ],
- "info": {"finished": 0,
- "host": "ubuntu",
- "meta.raw_payload.wsgi-start": {},
- "name": "wsgi",
- "project": "keystone",
- "service": "main",
- "started": 0},
- "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
- "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
- {"children": [],
- "info": {"finished": 41,
- "host": "ubuntu",
- "meta.raw_payload.wsgi-start": {},
- "meta.raw_payload.wsgi-stop": {},
- "name": "wsgi",
- "project": "keystone",
- "service": "main",
- "started": 88},
- "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
- "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}],
- "info": {"finished": 88, "name": "total", "started": 0}}
-
- self.assertEqual(expected, ceilometer.parse_notifications(events))
-
- def test_get_notifications(self):
- mock_ceil_client = mock.MagicMock()
- results = [mock.MagicMock(), mock.MagicMock()]
- mock_ceil_client.events.list.return_value = results
- base_id = "10"
-
- result = ceilometer.get_notifications(mock_ceil_client, base_id)
-
- expected_filter = [{"field": "base_id", "op": "eq", "value": base_id}]
- mock_ceil_client.events.list.assert_called_once_with(expected_filter,
- limit=100000)
- self.assertEqual(result, [results[0].to_dict(), results[1].to_dict()])
diff --git a/osprofiler/tests/test_notifier.py b/osprofiler/tests/test_notifier.py
deleted file mode 100644
index f3b9f64..0000000
--- a/osprofiler/tests/test_notifier.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import mock
-
-from osprofiler import notifier
-from osprofiler.tests import test
-
-
-class NotifierTestCase(test.TestCase):
-
- def tearDown(self):
- notifier.__notifier = notifier._noop_notifier
- super(NotifierTestCase, self).tearDown()
-
- def test_set(self):
-
- def test(info):
- pass
-
- notifier.set(test)
- self.assertEqual(notifier.get(), test)
-
- def test_get_default_notifier(self):
- self.assertEqual(notifier.get(), notifier._noop_notifier)
-
- def test_notify(self):
- m = mock.MagicMock()
- notifier.set(m)
- notifier.notify(10)
-
- m.assert_called_once_with(10)
-
- @mock.patch("osprofiler.notifier.base.Notifier.factory")
- def test_create(self, mock_factory):
-
- result = notifier.create("test", 10, b=20)
- mock_factory.assert_called_once_with("test", 10, b=20)
- self.assertEqual(mock_factory.return_value, result)
diff --git a/osprofiler/tests/test_opts.py b/osprofiler/tests/test_opts.py
deleted file mode 100644
index 1997b98..0000000
--- a/osprofiler/tests/test_opts.py
+++ /dev/null
@@ -1,64 +0,0 @@
-# Copyright 2016 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import mock
-from oslo_config import fixture
-from osprofiler import opts
-from osprofiler.tests import test
-
-
-class ConfigTestCase(test.TestCase):
- def setUp(self):
- super(ConfigTestCase, self).setUp()
- self.conf_fixture = self.useFixture(fixture.Config())
-
- def test_options_defaults(self):
- opts.set_defaults(self.conf_fixture.conf)
- self.assertFalse(self.conf_fixture.conf.profiler.enabled)
- self.assertFalse(self.conf_fixture.conf.profiler.trace_sqlalchemy)
- self.assertEqual("SECRET_KEY",
- self.conf_fixture.conf.profiler.hmac_keys)
- self.assertFalse(opts.is_trace_enabled(self.conf_fixture.conf))
- self.assertFalse(opts.is_db_trace_enabled(self.conf_fixture.conf))
-
- def test_options_defaults_override(self):
- opts.set_defaults(self.conf_fixture.conf, enabled=True,
- trace_sqlalchemy=True,
- hmac_keys="MY_KEY")
- self.assertTrue(self.conf_fixture.conf.profiler.enabled)
- self.assertTrue(self.conf_fixture.conf.profiler.trace_sqlalchemy)
- self.assertEqual("MY_KEY",
- self.conf_fixture.conf.profiler.hmac_keys)
- self.assertTrue(opts.is_trace_enabled(self.conf_fixture.conf))
- self.assertTrue(opts.is_db_trace_enabled(self.conf_fixture.conf))
-
- @mock.patch("osprofiler.web.enable")
- @mock.patch("osprofiler.web.disable")
- def test_web_trace_disabled(self, mock_disable, mock_enable):
- opts.set_defaults(self.conf_fixture.conf, hmac_keys="MY_KEY")
- opts.enable_web_trace(self.conf_fixture.conf)
- opts.disable_web_trace(self.conf_fixture.conf)
- self.assertEqual(0, mock_enable.call_count)
- self.assertEqual(0, mock_disable.call_count)
-
- @mock.patch("osprofiler.web.enable")
- @mock.patch("osprofiler.web.disable")
- def test_web_trace_enabled(self, mock_disable, mock_enable):
- opts.set_defaults(self.conf_fixture.conf, enabled=True,
- hmac_keys="MY_KEY")
- opts.enable_web_trace(self.conf_fixture.conf)
- opts.disable_web_trace(self.conf_fixture.conf)
- mock_enable.assert_called_once_with("MY_KEY")
- mock_disable.assert_called_once_with()
diff --git a/osprofiler/tests/test_profiler.py b/osprofiler/tests/test_profiler.py
deleted file mode 100644
index d5a2b8a..0000000
--- a/osprofiler/tests/test_profiler.py
+++ /dev/null
@@ -1,499 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import collections
-import copy
-import datetime
-import mock
-import re
-
-import six
-
-from osprofiler import profiler
-from osprofiler.tests import test
-
-
-class ProfilerGlobMethodsTestCase(test.TestCase):
-
- def test_get_profiler_not_inited(self):
- profiler._clean()
- self.assertIsNone(profiler.get())
-
- def test_get_profiler_and_init(self):
- p = profiler.init("secret", base_id="1", parent_id="2")
- self.assertEqual(profiler.get(), p)
-
- self.assertEqual(p.get_base_id(), "1")
- # NOTE(boris-42): until we make first start we don't have
- self.assertEqual(p.get_id(), "2")
-
- def test_start_not_inited(self):
- profiler._clean()
- profiler.start("name")
-
- def test_start(self):
- p = profiler.init("secret", base_id="1", parent_id="2")
- p.start = mock.MagicMock()
- profiler.start("name", info="info")
- p.start.assert_called_once_with("name", info="info")
-
- def test_stop_not_inited(self):
- profiler._clean()
- profiler.stop()
-
- def test_stop(self):
- p = profiler.init("secret", base_id="1", parent_id="2")
- p.stop = mock.MagicMock()
- profiler.stop(info="info")
- p.stop.assert_called_once_with(info="info")
-
-
-class ProfilerTestCase(test.TestCase):
-
- def test_profiler_get_base_id(self):
- prof = profiler._Profiler("secret", base_id="1", parent_id="2")
- self.assertEqual(prof.get_base_id(), "1")
-
- @mock.patch("osprofiler.profiler.uuid.uuid4")
- def test_profiler_get_parent_id(self, mock_uuid4):
- mock_uuid4.return_value = "42"
- prof = profiler._Profiler("secret", base_id="1", parent_id="2")
- prof.start("test")
- self.assertEqual(prof.get_parent_id(), "2")
-
- @mock.patch("osprofiler.profiler.uuid.uuid4")
- def test_profiler_get_base_id_unset_case(self, mock_uuid4):
- mock_uuid4.return_value = "42"
- prof = profiler._Profiler("secret")
- self.assertEqual(prof.get_base_id(), "42")
- self.assertEqual(prof.get_parent_id(), "42")
-
- @mock.patch("osprofiler.profiler.uuid.uuid4")
- def test_profiler_get_id(self, mock_uuid4):
- mock_uuid4.return_value = "43"
- prof = profiler._Profiler("secret")
- prof.start("test")
- self.assertEqual(prof.get_id(), "43")
-
- @mock.patch("osprofiler.profiler.datetime")
- @mock.patch("osprofiler.profiler.uuid.uuid4")
- @mock.patch("osprofiler.profiler.notifier.notify")
- def test_profiler_start(self, mock_notify, mock_uuid4, mock_datetime):
- mock_uuid4.return_value = "44"
- now = datetime.datetime.utcnow()
- mock_datetime.datetime.utcnow.return_value = now
-
- info = {"some": "info"}
- payload = {
- "name": "test-start",
- "base_id": "1",
- "parent_id": "2",
- "trace_id": "44",
- "info": info,
- "timestamp": now.strftime("%Y-%m-%dT%H:%M:%S.%f"),
- }
-
- prof = profiler._Profiler("secret", base_id="1", parent_id="2")
- prof.start("test", info=info)
-
- mock_notify.assert_called_once_with(payload)
-
- @mock.patch("osprofiler.profiler.datetime")
- @mock.patch("osprofiler.profiler.notifier.notify")
- def test_profiler_stop(self, mock_notify, mock_datetime):
- now = datetime.datetime.utcnow()
- mock_datetime.datetime.utcnow.return_value = now
- prof = profiler._Profiler("secret", base_id="1", parent_id="2")
- prof._trace_stack.append("44")
- prof._name.append("abc")
-
- info = {"some": "info"}
- prof.stop(info=info)
-
- payload = {
- "name": "abc-stop",
- "base_id": "1",
- "parent_id": "2",
- "trace_id": "44",
- "info": info,
- "timestamp": now.strftime("%Y-%m-%dT%H:%M:%S.%f"),
- }
-
- mock_notify.assert_called_once_with(payload)
- self.assertEqual(len(prof._name), 0)
- self.assertEqual(prof._trace_stack, collections.deque(["1", "2"]))
-
- def test_profiler_hmac(self):
- hmac = "secret"
- prof = profiler._Profiler(hmac, base_id="1", parent_id="2")
- self.assertEqual(hmac, prof.hmac_key)
-
-
-class WithTraceTestCase(test.TestCase):
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_with_trace(self, mock_start, mock_stop):
-
- with profiler.Trace("a", info="a1"):
- mock_start.assert_called_once_with("a", info="a1")
- mock_start.reset_mock()
- with profiler.Trace("b", info="b1"):
- mock_start.assert_called_once_with("b", info="b1")
- mock_stop.assert_called_once_with()
- mock_stop.reset_mock()
- mock_stop.assert_called_once_with()
-
-
-@profiler.trace("function", info={"info": "some_info"})
-def tracede_func(i):
- return i
-
-
-@profiler.trace("hide_args", hide_args=True)
-def trace_hide_args_func(a, i=10):
- return (a, i)
-
-
-class TraceDecoratorTestCase(test.TestCase):
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_with_args(self, mock_start, mock_stop):
- self.assertEqual(1, tracede_func(1))
- expected_info = {
- "info": "some_info",
- "function": {
- "name": "osprofiler.tests.test_profiler.tracede_func",
- "args": str((1,)),
- "kwargs": str({})
- }
- }
- mock_start.assert_called_once_with("function", info=expected_info)
- mock_stop.assert_called_once_with()
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_without_args(self, mock_start, mock_stop):
- self.assertEqual((1, 2), trace_hide_args_func(1, i=2))
- expected_info = {
- "function": {
- "name": "osprofiler.tests.test_profiler.trace_hide_args_func"
- }
- }
- mock_start.assert_called_once_with("hide_args", info=expected_info)
- mock_stop.assert_called_once_with()
-
-
-class FakeTracedCls(object):
-
- def method1(self, a, b, c=10):
- return a + b + c
-
- def method2(self, d, e):
- return d - e
-
- def method3(self, g=10, h=20):
- return g * h
-
- def _method(self, i):
- return i
-
-
-@profiler.trace_cls("rpc", info={"a": 10})
-class FakeTraceClassWithInfo(FakeTracedCls):
- pass
-
-
-@profiler.trace_cls("a", info={"b": 20}, hide_args=True)
-class FakeTraceClassHideArgs(FakeTracedCls):
- pass
-
-
-@profiler.trace_cls("rpc", trace_private=True)
-class FakeTracePrivate(FakeTracedCls):
- pass
-
-
-@profiler.trace_cls("rpc")
-class FakeTraceStatic(FakeTracedCls):
- @staticmethod
- def method4(arg):
- return arg
-
-
-def py3_info(info):
- # NOTE(boris-42): py33 I hate you.
- info_py3 = copy.deepcopy(info)
- new_name = re.sub("FakeTrace[^.]*", "FakeTracedCls",
- info_py3["function"]["name"])
- info_py3["function"]["name"] = new_name
- return info_py3
-
-
-def possible_mock_calls(name, info):
- # NOTE(boris-42): py33 I hate you.
- return [mock.call(name, info=info), mock.call(name, info=py3_info(info))]
-
-
-class TraceClsDecoratorTestCase(test.TestCase):
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_args(self, mock_start, mock_stop):
- fake_cls = FakeTraceClassWithInfo()
- self.assertEqual(30, fake_cls.method1(5, 15))
- expected_info = {
- "a": 10,
- "function": {
- "name": ("osprofiler.tests.test_profiler"
- ".FakeTraceClassWithInfo.method1"),
- "args": str((fake_cls, 5, 15)),
- "kwargs": str({})
- }
- }
- self.assertEqual(1, len(mock_start.call_args_list))
- self.assertIn(mock_start.call_args_list[0],
- possible_mock_calls("rpc", expected_info))
- mock_stop.assert_called_once_with()
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_kwargs(self, mock_start, mock_stop):
- fake_cls = FakeTraceClassWithInfo()
- self.assertEqual(50, fake_cls.method3(g=5, h=10))
- expected_info = {
- "a": 10,
- "function": {
- "name": ("osprofiler.tests.test_profiler"
- ".FakeTraceClassWithInfo.method3"),
- "args": str((fake_cls,)),
- "kwargs": str({"g": 5, "h": 10})
- }
- }
- self.assertEqual(1, len(mock_start.call_args_list))
- self.assertIn(mock_start.call_args_list[0],
- possible_mock_calls("rpc", expected_info))
- mock_stop.assert_called_once_with()
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_without_private(self, mock_start, mock_stop):
- fake_cls = FakeTraceClassHideArgs()
- self.assertEqual(10, fake_cls._method(10))
- self.assertFalse(mock_start.called)
- self.assertFalse(mock_stop.called)
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_without_args(self, mock_start, mock_stop):
- fake_cls = FakeTraceClassHideArgs()
- self.assertEqual(40, fake_cls.method1(5, 15, c=20))
- expected_info = {
- "b": 20,
- "function": {
- "name": ("osprofiler.tests.test_profiler"
- ".FakeTraceClassHideArgs.method1"),
- }
- }
-
- self.assertEqual(1, len(mock_start.call_args_list))
- self.assertIn(mock_start.call_args_list[0],
- possible_mock_calls("a", expected_info))
- mock_stop.assert_called_once_with()
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_private_methods(self, mock_start, mock_stop):
- fake_cls = FakeTracePrivate()
- self.assertEqual(5, fake_cls._method(5))
-
- expected_info = {
- "function": {
- "name": ("osprofiler.tests.test_profiler"
- ".FakeTracePrivate._method"),
- "args": str((fake_cls, 5)),
- "kwargs": str({})
- }
- }
-
- self.assertEqual(1, len(mock_start.call_args_list))
- self.assertIn(mock_start.call_args_list[0],
- possible_mock_calls("rpc", expected_info))
- mock_stop.assert_called_once_with()
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- @test.testcase.skip(
- "Static method tracing was disabled due the bug. This test should be "
- "skipped until we find the way to address it.")
- def test_static(self, mock_start, mock_stop):
- fake_cls = FakeTraceStatic()
-
- self.assertEqual(25, fake_cls.method4(25))
-
- expected_info = {
- "function": {
- # fixme(boris-42): Static methods are treated differently in
- # Python 2.x and Python 3.x. So in PY2 we
- # expect to see method4 because method is
- # static and doesn't have reference to class
- # - and FakeTraceStatic.method4 in PY3
- "name":
- "osprofiler.tests.test_profiler.method4" if six.PY2 else
- "osprofiler.tests.test_profiler.FakeTraceStatic.method4",
- "args": str((25,)),
- "kwargs": str({})
- }
- }
-
- self.assertEqual(1, len(mock_start.call_args_list))
- self.assertIn(mock_start.call_args_list[0],
- possible_mock_calls("rpc", expected_info))
- mock_stop.assert_called_once_with()
-
-
-@six.add_metaclass(profiler.TracedMeta)
-class FakeTraceWithMetaclassBase(object):
- __trace_args__ = {"name": "rpc",
- "info": {"a": 10}}
-
- def method1(self, a, b, c=10):
- return a + b + c
-
- def method2(self, d, e):
- return d - e
-
- def method3(self, g=10, h=20):
- return g * h
-
- def _method(self, i):
- return i
-
-
-class FakeTraceDummy(FakeTraceWithMetaclassBase):
- def method4(self, j):
- return j
-
-
-class FakeTraceWithMetaclassHideArgs(FakeTraceWithMetaclassBase):
- __trace_args__ = {"name": "a",
- "info": {"b": 20},
- "hide_args": True}
-
- def method5(self, k, l):
- return k + l
-
-
-class FakeTraceWithMetaclassPrivate(FakeTraceWithMetaclassBase):
- __trace_args__ = {"name": "rpc",
- "trace_private": True}
-
- def _new_private_method(self, m):
- return 2 * m
-
-
-class TraceWithMetaclassTestCase(test.TestCase):
-
- def test_no_name_exception(self):
- def define_class_with_no_name():
- @six.add_metaclass(profiler.TracedMeta)
- class FakeTraceWithMetaclassNoName(FakeTracedCls):
- pass
- self.assertRaises(TypeError, define_class_with_no_name, 1)
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_args(self, mock_start, mock_stop):
- fake_cls = FakeTraceWithMetaclassBase()
- self.assertEqual(30, fake_cls.method1(5, 15))
- expected_info = {
- "a": 10,
- "function": {
- "name": ("osprofiler.tests.test_profiler"
- ".FakeTraceWithMetaclassBase.method1"),
- "args": str((fake_cls, 5, 15)),
- "kwargs": str({})
- }
- }
- self.assertEqual(1, len(mock_start.call_args_list))
- self.assertIn(mock_start.call_args_list[0],
- possible_mock_calls("rpc", expected_info))
- mock_stop.assert_called_once_with()
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_kwargs(self, mock_start, mock_stop):
- fake_cls = FakeTraceWithMetaclassBase()
- self.assertEqual(50, fake_cls.method3(g=5, h=10))
- expected_info = {
- "a": 10,
- "function": {
- "name": ("osprofiler.tests.test_profiler"
- ".FakeTraceWithMetaclassBase.method3"),
- "args": str((fake_cls,)),
- "kwargs": str({"g": 5, "h": 10})
- }
- }
- self.assertEqual(1, len(mock_start.call_args_list))
- self.assertIn(mock_start.call_args_list[0],
- possible_mock_calls("rpc", expected_info))
- mock_stop.assert_called_once_with()
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_without_private(self, mock_start, mock_stop):
- fake_cls = FakeTraceWithMetaclassHideArgs()
- self.assertEqual(10, fake_cls._method(10))
- self.assertFalse(mock_start.called)
- self.assertFalse(mock_stop.called)
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_without_args(self, mock_start, mock_stop):
- fake_cls = FakeTraceWithMetaclassHideArgs()
- self.assertEqual(20, fake_cls.method5(5, 15))
- expected_info = {
- "b": 20,
- "function": {
- "name": ("osprofiler.tests.test_profiler"
- ".FakeTraceWithMetaclassHideArgs.method5")
- }
- }
-
- self.assertEqual(1, len(mock_start.call_args_list))
- self.assertIn(mock_start.call_args_list[0],
- possible_mock_calls("a", expected_info))
- mock_stop.assert_called_once_with()
-
- @mock.patch("osprofiler.profiler.stop")
- @mock.patch("osprofiler.profiler.start")
- def test_private_methods(self, mock_start, mock_stop):
- fake_cls = FakeTraceWithMetaclassPrivate()
- self.assertEqual(10, fake_cls._new_private_method(5))
-
- expected_info = {
- "function": {
- "name": ("osprofiler.tests.test_profiler"
- ".FakeTraceWithMetaclassPrivate._new_private_method"),
- "args": str((fake_cls, 5)),
- "kwargs": str({})
- }
- }
-
- self.assertEqual(1, len(mock_start.call_args_list))
- self.assertIn(mock_start.call_args_list[0],
- possible_mock_calls("rpc", expected_info))
- mock_stop.assert_called_once_with()
diff --git a/osprofiler/tests/test_sqlalchemy.py b/osprofiler/tests/test_sqlalchemy.py
deleted file mode 100644
index 1494da6..0000000
--- a/osprofiler/tests/test_sqlalchemy.py
+++ /dev/null
@@ -1,71 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import mock
-
-from osprofiler import sqlalchemy
-from osprofiler.tests import test
-
-
-class SqlalchemyTracingTestCase(test.TestCase):
-
- @mock.patch("osprofiler.sqlalchemy.profiler")
- def test_before_execute(self, mock_profiler):
- handler = sqlalchemy._before_cursor_execute("sql")
-
- handler(mock.MagicMock(), 1, 2, 3, 4, 5)
- expected_info = {"db": {"statement": 2, "params": 3}}
- mock_profiler.start.assert_called_once_with("sql", info=expected_info)
-
- @mock.patch("osprofiler.sqlalchemy.profiler")
- def test_after_execute(self, mock_profiler):
- handler = sqlalchemy._after_cursor_execute()
- handler(mock.MagicMock(), 1, 2, 3, 4, 5)
- mock_profiler.stop.assert_called_once_with()
-
- @mock.patch("osprofiler.sqlalchemy._before_cursor_execute")
- @mock.patch("osprofiler.sqlalchemy._after_cursor_execute")
- def test_add_tracing(self, mock_after_exc, mock_before_exc):
- sa = mock.MagicMock()
- engine = mock.MagicMock()
-
- mock_before_exc.return_value = "before"
- mock_after_exc.return_value = "after"
-
- sqlalchemy.add_tracing(sa, engine, "sql")
-
- mock_before_exc.assert_called_once_with("sql")
- mock_after_exc.assert_called_once_with()
- expected_calls = [
- mock.call(engine, "before_cursor_execute", "before"),
- mock.call(engine, "after_cursor_execute", "after")
- ]
- self.assertEqual(sa.event.listen.call_args_list, expected_calls)
-
- @mock.patch("osprofiler.sqlalchemy._before_cursor_execute")
- @mock.patch("osprofiler.sqlalchemy._after_cursor_execute")
- def test_disable_and_enable(self, mock_after_exc, mock_before_exc):
- sqlalchemy.disable()
-
- sa = mock.MagicMock()
- engine = mock.MagicMock()
- sqlalchemy.add_tracing(sa, engine, "sql")
- self.assertFalse(mock_after_exc.called)
- self.assertFalse(mock_before_exc.called)
-
- sqlalchemy.enable()
- sqlalchemy.add_tracing(sa, engine, "sql")
- self.assertTrue(mock_after_exc.called)
- self.assertTrue(mock_before_exc.called)
diff --git a/osprofiler/tests/test_utils.py b/osprofiler/tests/test_utils.py
deleted file mode 100644
index 19aff30..0000000
--- a/osprofiler/tests/test_utils.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import base64
-import hashlib
-import hmac
-
-import mock
-
-from osprofiler import _utils as utils
-from osprofiler.tests import test
-
-
-class UtilsTestCase(test.TestCase):
-
- def test_split(self):
- self.assertEqual([1, 2], utils.split([1, 2]))
- self.assertEqual(["A", "B"], utils.split("A, B"))
- self.assertEqual(["A", " B"], utils.split("A, B", strip=False))
-
- def test_split_wrong_type(self):
- self.assertRaises(TypeError, utils.split, 1)
-
- def test_binary_encode_and_decode(self):
- self.assertEqual("text",
- utils.binary_decode(utils.binary_encode("text")))
-
- def test_binary_encode_invalid_type(self):
- self.assertRaises(TypeError, utils.binary_encode, 1234)
-
- def test_binary_encode_binary_type(self):
- binary = utils.binary_encode("text")
- self.assertEqual(binary, utils.binary_encode(binary))
-
- def test_binary_decode_invalid_type(self):
- self.assertRaises(TypeError, utils.binary_decode, 1234)
-
- def test_binary_decode_text_type(self):
- self.assertEqual("text", utils.binary_decode("text"))
-
- def test_generate_hmac(self):
- hmac_key = "secrete"
- data = "my data"
-
- h = hmac.new(utils.binary_encode(hmac_key), digestmod=hashlib.sha1)
- h.update(utils.binary_encode(data))
-
- self.assertEqual(h.hexdigest(), utils.generate_hmac(data, hmac_key))
-
- def test_signed_pack_unpack(self):
- hmac = "secret"
- data = {"some": "data"}
-
- packed_data, hmac_data = utils.signed_pack(data, hmac)
-
- process_data = utils.signed_unpack(packed_data, hmac_data, [hmac])
- self.assertIn("hmac_key", process_data)
- process_data.pop("hmac_key")
- self.assertEqual(data, process_data)
-
- def test_signed_pack_unpack_many_keys(self):
- keys = ["secret", "secret2", "secret3"]
- data = {"some": "data"}
- packed_data, hmac_data = utils.signed_pack(data, keys[-1])
-
- process_data = utils.signed_unpack(packed_data, hmac_data, keys)
- self.assertEqual(keys[-1], process_data["hmac_key"])
-
- def test_signed_pack_unpack_many_wrong_keys(self):
- keys = ["secret", "secret2", "secret3"]
- data = {"some": "data"}
- packed_data, hmac_data = utils.signed_pack(data, "password")
-
- process_data = utils.signed_unpack(packed_data, hmac_data, keys)
- self.assertIsNone(process_data)
-
- def test_signed_unpack_wrong_key(self):
- data = {"some": "data"}
- packed_data, hmac_data = utils.signed_pack(data, "secret")
-
- self.assertIsNone(utils.signed_unpack(packed_data, hmac_data, "wrong"))
-
- def test_signed_unpack_no_key_or_hmac_data(self):
- data = {"some": "data"}
- packed_data, hmac_data = utils.signed_pack(data, "secret")
- self.assertIsNone(utils.signed_unpack(packed_data, hmac_data, None))
- self.assertIsNone(utils.signed_unpack(packed_data, None, "secret"))
- self.assertIsNone(utils.signed_unpack(packed_data, " ", "secret"))
-
- @mock.patch("osprofiler._utils.generate_hmac")
- def test_singed_unpack_generate_hmac_failed(self, mock_generate_hmac):
- mock_generate_hmac.side_effect = Exception
- self.assertIsNone(utils.signed_unpack("data", "hmac_data", "hmac_key"))
-
- def test_signed_unpack_invalid_json(self):
- hmac = "secret"
- data = base64.urlsafe_b64encode(utils.binary_encode("not_a_json"))
- hmac_data = utils.generate_hmac(data, hmac)
-
- self.assertIsNone(utils.signed_unpack(data, hmac_data, hmac))
-
- def test_itersubclasses(self):
-
- class A(object):
- pass
-
- class B(A):
- pass
-
- class C(A):
- pass
-
- class D(C):
- pass
-
- self.assertEqual([B, C, D], list(utils.itersubclasses(A)))
-
- class E(type):
- pass
-
- self.assertEqual([], list(utils.itersubclasses(E)))
diff --git a/osprofiler/tests/test_web.py b/osprofiler/tests/test_web.py
deleted file mode 100644
index 7355ae2..0000000
--- a/osprofiler/tests/test_web.py
+++ /dev/null
@@ -1,308 +0,0 @@
-# Copyright 2014 Mirantis Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import mock
-from webob import response as webob_response
-
-from osprofiler import _utils as utils
-from osprofiler import profiler
-from osprofiler import web
-
-from osprofiler.tests import test
-
-
-def dummy_app(environ, response):
- res = webob_response.Response()
- return res(environ, response)
-
-
-class WebTestCase(test.TestCase):
-
- def setUp(self):
- super(WebTestCase, self).setUp()
- profiler._clean()
- self.addCleanup(profiler._clean)
-
- def test_get_trace_id_headers_no_hmac(self):
- profiler.init(None, base_id="y", parent_id="z")
- headers = web.get_trace_id_headers()
- self.assertEqual(headers, {})
-
- def test_get_trace_id_headers(self):
- profiler.init("key", base_id="y", parent_id="z")
- headers = web.get_trace_id_headers()
- self.assertEqual(sorted(headers.keys()),
- sorted(["X-Trace-Info", "X-Trace-HMAC"]))
-
- trace_info = utils.signed_unpack(headers["X-Trace-Info"],
- headers["X-Trace-HMAC"], ["key"])
- self.assertIn("hmac_key", trace_info)
- self.assertEqual("key", trace_info.pop("hmac_key"))
- self.assertEqual({"parent_id": "z", "base_id": "y"}, trace_info)
-
- @mock.patch("osprofiler.profiler.get")
- def test_get_trace_id_headers_no_profiler(self, mock_get_profiler):
- mock_get_profiler.return_value = False
- headers = web.get_trace_id_headers()
- self.assertEqual(headers, {})
-
-
-class WebMiddlewareTestCase(test.TestCase):
- def setUp(self):
- super(WebMiddlewareTestCase, self).setUp()
- profiler._clean()
- # it's default state of _ENABLED param, so let's set it here
- web._ENABLED = None
- self.addCleanup(profiler._clean)
-
- def tearDown(self):
- web.enable()
- super(WebMiddlewareTestCase, self).tearDown()
-
- def test_factory(self):
- mock_app = mock.MagicMock()
- local_conf = {"enabled": True, "hmac_keys": "123"}
-
- factory = web.WsgiMiddleware.factory(None, **local_conf)
- wsgi = factory(mock_app)
-
- self.assertEqual(wsgi.application, mock_app)
- self.assertEqual(wsgi.name, "wsgi")
- self.assertTrue(wsgi.enabled)
- self.assertEqual(wsgi.hmac_keys, [local_conf["hmac_keys"]])
-
- def _test_wsgi_middleware_with_invalid_trace(self, headers, hmac_key,
- mock_profiler_init,
- enabled=True):
- request = mock.MagicMock()
- request.get_response.return_value = "yeah!"
- request.headers = headers
-
- middleware = web.WsgiMiddleware("app", hmac_key, enabled=enabled)
- self.assertEqual("yeah!", middleware(request))
- request.get_response.assert_called_once_with("app")
- self.assertEqual(0, mock_profiler_init.call_count)
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_disabled(self, mock_profiler_init):
- hmac_key = "secret"
- pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
- headers = {
- "a": "1",
- "b": "2",
- "X-Trace-Info": pack[0],
- "X-Trace-HMAC": pack[1]
- }
-
- self._test_wsgi_middleware_with_invalid_trace(headers, hmac_key,
- mock_profiler_init,
- enabled=False)
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_no_trace(self, mock_profiler_init):
- headers = {
- "a": "1",
- "b": "2"
- }
- self._test_wsgi_middleware_with_invalid_trace(headers, "secret",
- mock_profiler_init)
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_invalid_trace_headers(self, mock_profiler_init):
- headers = {
- "a": "1",
- "b": "2",
- "X-Trace-Info": "abbababababa",
- "X-Trace-HMAC": "abbababababa"
- }
- self._test_wsgi_middleware_with_invalid_trace(headers, "secret",
- mock_profiler_init)
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_no_trace_hmac(self, mock_profiler_init):
- hmac_key = "secret"
- pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
- headers = {
- "a": "1",
- "b": "2",
- "X-Trace-Info": pack[0]
- }
- self._test_wsgi_middleware_with_invalid_trace(headers, hmac_key,
- mock_profiler_init)
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_invalid_hmac(self, mock_profiler_init):
- hmac_key = "secret"
- pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
- headers = {
- "a": "1",
- "b": "2",
- "X-Trace-Info": pack[0],
- "X-Trace-HMAC": "not valid hmac"
- }
- self._test_wsgi_middleware_with_invalid_trace(headers, hmac_key,
- mock_profiler_init)
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_invalid_trace_info(self, mock_profiler_init):
- hmac_key = "secret"
- pack = utils.signed_pack([{"base_id": "1"}, {"parent_id": "2"}],
- hmac_key)
- headers = {
- "a": "1",
- "b": "2",
- "X-Trace-Info": pack[0],
- "X-Trace-HMAC": pack[1]
- }
- self._test_wsgi_middleware_with_invalid_trace(headers, hmac_key,
- mock_profiler_init)
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_key_passthrough(self, mock_profiler_init):
- hmac_key = "secret2"
- request = mock.MagicMock()
- request.get_response.return_value = "yeah!"
- request.url = "someurl"
- request.host_url = "someurl"
- request.path = "path"
- request.query_string = "query"
- request.method = "method"
- request.scheme = "scheme"
-
- pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
-
- request.headers = {
- "a": "1",
- "b": "2",
- "X-Trace-Info": pack[0],
- "X-Trace-HMAC": pack[1]
- }
-
- middleware = web.WsgiMiddleware("app", "secret1,%s" % hmac_key,
- enabled=True)
- self.assertEqual("yeah!", middleware(request))
- mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
- base_id="1",
- parent_id="2")
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_key_passthrough2(self, mock_profiler_init):
- hmac_key = "secret1"
- request = mock.MagicMock()
- request.get_response.return_value = "yeah!"
- request.url = "someurl"
- request.host_url = "someurl"
- request.path = "path"
- request.query_string = "query"
- request.method = "method"
- request.scheme = "scheme"
-
- pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
-
- request.headers = {
- "a": "1",
- "b": "2",
- "X-Trace-Info": pack[0],
- "X-Trace-HMAC": pack[1]
- }
-
- middleware = web.WsgiMiddleware("app", "%s,secret2" % hmac_key,
- enabled=True)
- self.assertEqual("yeah!", middleware(request))
- mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
- base_id="1",
- parent_id="2")
-
- @mock.patch("osprofiler.web.profiler.Trace")
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware(self, mock_profiler_init, mock_profiler_trace):
- hmac_key = "secret"
- request = mock.MagicMock()
- request.get_response.return_value = "yeah!"
- request.url = "someurl"
- request.host_url = "someurl"
- request.path = "path"
- request.query_string = "query"
- request.method = "method"
- request.scheme = "scheme"
-
- pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
-
- request.headers = {
- "a": "1",
- "b": "2",
- "X-Trace-Info": pack[0],
- "X-Trace-HMAC": pack[1]
- }
-
- middleware = web.WsgiMiddleware("app", hmac_key, enabled=True)
- self.assertEqual("yeah!", middleware(request))
- mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
- base_id="1",
- parent_id="2")
- expected_info = {
- "request": {
- "path": request.path,
- "query": request.query_string,
- "method": request.method,
- "scheme": request.scheme
- }
- }
- mock_profiler_trace.assert_called_once_with("wsgi", info=expected_info)
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_disable_via_python(self, mock_profiler_init):
- request = mock.MagicMock()
- request.get_response.return_value = "yeah!"
- web.disable()
- middleware = web.WsgiMiddleware("app", "hmac_key", enabled=True)
- self.assertEqual("yeah!", middleware(request))
- self.assertEqual(mock_profiler_init.call_count, 0)
-
- @mock.patch("osprofiler.web.profiler.init")
- def test_wsgi_middleware_enable_via_python(self, mock_profiler_init):
- request = mock.MagicMock()
- request.get_response.return_value = "yeah!"
- request.url = "someurl"
- request.host_url = "someurl"
- request.path = "path"
- request.query_string = "query"
- request.method = "method"
- request.scheme = "scheme"
- hmac_key = "super_secret_key2"
-
- pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
- request.headers = {
- "a": "1",
- "b": "2",
- "X-Trace-Info": pack[0],
- "X-Trace-HMAC": pack[1]
- }
-
- web.enable("super_secret_key1,super_secret_key2")
- middleware = web.WsgiMiddleware("app", enabled=True)
- self.assertEqual("yeah!", middleware(request))
- mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
- base_id="1",
- parent_id="2")
-
- def test_disable(self):
- web.disable()
- self.assertFalse(web._ENABLED)
-
- def test_enabled(self):
- web.disable()
- web.enable()
- self.assertTrue(web._ENABLED)
diff --git a/osprofiler/tests/unit/__init__.py b/osprofiler/tests/unit/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/osprofiler/tests/unit/cmd/__init__.py b/osprofiler/tests/unit/cmd/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/osprofiler/tests/unit/cmd/test_shell.py b/osprofiler/tests/unit/cmd/test_shell.py
new file mode 100644
index 0000000..20293ea
--- /dev/null
+++ b/osprofiler/tests/unit/cmd/test_shell.py
@@ -0,0 +1,241 @@
+# Copyright 2014 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import os
+import sys
+
+import ddt
+import mock
+import six
+
+from osprofiler.cmd import shell
+from osprofiler import exc
+from osprofiler.tests import test
+
+
+@ddt.ddt
+class ShellTestCase(test.TestCase):
+
+ TRACE_ID = "c598094d-bbee-40b6-b317-d76003b679d3"
+
+ def setUp(self):
+ super(ShellTestCase, self).setUp()
+ self.old_environment = os.environ.copy()
+ os.environ = {
+ "OS_USERNAME": "username",
+ "OS_USER_ID": "user_id",
+ "OS_PASSWORD": "password",
+ "OS_USER_DOMAIN_ID": "user_domain_id",
+ "OS_USER_DOMAIN_NAME": "user_domain_name",
+ "OS_PROJECT_DOMAIN_ID": "project_domain_id",
+ "OS_PROJECT_DOMAIN_NAME": "project_domain_name",
+ "OS_PROJECT_ID": "project_id",
+ "OS_PROJECT_NAME": "project_name",
+ "OS_TENANT_ID": "tenant_id",
+ "OS_TENANT_NAME": "tenant_name",
+ "OS_AUTH_URL": "http://127.0.0.1:5000/v3/",
+ "OS_AUTH_TOKEN": "pass",
+ "OS_CACERT": "/path/to/cacert",
+ "OS_SERVICE_TYPE": "service_type",
+ "OS_ENDPOINT_TYPE": "public",
+ "OS_REGION_NAME": "test"
+ }
+
+ self.ceiloclient = mock.MagicMock()
+ sys.modules["ceilometerclient"] = self.ceiloclient
+ self.addCleanup(sys.modules.pop, "ceilometerclient", None)
+ ceilo_modules = ["client", "shell"]
+ for module in ceilo_modules:
+ sys.modules["ceilometerclient.%s" % module] = getattr(
+ self.ceiloclient, module)
+ self.addCleanup(
+ sys.modules.pop, "ceilometerclient.%s" % module, None)
+
+ def tearDown(self):
+ super(ShellTestCase, self).tearDown()
+ os.environ = self.old_environment
+
+ def _trace_show_cmd(self, format_=None):
+ cmd = "trace show %s" % self.TRACE_ID
+ return cmd if format_ is None else "%s --%s" % (cmd, format_)
+
+ @mock.patch("sys.stdout", six.StringIO())
+ @mock.patch("osprofiler.cmd.shell.OSProfilerShell")
+ def test_shell_main(self, mock_shell):
+ mock_shell.side_effect = exc.CommandError("some_message")
+ shell.main()
+ self.assertEqual("some_message\n", sys.stdout.getvalue())
+
+ def run_command(self, cmd):
+ shell.OSProfilerShell(cmd.split())
+
+ def _test_with_command_error(self, cmd, expected_message):
+ try:
+ self.run_command(cmd)
+ except exc.CommandError as actual_error:
+ self.assertEqual(str(actual_error), expected_message)
+ else:
+ raise ValueError(
+ "Expected: `osprofiler.exc.CommandError` is raised with "
+ "message: '%s'." % expected_message)
+
+ def test_username_is_not_presented(self):
+ os.environ.pop("OS_USERNAME")
+ msg = ("You must provide a username via either --os-username or "
+ "via env[OS_USERNAME]")
+ self._test_with_command_error(self._trace_show_cmd(), msg)
+
+ def test_password_is_not_presented(self):
+ os.environ.pop("OS_PASSWORD")
+ msg = ("You must provide a password via either --os-password or "
+ "via env[OS_PASSWORD]")
+ self._test_with_command_error(self._trace_show_cmd(), msg)
+
+ def test_auth_url(self):
+ os.environ.pop("OS_AUTH_URL")
+ msg = ("You must provide an auth url via either --os-auth-url or "
+ "via env[OS_AUTH_URL]")
+ self._test_with_command_error(self._trace_show_cmd(), msg)
+
+ def test_no_project_and_domain_set(self):
+ os.environ.pop("OS_PROJECT_ID")
+ os.environ.pop("OS_PROJECT_NAME")
+ os.environ.pop("OS_TENANT_ID")
+ os.environ.pop("OS_TENANT_NAME")
+ os.environ.pop("OS_USER_DOMAIN_ID")
+ os.environ.pop("OS_USER_DOMAIN_NAME")
+
+ msg = ("You must provide a project_id via either --os-project-id or "
+ "via env[OS_PROJECT_ID] and a domain_name via either "
+ "--os-user-domain-name or via env[OS_USER_DOMAIN_NAME] or a "
+ "domain_id via either --os-user-domain-id or via "
+ "env[OS_USER_DOMAIN_ID]")
+ self._test_with_command_error(self._trace_show_cmd(), msg)
+
+ def test_trace_show_ceilometerclient_is_missed(self):
+ sys.modules["ceilometerclient"] = None
+ sys.modules["ceilometerclient.client"] = None
+ sys.modules["ceilometerclient.shell"] = None
+
+ msg = ("To use this command, you should install "
+ "'ceilometerclient' manually. Use command:\n "
+ "'pip install python-ceilometerclient'.")
+ self._test_with_command_error(self._trace_show_cmd(), msg)
+
+ def test_trace_show_unauthorized(self):
+ class FakeHTTPUnauthorized(Exception):
+ http_status = 401
+
+ self.ceiloclient.client.get_client.side_effect = FakeHTTPUnauthorized
+
+ msg = "Invalid OpenStack Identity credentials."
+ self._test_with_command_error(self._trace_show_cmd(), msg)
+
+ def test_trace_show_unknown_error(self):
+ self.ceiloclient.client.get_client.side_effect = Exception("test")
+ msg = "Error occurred while connecting to Ceilometer: test."
+ self._test_with_command_error(self._trace_show_cmd(), msg)
+
+ @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
+ def test_trace_show_no_selected_format(self, mock_get):
+ mock_get.return_value = self._create_mock_notifications()
+ msg = ("You should choose one of the following output formats: "
+ "json, html or dot.")
+ self._test_with_command_error(self._trace_show_cmd(), msg)
+
+ @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
+ @ddt.data(None, {"info": {"started": 0, "finished": 1, "name": "total"},
+ "children": []})
+ def test_trace_show_trace_id_not_found(self, notifications, mock_get):
+ mock_get.return_value = notifications
+
+ msg = ("Trace with UUID %s not found. Please check the HMAC key "
+ "used in the command." % self.TRACE_ID)
+
+ self._test_with_command_error(self._trace_show_cmd(), msg)
+
+ def _create_mock_notifications(self):
+ notifications = {
+ "info": {
+ "started": 0,
+ "finished": 1,
+ "name": "total"
+ },
+ "children": [{
+ "info": {
+ "started": 0,
+ "finished": 1,
+ "name": "total"
+ },
+ "children": []
+ }]
+ }
+ return notifications
+
+ @mock.patch("sys.stdout", six.StringIO())
+ @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
+ def test_trace_show_in_json(self, mock_get):
+ notifications = self._create_mock_notifications()
+ mock_get.return_value = notifications
+
+ self.run_command(self._trace_show_cmd(format_="json"))
+ self.assertEqual("%s\n" % json.dumps(notifications, indent=2,
+ separators=(",", ": "),),
+ sys.stdout.getvalue())
+
+ @mock.patch("sys.stdout", six.StringIO())
+ @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
+ def test_trace_show_in_html(self, mock_get):
+ notifications = self._create_mock_notifications()
+ mock_get.return_value = notifications
+
+ # NOTE(akurilin): to simplify assert statement, html-template should be
+ # replaced.
+ html_template = (
+ "A long time ago in a galaxy far, far away..."
+ " some_data = $DATA"
+ "It is a period of civil war. Rebel"
+ "spaceships, striking from a hidden"
+ "base, have won their first victory"
+ "against the evil Galactic Empire.")
+
+ with mock.patch("osprofiler.cmd.commands.open",
+ mock.mock_open(read_data=html_template), create=True):
+ self.run_command(self._trace_show_cmd(format_="html"))
+ self.assertEqual("A long time ago in a galaxy far, far away..."
+ " some_data = %s"
+ "It is a period of civil war. Rebel"
+ "spaceships, striking from a hidden"
+ "base, have won their first victory"
+ "against the evil Galactic Empire."
+ "\n" % json.dumps(notifications, indent=4,
+ separators=(",", ": ")),
+ sys.stdout.getvalue())
+
+ @mock.patch("sys.stdout", six.StringIO())
+ @mock.patch("osprofiler.drivers.ceilometer.Ceilometer.get_report")
+ def test_trace_show_write_to_file(self, mock_get):
+ notifications = self._create_mock_notifications()
+ mock_get.return_value = notifications
+
+ with mock.patch("osprofiler.cmd.commands.open",
+ mock.mock_open(), create=True) as mock_open:
+ self.run_command("%s --out='/file'" %
+ self._trace_show_cmd(format_="json"))
+
+ output = mock_open.return_value.__enter__.return_value
+ output.write.assert_called_once_with(
+ json.dumps(notifications, indent=2, separators=(",", ": ")))
diff --git a/osprofiler/tests/unit/doc/__init__.py b/osprofiler/tests/unit/doc/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/osprofiler/tests/unit/doc/test_specs.py b/osprofiler/tests/unit/doc/test_specs.py
new file mode 100644
index 0000000..fe2b867
--- /dev/null
+++ b/osprofiler/tests/unit/doc/test_specs.py
@@ -0,0 +1,119 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import glob
+import os
+import re
+
+import docutils.core
+
+from osprofiler.tests import test
+
+
+class TitlesTestCase(test.TestCase):
+
+ specs_path = os.path.join(
+ os.path.dirname(__file__),
+ os.pardir, os.pardir, os.pardir, os.pardir,
+ "doc", "specs")
+
+ def _get_title(self, section_tree):
+ section = {"subtitles": []}
+ for node in section_tree:
+ if node.tagname == "title":
+ section["name"] = node.rawsource
+ elif node.tagname == "section":
+ subsection = self._get_title(node)
+ section["subtitles"].append(subsection["name"])
+ return section
+
+ def _get_titles(self, spec):
+ titles = {}
+ for node in spec:
+ if node.tagname == "section":
+ # Note subsection subtitles are thrown away
+ section = self._get_title(node)
+ titles[section["name"]] = section["subtitles"]
+ return titles
+
+ def _check_titles(self, filename, expect, actual):
+ missing_sections = [x for x in expect.keys() if x not in actual.keys()]
+ extra_sections = [x for x in actual.keys() if x not in expect.keys()]
+
+ msgs = []
+ if len(missing_sections) > 0:
+ msgs.append("Missing sections: %s" % missing_sections)
+ if len(extra_sections) > 0:
+ msgs.append("Extra sections: %s" % extra_sections)
+
+ for section in expect.keys():
+ missing_subsections = [x for x in expect[section]
+ if x not in actual.get(section, {})]
+ # extra subsections are allowed
+ if len(missing_subsections) > 0:
+ msgs.append("Section '%s' is missing subsections: %s"
+ % (section, missing_subsections))
+
+ if len(msgs) > 0:
+ self.fail("While checking '%s':\n %s"
+ % (filename, "\n ".join(msgs)))
+
+ def _check_lines_wrapping(self, tpl, raw):
+ for i, line in enumerate(raw.split("\n")):
+ if "http://" in line or "https://" in line:
+ continue
+ self.assertTrue(
+ len(line) < 80,
+ msg="%s:%d: Line limited to a maximum of 79 characters." %
+ (tpl, i+1))
+
+ def _check_no_cr(self, tpl, raw):
+ matches = re.findall("\r", raw)
+ self.assertEqual(
+ len(matches), 0,
+ "Found %s literal carriage returns in file %s" %
+ (len(matches), tpl))
+
+ def _check_trailing_spaces(self, tpl, raw):
+ for i, line in enumerate(raw.split("\n")):
+ trailing_spaces = re.findall(" +$", line)
+ self.assertEqual(
+ len(trailing_spaces), 0,
+ "Found trailing spaces on line %s of %s" % (i+1, tpl))
+
+ def test_template(self):
+ with open(os.path.join(self.specs_path, "template.rst")) as f:
+ template = f.read()
+
+ spec = docutils.core.publish_doctree(template)
+ template_titles = self._get_titles(spec)
+
+ for d in ["implemented", "in-progress"]:
+ spec_dir = "%s/%s" % (self.specs_path, d)
+
+ self.assertTrue(os.path.isdir(spec_dir),
+ "%s is not a directory" % spec_dir)
+ for filename in glob.glob(spec_dir + "/*"):
+ if filename.endswith("README.rst"):
+ continue
+
+ self.assertTrue(
+ filename.endswith(".rst"),
+ "spec's file must have .rst ext. Found: %s" % filename)
+ with open(filename) as f:
+ data = f.read()
+
+ titles = self._get_titles(docutils.core.publish_doctree(data))
+ self._check_titles(filename, template_titles, titles)
+ self._check_lines_wrapping(filename, data)
+ self._check_no_cr(filename, data)
+ self._check_trailing_spaces(filename, data)
diff --git a/osprofiler/tests/unit/drivers/__init__.py b/osprofiler/tests/unit/drivers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/osprofiler/tests/unit/drivers/test_base.py b/osprofiler/tests/unit/drivers/test_base.py
new file mode 100644
index 0000000..462559b
--- /dev/null
+++ b/osprofiler/tests/unit/drivers/test_base.py
@@ -0,0 +1,125 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from osprofiler.drivers import base
+from osprofiler.tests import test
+
+
+class NotifierBaseTestCase(test.TestCase):
+
+ def test_factory(self):
+
+ class A(base.Driver):
+ @classmethod
+ def get_name(cls):
+ return "a"
+
+ def notify(self, a):
+ return a
+
+ self.assertEqual(10, base.get_driver("a://").notify(10))
+
+ def test_factory_with_args(self):
+
+ class B(base.Driver):
+
+ def __init__(self, c_str, a, b=10):
+ self.a = a
+ self.b = b
+
+ @classmethod
+ def get_name(cls):
+ return "b"
+
+ def notify(self, c):
+ return self.a + self.b + c
+
+ self.assertEqual(22, base.get_driver("b://", 5, b=7).notify(10))
+
+ def test_driver_not_found(self):
+ self.assertRaises(ValueError, base.get_driver,
+ "Driver not found for connection string: "
+ "nonexisting://")
+
+ def test_plugins_are_imported(self):
+ base.get_driver("messaging://", mock.MagicMock(), "context",
+ "transport", "host")
+
+ def test_build_empty_tree(self):
+ class C(base.Driver):
+ @classmethod
+ def get_name(cls):
+ return "c"
+
+ self.assertEqual([], base.get_driver("c://")._build_tree({}))
+
+ def test_build_complex_tree(self):
+ class D(base.Driver):
+ @classmethod
+ def get_name(cls):
+ return "d"
+
+ test_input = {
+ "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}},
+ "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}},
+ "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}},
+ "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}},
+ "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}},
+ "113": {"parent_id": "11", "trace_id": "113",
+ "info": {"started": 3}},
+ "112": {"parent_id": "11", "trace_id": "112",
+ "info": {"started": 2}},
+ "114": {"parent_id": "11", "trace_id": "114",
+ "info": {"started": 5}}
+ }
+
+ expected_output = [
+ {
+ "parent_id": "0",
+ "trace_id": "1",
+ "info": {"started": 0},
+ "children": [
+ {
+ "parent_id": "1",
+ "trace_id": "11",
+ "info": {"started": 1},
+ "children": [
+ {"parent_id": "11", "trace_id": "112",
+ "info": {"started": 2}, "children": []},
+ {"parent_id": "11", "trace_id": "113",
+ "info": {"started": 3}, "children": []},
+ {"parent_id": "11", "trace_id": "114",
+ "info": {"started": 5}, "children": []}
+ ]
+ }
+ ]
+ },
+ {
+ "parent_id": "0",
+ "trace_id": "2",
+ "info": {"started": 1},
+ "children": [
+ {"parent_id": "2", "trace_id": "21",
+ "info": {"started": 6}, "children": []},
+ {"parent_id": "2", "trace_id": "22",
+ "info": {"started": 7}, "children": []}
+ ]
+ }
+ ]
+
+ self.assertEqual(
+ expected_output, base.get_driver("d://")._build_tree(test_input))
diff --git a/osprofiler/tests/unit/drivers/test_ceilometer.py b/osprofiler/tests/unit/drivers/test_ceilometer.py
new file mode 100644
index 0000000..127d60f
--- /dev/null
+++ b/osprofiler/tests/unit/drivers/test_ceilometer.py
@@ -0,0 +1,423 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from osprofiler.drivers.ceilometer import Ceilometer
+from osprofiler.tests import test
+
+
+class CeilometerParserTestCase(test.TestCase):
+ def setUp(self):
+ super(CeilometerParserTestCase, self).setUp()
+ self.ceilometer = Ceilometer("ceilometer://",
+ ceilometer_api_version="2")
+
+ def test_build_empty_tree(self):
+ self.assertEqual([], self.ceilometer._build_tree({}))
+
+ def test_build_complex_tree(self):
+ test_input = {
+ "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}},
+ "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}},
+ "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}},
+ "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}},
+ "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}},
+ "113": {"parent_id": "11", "trace_id": "113",
+ "info": {"started": 3}},
+ "112": {"parent_id": "11", "trace_id": "112",
+ "info": {"started": 2}},
+ "114": {"parent_id": "11", "trace_id": "114",
+ "info": {"started": 5}}
+ }
+
+ expected_output = [
+ {
+ "parent_id": "0",
+ "trace_id": "1",
+ "info": {"started": 0},
+ "children": [
+ {
+ "parent_id": "1",
+ "trace_id": "11",
+ "info": {"started": 1},
+ "children": [
+ {"parent_id": "11", "trace_id": "112",
+ "info": {"started": 2}, "children": []},
+ {"parent_id": "11", "trace_id": "113",
+ "info": {"started": 3}, "children": []},
+ {"parent_id": "11", "trace_id": "114",
+ "info": {"started": 5}, "children": []}
+ ]
+ }
+ ]
+ },
+ {
+ "parent_id": "0",
+ "trace_id": "2",
+ "info": {"started": 1},
+ "children": [
+ {"parent_id": "2", "trace_id": "21",
+ "info": {"started": 6}, "children": []},
+ {"parent_id": "2", "trace_id": "22",
+ "info": {"started": 7}, "children": []}
+ ]
+ }
+ ]
+
+ result = self.ceilometer._build_tree(test_input)
+ self.assertEqual(expected_output, result)
+
+ def test_get_report_empty(self):
+ self.ceilometer.client = mock.MagicMock()
+ self.ceilometer.client.events.list.return_value = []
+
+ expected = {
+ "info": {
+ "name": "total",
+ "started": 0,
+ "finished": None,
+ "last_trace_started": None
+ },
+ "children": [],
+ "stats": {},
+ }
+
+ base_id = "10"
+ self.assertEqual(expected, self.ceilometer.get_report(base_id))
+
+ def test_get_report(self):
+ self.ceilometer.client = mock.MagicMock()
+ results = [mock.MagicMock(), mock.MagicMock(), mock.MagicMock(),
+ mock.MagicMock(), mock.MagicMock()]
+
+ self.ceilometer.client.events.list.return_value = results
+ results[0].to_dict.return_value = {
+ "traits": [
+ {
+ "type": "string",
+ "name": "base_id",
+ "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+ {
+ "type": "string",
+ "name": "host",
+ "value": "ubuntu"
+ },
+ {
+ "type": "string",
+ "name": "method",
+ "value": "POST"
+ },
+ {
+ "type": "string",
+ "name": "name",
+ "value": "wsgi-start"
+ },
+ {
+ "type": "string",
+ "name": "parent_id",
+ "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+ {
+ "type": "string",
+ "name": "project",
+ "value": "keystone"
+ },
+ {
+ "type": "string",
+ "name": "service",
+ "value": "main"
+ },
+ {
+ "type": "string",
+ "name": "timestamp",
+ "value": "2015-12-23T14:02:22.338776"
+ },
+ {
+ "type": "string",
+ "name": "trace_id",
+ "value": "06320327-2c2c-45ae-923a-515de890276a"
+ }
+ ],
+ "raw": {},
+ "generated": "2015-12-23T10:41:38.415793",
+ "event_type": "profiler.main",
+ "message_id": "65fc1553-3082-4a6f-9d1e-0e3183f57a47"}
+
+ results[1].to_dict.return_value = {
+ "traits":
+ [
+ {
+ "type": "string",
+ "name": "base_id",
+ "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+ {
+ "type": "string",
+ "name": "host",
+ "value": "ubuntu"
+ },
+ {
+ "type": "string",
+ "name": "name",
+ "value": "wsgi-stop"
+ },
+ {
+ "type": "string",
+ "name": "parent_id",
+ "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+ {
+ "type": "string",
+ "name": "project",
+ "value": "keystone"
+ },
+ {
+ "type": "string",
+ "name": "service",
+ "value": "main"
+ },
+ {
+ "type": "string",
+ "name": "timestamp",
+ "value": "2015-12-23T14:02:22.380405"
+ },
+ {
+ "type": "string",
+ "name": "trace_id",
+ "value": "016c97fd-87f3-40b2-9b55-e431156b694b"
+ }
+ ],
+ "raw": {},
+ "generated": "2015-12-23T10:41:38.406052",
+ "event_type": "profiler.main",
+ "message_id": "3256d9f1-48ba-4ac5-a50b-64fa42c6e264"}
+
+ results[2].to_dict.return_value = {
+ "traits":
+ [
+ {
+ "type": "string",
+ "name": "base_id",
+ "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+ {
+ "type": "string",
+ "name": "db.params",
+ "value": "[]"
+ },
+ {
+ "type": "string",
+ "name": "db.statement",
+ "value": "SELECT 1"
+ },
+ {
+ "type": "string",
+ "name": "host",
+ "value": "ubuntu"
+ },
+ {
+ "type": "string",
+ "name": "name",
+ "value": "db-start"
+ },
+ {
+ "type": "string",
+ "name": "parent_id",
+ "value": "06320327-2c2c-45ae-923a-515de890276a"
+ },
+ {
+ "type": "string",
+ "name": "project",
+ "value": "keystone"
+ },
+ {
+ "type": "string",
+ "name": "service",
+ "value": "main"
+ },
+ {
+ "type": "string",
+ "name": "timestamp",
+ "value": "2015-12-23T14:02:22.395365"
+ },
+ {
+ "type": "string",
+ "name": "trace_id",
+ "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"
+ }
+ ],
+ "raw": {},
+ "generated": "2015-12-23T10:41:38.984161",
+ "event_type": "profiler.main",
+ "message_id": "60368aa4-16f0-4f37-a8fb-89e92fdf36ff"}
+
+ results[3].to_dict.return_value = {
+ "traits":
+ [
+ {
+ "type": "string",
+ "name": "base_id",
+ "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+ {
+ "type": "string",
+ "name": "host",
+ "value": "ubuntu"
+ },
+ {
+ "type": "string",
+ "name": "name",
+ "value": "db-stop"
+ },
+ {
+ "type": "string",
+ "name": "parent_id",
+ "value": "06320327-2c2c-45ae-923a-515de890276a"
+ },
+ {
+ "type": "string",
+ "name": "project",
+ "value": "keystone"
+ },
+ {
+ "type": "string",
+ "name": "service",
+ "value": "main"
+ },
+ {
+ "type": "string",
+ "name": "timestamp",
+ "value": "2015-12-23T14:02:22.415486"
+ },
+ {
+ "type": "string",
+ "name": "trace_id",
+ "value": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"
+ }
+ ],
+ "raw": {},
+ "generated": "2015-12-23T10:41:39.019378",
+ "event_type": "profiler.main",
+ "message_id": "3fbeb339-55c5-4f28-88e4-15bee251dd3d"}
+
+ results[4].to_dict.return_value = {
+ "traits":
+ [
+ {
+ "type": "string",
+ "name": "base_id",
+ "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+ {
+ "type": "string",
+ "name": "host",
+ "value": "ubuntu"
+ },
+ {
+ "type": "string",
+ "name": "method",
+ "value": "GET"
+ },
+ {
+ "type": "string",
+ "name": "name",
+ "value": "wsgi-start"
+ },
+ {
+ "type": "string",
+ "name": "parent_id",
+ "value": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+ {
+ "type": "string",
+ "name": "project",
+ "value": "keystone"
+ },
+ {
+ "type": "string",
+ "name": "service",
+ "value": "main"
+ },
+ {
+ "type": "string",
+ "name": "timestamp",
+ "value": "2015-12-23T14:02:22.427444"
+ },
+ {
+ "type": "string",
+ "name": "trace_id",
+ "value": "016c97fd-87f3-40b2-9b55-e431156b694b"
+ }
+ ],
+ "raw": {},
+ "generated": "2015-12-23T10:41:38.360409",
+ "event_type": "profiler.main",
+ "message_id": "57b971a9-572f-4f29-9838-3ed2564c6b5b"}
+
+ expected = {"children": [
+ {"children": [{"children": [],
+ "info": {"finished": 76,
+ "host": "ubuntu",
+ "meta.raw_payload.db-start": {},
+ "meta.raw_payload.db-stop": {},
+ "name": "db",
+ "project": "keystone",
+ "service": "main",
+ "started": 56,
+ "exception": "None"},
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}
+ ],
+ "info": {"finished": 0,
+ "host": "ubuntu",
+ "meta.raw_payload.wsgi-start": {},
+ "name": "wsgi",
+ "project": "keystone",
+ "service": "main",
+ "started": 0},
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
+ {"children": [],
+ "info": {"finished": 41,
+ "host": "ubuntu",
+ "meta.raw_payload.wsgi-start": {},
+ "meta.raw_payload.wsgi-stop": {},
+ "name": "wsgi",
+ "project": "keystone",
+ "service": "main",
+ "started": 88,
+ "exception": "None"},
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}],
+ "info": {
+ "finished": 88,
+ "name": "total",
+ "started": 0,
+ "last_trace_started": 88
+ },
+ "stats": {"db": {"count": 1, "duration": 20},
+ "wsgi": {"count": 2, "duration": -47}},
+ }
+
+ base_id = "10"
+
+ result = self.ceilometer.get_report(base_id)
+
+ expected_filter = [{"field": "base_id", "op": "eq", "value": base_id}]
+ self.ceilometer.client.events.list.assert_called_once_with(
+ expected_filter, limit=100000)
+ self.assertEqual(expected, result)
diff --git a/osprofiler/tests/unit/drivers/test_elasticsearch.py b/osprofiler/tests/unit/drivers/test_elasticsearch.py
new file mode 100644
index 0000000..c7385de
--- /dev/null
+++ b/osprofiler/tests/unit/drivers/test_elasticsearch.py
@@ -0,0 +1,114 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from osprofiler.drivers.elasticsearch_driver import ElasticsearchDriver
+from osprofiler.tests import test
+
+
+class ElasticsearchTestCase(test.TestCase):
+
+ def setUp(self):
+ super(ElasticsearchTestCase, self).setUp()
+ self.elasticsearch = ElasticsearchDriver("elasticsearch://localhost")
+ self.elasticsearch.project = "project"
+ self.elasticsearch.service = "service"
+
+ def test_init_and_notify(self):
+ self.elasticsearch.client = mock.MagicMock()
+ self.elasticsearch.client.reset_mock()
+ project = "project"
+ service = "service"
+ host = "host"
+
+ info = {
+ "a": 10,
+ "project": project,
+ "service": service,
+ "host": host
+ }
+ self.elasticsearch.notify(info)
+
+ self.elasticsearch.client\
+ .index.assert_called_once_with(index="osprofiler-notifications",
+ doc_type="notification",
+ body=info)
+
+ def test_get_empty_report(self):
+ self.elasticsearch.client = mock.MagicMock()
+ self.elasticsearch.client.search = mock\
+ .MagicMock(return_value={"_scroll_id": "1", "hits": {"hits": []}})
+ self.elasticsearch.client.reset_mock()
+
+ get_report = self.elasticsearch.get_report
+ base_id = "abacaba"
+
+ get_report(base_id)
+
+ self.elasticsearch.client\
+ .search.assert_called_once_with(index="osprofiler-notifications",
+ doc_type="notification",
+ size=10000,
+ scroll="2m",
+ body={"query": {
+ "match": {"base_id": base_id}}
+ })
+
+ def test_get_non_empty_report(self):
+ base_id = "1"
+ elasticsearch_first_response = {
+ "_scroll_id": "1",
+ "hits": {
+ "hits": [
+ {
+ "_source": {
+ "timestamp": "2016-08-10T16:58:03.064438",
+ "base_id": base_id,
+ "project": "project",
+ "service": "service",
+ "parent_id": "0",
+ "name": "test",
+ "info": {
+ "host": "host"
+ },
+ "trace_id": "1"
+ }
+ }
+ ]}}
+ elasticsearch_second_response = {
+ "_scroll_id": base_id,
+ "hits": {"hits": []}}
+ self.elasticsearch.client = mock.MagicMock()
+ self.elasticsearch.client.search = \
+ mock.MagicMock(return_value=elasticsearch_first_response)
+ self.elasticsearch.client.scroll = \
+ mock.MagicMock(return_value=elasticsearch_second_response)
+
+ self.elasticsearch.client.reset_mock()
+
+ self.elasticsearch.get_report(base_id)
+
+ self.elasticsearch.client\
+ .search.assert_called_once_with(index="osprofiler-notifications",
+ doc_type="notification",
+ size=10000,
+ scroll="2m",
+ body={"query": {
+ "match": {"base_id": base_id}}
+ })
+
+ self.elasticsearch.client\
+ .scroll.assert_called_once_with(scroll_id=base_id, scroll="2m")
diff --git a/osprofiler/tests/unit/drivers/test_loginsight.py b/osprofiler/tests/unit/drivers/test_loginsight.py
new file mode 100644
index 0000000..5e28aee
--- /dev/null
+++ b/osprofiler/tests/unit/drivers/test_loginsight.py
@@ -0,0 +1,296 @@
+# Copyright (c) 2016 VMware, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+
+import ddt
+import mock
+
+from osprofiler.drivers import loginsight
+from osprofiler import exc
+from osprofiler.tests import test
+
+
+@ddt.ddt
+class LogInsightDriverTestCase(test.TestCase):
+
+ BASE_ID = "8d28af1e-acc0-498c-9890-6908e33eff5f"
+
+ def setUp(self):
+ super(LogInsightDriverTestCase, self).setUp()
+ self._client = mock.Mock(spec=loginsight.LogInsightClient)
+ self._project = "cinder"
+ self._service = "osapi_volume"
+ self._host = "ubuntu"
+ with mock.patch.object(loginsight, "LogInsightClient",
+ return_value=self._client):
+ self._driver = loginsight.LogInsightDriver(
+ "loginsight://username:password@host",
+ project=self._project,
+ service=self._service,
+ host=self._host)
+
+ @mock.patch.object(loginsight, "LogInsightClient")
+ def test_init(self, client_class):
+ client = mock.Mock()
+ client_class.return_value = client
+
+ loginsight.LogInsightDriver("loginsight://username:password@host")
+ client_class.assert_called_once_with("host", "username", "password")
+ client.login.assert_called_once_with()
+
+ @ddt.data("loginsight://username@host",
+ "loginsight://username:p@ssword@host",
+ "loginsight://us:rname:password@host")
+ def test_init_with_invalid_connection_string(self, conn_str):
+ self.assertRaises(ValueError, loginsight.LogInsightDriver, conn_str)
+
+ @mock.patch.object(loginsight, "LogInsightClient")
+ def test_init_with_special_chars_in_conn_str(self, client_class):
+ client = mock.Mock()
+ client_class.return_value = client
+
+ loginsight.LogInsightDriver("loginsight://username:p%40ssword@host")
+ client_class.assert_called_once_with("host", "username", "p@ssword")
+ client.login.assert_called_once_with()
+
+ def test_get_name(self):
+ self.assertEqual("loginsight", self._driver.get_name())
+
+ def _create_trace(self,
+ name,
+ timestamp,
+ parent_id="8d28af1e-acc0-498c-9890-6908e33eff5f",
+ base_id=BASE_ID,
+ trace_id="e465db5c-9672-45a1-b90b-da918f30aef6"):
+ return {"parent_id": parent_id,
+ "name": name,
+ "base_id": base_id,
+ "trace_id": trace_id,
+ "timestamp": timestamp,
+ "info": {"host": self._host}}
+
+ def _create_start_trace(self):
+ return self._create_trace("wsgi-start", "2016-10-04t11:50:21.902303")
+
+ def _create_stop_trace(self):
+ return self._create_trace("wsgi-stop", "2016-10-04t11:50:30.123456")
+
+ @mock.patch("json.dumps")
+ def test_notify(self, dumps):
+ json_str = mock.sentinel.json_str
+ dumps.return_value = json_str
+
+ trace = self._create_stop_trace()
+ self._driver.notify(trace)
+
+ trace["project"] = self._project
+ trace["service"] = self._service
+ exp_event = {"text": "OSProfiler trace",
+ "fields": [{"name": "base_id",
+ "content": trace["base_id"]},
+ {"name": "trace_id",
+ "content": trace["trace_id"]},
+ {"name": "project",
+ "content": trace["project"]},
+ {"name": "service",
+ "content": trace["service"]},
+ {"name": "name",
+ "content": trace["name"]},
+ {"name": "trace",
+ "content": json_str}]
+ }
+ self._client.send_event.assert_called_once_with(exp_event)
+
+ @mock.patch.object(loginsight.LogInsightDriver, "_append_results")
+ @mock.patch.object(loginsight.LogInsightDriver, "_parse_results")
+ def test_get_report(self, parse_results, append_results):
+ start_trace = self._create_start_trace()
+ start_trace["project"] = self._project
+ start_trace["service"] = self._service
+
+ stop_trace = self._create_stop_trace()
+ stop_trace["project"] = self._project
+ stop_trace["service"] = self._service
+
+ resp = {"events": [{"text": "OSProfiler trace",
+ "fields": [{"name": "trace",
+ "content": json.dumps(start_trace)
+ }
+ ]
+ },
+ {"text": "OSProfiler trace",
+ "fields": [{"name": "trace",
+ "content": json.dumps(stop_trace)
+ }
+ ]
+ }
+ ]
+ }
+ self._client.query_events = mock.Mock(return_value=resp)
+
+ self._driver.get_report(self.BASE_ID)
+ self._client.query_events.assert_called_once_with({"base_id":
+ self.BASE_ID})
+ append_results.assert_has_calls(
+ [mock.call(start_trace["trace_id"], start_trace["parent_id"],
+ start_trace["name"], start_trace["project"],
+ start_trace["service"], start_trace["info"]["host"],
+ start_trace["timestamp"], start_trace),
+ mock.call(stop_trace["trace_id"], stop_trace["parent_id"],
+ stop_trace["name"], stop_trace["project"],
+ stop_trace["service"], stop_trace["info"]["host"],
+ stop_trace["timestamp"], stop_trace)
+ ])
+ parse_results.assert_called_once_with()
+
+
+class LogInsightClientTestCase(test.TestCase):
+
+ def setUp(self):
+ super(LogInsightClientTestCase, self).setUp()
+ self._host = "localhost"
+ self._username = "username"
+ self._password = "password"
+ self._client = loginsight.LogInsightClient(
+ self._host, self._username, self._password)
+ self._client._session_id = "4ff800d1-3175-4b49-9209-39714ea56416"
+
+ def test_check_response_login_timeout(self):
+ resp = mock.Mock(status_code=440)
+ self.assertRaises(
+ exc.LogInsightLoginTimeout, self._client._check_response, resp)
+
+ def test_check_response_api_error(self):
+ resp = mock.Mock(status_code=401, ok=False)
+ resp.text = json.dumps(
+ {"errorMessage": "Invalid username or password.",
+ "errorCode": "FIELD_ERROR"})
+ e = self.assertRaises(
+ exc.LogInsightAPIError, self._client._check_response, resp)
+ self.assertEqual("Invalid username or password.", str(e))
+
+ @mock.patch("requests.Request")
+ @mock.patch("json.dumps")
+ @mock.patch.object(loginsight.LogInsightClient, "_check_response")
+ def test_send_request(self, check_resp, json_dumps, request_class):
+ req = mock.Mock()
+ request_class.return_value = req
+ prep_req = mock.sentinel.prep_req
+ req.prepare = mock.Mock(return_value=prep_req)
+
+ data = mock.sentinel.data
+ json_dumps.return_value = data
+
+ self._client._session = mock.Mock()
+ resp = mock.Mock()
+ self._client._session.send = mock.Mock(return_value=resp)
+ resp_json = mock.sentinel.resp_json
+ resp.json = mock.Mock(return_value=resp_json)
+
+ header = {"X-LI-Session-Id": "foo"}
+ body = mock.sentinel.body
+ params = mock.sentinel.params
+ ret = self._client._send_request(
+ "get", "https", "api/v1/events", header, body, params)
+
+ self.assertEqual(resp_json, ret)
+ exp_headers = {"X-LI-Session-Id": "foo",
+ "content-type": "application/json"}
+ request_class.assert_called_once_with(
+ "get", "https://localhost:9543/api/v1/events", headers=exp_headers,
+ data=data, params=mock.sentinel.params)
+ self._client._session.send.assert_called_once_with(prep_req,
+ verify=False)
+ check_resp.assert_called_once_with(resp)
+
+ @mock.patch.object(loginsight.LogInsightClient, "_send_request")
+ def test_is_current_session_active_with_active_session(self, send_request):
+ self.assertTrue(self._client._is_current_session_active())
+ exp_header = {"X-LI-Session-Id": self._client._session_id}
+ send_request.assert_called_once_with(
+ "get", "https", "api/v1/sessions/current", headers=exp_header)
+
+ @mock.patch.object(loginsight.LogInsightClient, "_send_request")
+ def test_is_current_session_active_with_expired_session(self,
+ send_request):
+ send_request.side_effect = exc.LogInsightLoginTimeout
+
+ self.assertFalse(self._client._is_current_session_active())
+ send_request.assert_called_once_with(
+ "get", "https", "api/v1/sessions/current",
+ headers={"X-LI-Session-Id": self._client._session_id})
+
+ @mock.patch.object(loginsight.LogInsightClient,
+ "_is_current_session_active", return_value=True)
+ @mock.patch.object(loginsight.LogInsightClient, "_send_request")
+ def test_login_with_current_session_active(self, send_request,
+ is_current_session_active):
+ self._client.login()
+ is_current_session_active.assert_called_once_with()
+ send_request.assert_not_called()
+
+ @mock.patch.object(loginsight.LogInsightClient,
+ "_is_current_session_active", return_value=False)
+ @mock.patch.object(loginsight.LogInsightClient, "_send_request")
+ def test_login(self, send_request, is_current_session_active):
+ new_session_id = "569a80aa-be5c-49e5-82c1-bb62392d2667"
+ resp = {"sessionId": new_session_id}
+ send_request.return_value = resp
+
+ self._client.login()
+ is_current_session_active.assert_called_once_with()
+ exp_body = {"username": self._username, "password": self._password}
+ send_request.assert_called_once_with(
+ "post", "https", "api/v1/sessions", body=exp_body)
+ self.assertEqual(new_session_id, self._client._session_id)
+
+ @mock.patch.object(loginsight.LogInsightClient, "_send_request")
+ def test_send_event(self, send_request):
+ event = mock.sentinel.event
+ self._client.send_event(event)
+
+ exp_body = {"events": [event]}
+ exp_path = ("api/v1/events/ingest/%s" %
+ self._client.LI_OSPROFILER_AGENT_ID)
+ send_request.assert_called_once_with(
+ "post", "http", exp_path, body=exp_body)
+
+ @mock.patch.object(loginsight.LogInsightClient, "_send_request")
+ def test_query_events(self, send_request):
+ resp = mock.sentinel.response
+ send_request.return_value = resp
+
+ self.assertEqual(resp, self._client.query_events({"foo": "bar"}))
+ exp_header = {"X-LI-Session-Id": self._client._session_id}
+ exp_params = {"limit": 20000, "timeout": self._client._query_timeout}
+ send_request.assert_called_once_with(
+ "get", "https", "api/v1/events/foo/CONTAINS+bar/timestamp/GT+0",
+ headers=exp_header, params=exp_params)
+
+ @mock.patch.object(loginsight.LogInsightClient, "_send_request")
+ @mock.patch.object(loginsight.LogInsightClient, "login")
+ def test_query_events_with_session_expiry(self, login, send_request):
+ resp = mock.sentinel.response
+ send_request.side_effect = [exc.LogInsightLoginTimeout, resp]
+
+ self.assertEqual(resp, self._client.query_events({"foo": "bar"}))
+ login.assert_called_once_with()
+ exp_header = {"X-LI-Session-Id": self._client._session_id}
+ exp_params = {"limit": 20000, "timeout": self._client._query_timeout}
+ exp_send_request_call = mock.call(
+ "get", "https", "api/v1/events/foo/CONTAINS+bar/timestamp/GT+0",
+ headers=exp_header, params=exp_params)
+ send_request.assert_has_calls([exp_send_request_call]*2)
diff --git a/osprofiler/tests/unit/drivers/test_messaging.py b/osprofiler/tests/unit/drivers/test_messaging.py
new file mode 100644
index 0000000..ad59c73
--- /dev/null
+++ b/osprofiler/tests/unit/drivers/test_messaging.py
@@ -0,0 +1,55 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from osprofiler.drivers import base
+from osprofiler.tests import test
+
+
+class MessagingTestCase(test.TestCase):
+
+ def test_init_and_notify(self):
+
+ messaging = mock.MagicMock()
+ context = "context"
+ transport = "transport"
+ project = "project"
+ service = "service"
+ host = "host"
+
+ notify_func = base.get_driver(
+ "messaging://", messaging, context, transport,
+ project, service, host).notify
+
+ messaging.Notifier.assert_called_once_with(
+ transport, publisher_id=host, driver="messaging",
+ topics=["profiler"], retry=0)
+
+ info = {
+ "a": 10,
+ "project": project,
+ "service": service,
+ "host": host
+ }
+ notify_func(info)
+
+ messaging.Notifier().info.assert_called_once_with(
+ context, "profiler.service", info)
+
+ messaging.reset_mock()
+ notify_func(info, context="my_context")
+ messaging.Notifier().info.assert_called_once_with(
+ "my_context", "profiler.service", info)
diff --git a/osprofiler/tests/unit/drivers/test_mongodb.py b/osprofiler/tests/unit/drivers/test_mongodb.py
new file mode 100644
index 0000000..2a6b782
--- /dev/null
+++ b/osprofiler/tests/unit/drivers/test_mongodb.py
@@ -0,0 +1,322 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from osprofiler.drivers.mongodb import MongoDB
+from osprofiler.tests import test
+
+
+class MongoDBParserTestCase(test.TestCase):
+ def setUp(self):
+ super(MongoDBParserTestCase, self).setUp()
+ self.mongodb = MongoDB("mongodb://localhost")
+
+ def test_build_empty_tree(self):
+ self.assertEqual([], self.mongodb._build_tree({}))
+
+ def test_build_complex_tree(self):
+ test_input = {
+ "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}},
+ "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}},
+ "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}},
+ "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}},
+ "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}},
+ "113": {"parent_id": "11", "trace_id": "113",
+ "info": {"started": 3}},
+ "112": {"parent_id": "11", "trace_id": "112",
+ "info": {"started": 2}},
+ "114": {"parent_id": "11", "trace_id": "114",
+ "info": {"started": 5}}
+ }
+
+ expected_output = [
+ {
+ "parent_id": "0",
+ "trace_id": "1",
+ "info": {"started": 0},
+ "children": [
+ {
+ "parent_id": "1",
+ "trace_id": "11",
+ "info": {"started": 1},
+ "children": [
+ {"parent_id": "11", "trace_id": "112",
+ "info": {"started": 2}, "children": []},
+ {"parent_id": "11", "trace_id": "113",
+ "info": {"started": 3}, "children": []},
+ {"parent_id": "11", "trace_id": "114",
+ "info": {"started": 5}, "children": []}
+ ]
+ }
+ ]
+ },
+ {
+ "parent_id": "0",
+ "trace_id": "2",
+ "info": {"started": 1},
+ "children": [
+ {"parent_id": "2", "trace_id": "21",
+ "info": {"started": 6}, "children": []},
+ {"parent_id": "2", "trace_id": "22",
+ "info": {"started": 7}, "children": []}
+ ]
+ }
+ ]
+
+ result = self.mongodb._build_tree(test_input)
+ self.assertEqual(expected_output, result)
+
+ def test_get_report_empty(self):
+ self.mongodb.db = mock.MagicMock()
+ self.mongodb.db.profiler.find.return_value = []
+
+ expected = {
+ "info": {
+ "name": "total",
+ "started": 0,
+ "finished": None,
+ "last_trace_started": None
+ },
+ "children": [],
+ "stats": {},
+ }
+
+ base_id = "10"
+ self.assertEqual(expected, self.mongodb.get_report(base_id))
+
+ def test_get_report(self):
+ self.mongodb.db = mock.MagicMock()
+ results = [
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "request": {
+ "path": "/v2/a322b5049d224a90bf8786c644409400/volumes",
+ "scheme": "http",
+ "method": "POST",
+ "query": ""
+ },
+ "service": None
+ },
+ "name": "wsgi-start",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.338776",
+ "trace_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "project": "keystone",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "service": None
+ },
+ "name": "wsgi-stop",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.380405",
+ "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf",
+ "project": "keystone",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "db": {
+ "params": {
+
+ },
+ "statement": "SELECT 1"
+ },
+ "service": None
+ },
+ "name": "db-start",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.395365",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a",
+ "project": "keystone",
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "service": None
+ },
+ "name": "db-stop",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.415486",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a",
+ "project": "keystone",
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "request": {
+ "path": "/v2/a322b5049d224a90bf8786c644409400/volumes",
+ "scheme": "http",
+ "method": "GET",
+ "query": ""
+ },
+ "service": None
+ },
+ "name": "wsgi-start",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.427444",
+ "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b",
+ "project": "keystone",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ }]
+
+ expected = {"children": [{"children": [{
+ "children": [],
+ "info": {"finished": 76,
+ "host": "ubuntu",
+ "meta.raw_payload.db-start": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"db": {"params": {},
+ "statement": "SELECT 1"},
+ "host": "ubuntu",
+ "project": None,
+ "service": None},
+ "name": "db-start",
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.395365",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"},
+ "meta.raw_payload.db-stop": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"host": "ubuntu",
+ "project": None,
+ "service": None},
+ "name": "db-stop",
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.415486",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"},
+ "name": "db",
+ "project": "keystone",
+ "service": "main",
+ "started": 56,
+ "exception": "None"},
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}],
+
+ "info": {"finished": 0,
+ "host": "ubuntu",
+ "meta.raw_payload.wsgi-start": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"host": "ubuntu",
+ "project": None,
+ "request": {"method": "POST",
+ "path": "/v2/a322b5049d224a90bf8"
+ "786c644409400/volumes",
+ "query": "",
+ "scheme": "http"},
+ "service": None},
+ "name": "wsgi-start",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.338776",
+ "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
+ "name": "wsgi",
+ "project": "keystone",
+ "service": "main",
+ "started": 0},
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
+
+ {"children": [],
+ "info": {"finished": 41,
+ "host": "ubuntu",
+ "meta.raw_payload.wsgi-stop": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"host": "ubuntu",
+ "project": None,
+ "service": None},
+ "name": "wsgi-stop",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.380405",
+ "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf"},
+ "name": "wsgi",
+ "project": "keystone",
+ "service": "main",
+ "started": 41,
+ "exception": "None"},
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf"},
+
+ {"children": [],
+ "info": {"finished": 88,
+ "host": "ubuntu",
+ "meta.raw_payload.wsgi-start": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"host": "ubuntu",
+ "project": None,
+ "request": {"method": "GET",
+ "path": "/v2/a322b5049d224a90bf"
+ "8786c644409400/volumes",
+ "query": "",
+ "scheme": "http"},
+ "service": None},
+ "name": "wsgi-start",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.427444",
+ "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"},
+ "name": "wsgi",
+ "project": "keystone",
+ "service": "main",
+ "started": 88},
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}],
+ "info": {
+ "finished": 88,
+ "name": "total",
+ "started": 0,
+ "last_trace_started": 88
+ },
+ "stats": {"db": {"count": 1, "duration": 20},
+ "wsgi": {"count": 3, "duration": 0}}}
+
+ self.mongodb.db.profiler.find.return_value = results
+
+ base_id = "10"
+
+ result = self.mongodb.get_report(base_id)
+
+ expected_filter = [{"base_id": base_id}, {"_id": 0}]
+ self.mongodb.db.profiler.find.assert_called_once_with(
+ *expected_filter)
+ self.assertEqual(expected, result)
diff --git a/osprofiler/tests/unit/drivers/test_redis_driver.py b/osprofiler/tests/unit/drivers/test_redis_driver.py
new file mode 100644
index 0000000..369c733
--- /dev/null
+++ b/osprofiler/tests/unit/drivers/test_redis_driver.py
@@ -0,0 +1,332 @@
+# Copyright 2016 Mirantis Inc.
+# Copyright 2016 IBM Corporation.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from oslo_serialization import jsonutils
+
+from osprofiler.drivers.redis_driver import Redis
+from osprofiler.tests import test
+
+
+class RedisParserTestCase(test.TestCase):
+ def setUp(self):
+ super(RedisParserTestCase, self).setUp()
+ self.redisdb = Redis("redis://localhost:6379")
+
+ def test_build_empty_tree(self):
+ self.assertEqual([], self.redisdb._build_tree({}))
+
+ def test_build_complex_tree(self):
+ test_input = {
+ "2": {"parent_id": "0", "trace_id": "2", "info": {"started": 1}},
+ "1": {"parent_id": "0", "trace_id": "1", "info": {"started": 0}},
+ "21": {"parent_id": "2", "trace_id": "21", "info": {"started": 6}},
+ "22": {"parent_id": "2", "trace_id": "22", "info": {"started": 7}},
+ "11": {"parent_id": "1", "trace_id": "11", "info": {"started": 1}},
+ "113": {"parent_id": "11", "trace_id": "113",
+ "info": {"started": 3}},
+ "112": {"parent_id": "11", "trace_id": "112",
+ "info": {"started": 2}},
+ "114": {"parent_id": "11", "trace_id": "114",
+ "info": {"started": 5}}
+ }
+
+ expected_output = [
+ {
+ "parent_id": "0",
+ "trace_id": "1",
+ "info": {"started": 0},
+ "children": [
+ {
+ "parent_id": "1",
+ "trace_id": "11",
+ "info": {"started": 1},
+ "children": [
+ {"parent_id": "11", "trace_id": "112",
+ "info": {"started": 2}, "children": []},
+ {"parent_id": "11", "trace_id": "113",
+ "info": {"started": 3}, "children": []},
+ {"parent_id": "11", "trace_id": "114",
+ "info": {"started": 5}, "children": []}
+ ]
+ }
+ ]
+ },
+ {
+ "parent_id": "0",
+ "trace_id": "2",
+ "info": {"started": 1},
+ "children": [
+ {"parent_id": "2", "trace_id": "21",
+ "info": {"started": 6}, "children": []},
+ {"parent_id": "2", "trace_id": "22",
+ "info": {"started": 7}, "children": []}
+ ]
+ }
+ ]
+
+ result = self.redisdb._build_tree(test_input)
+ self.assertEqual(expected_output, result)
+
+ def test_get_report_empty(self):
+ self.redisdb.db = mock.MagicMock()
+ self.redisdb.db.scan_iter.return_value = []
+
+ expected = {
+ "info": {
+ "name": "total",
+ "started": 0,
+ "finished": None,
+ "last_trace_started": None
+ },
+ "children": [],
+ "stats": {},
+ }
+
+ base_id = "10"
+ self.assertEqual(expected, self.redisdb.get_report(base_id))
+
+ def test_get_report(self):
+ self.redisdb.db = mock.MagicMock()
+ result_elements = [
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "request": {
+ "path": "/v2/a322b5049d224a90bf8786c644409400/volumes",
+ "scheme": "http",
+ "method": "POST",
+ "query": ""
+ },
+ "service": None
+ },
+ "name": "wsgi-start",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.338776",
+ "trace_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "project": "keystone",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "service": None
+ },
+ "name": "wsgi-stop",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.380405",
+ "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf",
+ "project": "keystone",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "db": {
+ "params": {
+
+ },
+ "statement": "SELECT 1"
+ },
+ "service": None
+ },
+ "name": "db-start",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.395365",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a",
+ "project": "keystone",
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "service": None
+ },
+ "name": "db-stop",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.415486",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a",
+ "project": "keystone",
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ },
+
+ {
+ "info": {
+ "project": None,
+ "host": "ubuntu",
+ "request": {
+ "path": "/v2/a322b5049d224a90bf8786c644409400/volumes",
+ "scheme": "http",
+ "method": "GET",
+ "query": ""
+ },
+ "service": None
+ },
+ "name": "wsgi-start",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.427444",
+ "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b",
+ "project": "keystone",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4"
+ }]
+ results = {result["base_id"] + "_" + result["trace_id"] +
+ "_" + result["timestamp"]: result
+ for result in result_elements}
+
+ expected = {"children": [{"children": [{
+ "children": [],
+ "info": {"finished": 76,
+ "host": "ubuntu",
+ "meta.raw_payload.db-start": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"db": {"params": {},
+ "statement": "SELECT 1"},
+ "host": "ubuntu",
+ "project": None,
+ "service": None},
+ "name": "db-start",
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.395365",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"},
+ "meta.raw_payload.db-stop": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"host": "ubuntu",
+ "project": None,
+ "service": None},
+ "name": "db-stop",
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.415486",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"},
+ "name": "db",
+ "project": "keystone",
+ "service": "main",
+ "started": 56,
+ "exception": "None"},
+ "parent_id": "06320327-2c2c-45ae-923a-515de890276a",
+ "trace_id": "1baf1d24-9ca9-4f4c-bd3f-01b7e0c0735a"}],
+
+ "info": {"finished": 0,
+ "host": "ubuntu",
+ "meta.raw_payload.wsgi-start": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"host": "ubuntu",
+ "project": None,
+ "request": {"method": "POST",
+ "path": "/v2/a322b5049d224a90bf8"
+ "786c644409400/volumes",
+ "query": "",
+ "scheme": "http"},
+ "service": None},
+ "name": "wsgi-start",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.338776",
+ "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
+ "name": "wsgi",
+ "project": "keystone",
+ "service": "main",
+ "started": 0},
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "trace_id": "06320327-2c2c-45ae-923a-515de890276a"},
+
+ {"children": [],
+ "info": {"finished": 41,
+ "host": "ubuntu",
+ "meta.raw_payload.wsgi-stop": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"host": "ubuntu",
+ "project": None,
+ "service": None},
+ "name": "wsgi-stop",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.380405",
+ "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf"},
+ "name": "wsgi",
+ "project": "keystone",
+ "service": "main",
+ "started": 41,
+ "exception": "None"},
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "trace_id": "839ca3f1-afcb-45be-a4a1-679124c552bf"},
+
+ {"children": [],
+ "info": {"finished": 88,
+ "host": "ubuntu",
+ "meta.raw_payload.wsgi-start": {
+ "base_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "info": {"host": "ubuntu",
+ "project": None,
+ "request": {"method": "GET",
+ "path": "/v2/a322b5049d224a90bf"
+ "8786c644409400/volumes",
+ "query": "",
+ "scheme": "http"},
+ "service": None},
+ "name": "wsgi-start",
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "project": "keystone",
+ "service": "main",
+ "timestamp": "2015-12-23T14:02:22.427444",
+ "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"},
+ "name": "wsgi",
+ "project": "keystone",
+ "service": "main",
+ "started": 88},
+ "parent_id": "7253ca8c-33b3-4f84-b4f1-f5a4311ddfa4",
+ "trace_id": "016c97fd-87f3-40b2-9b55-e431156b694b"}],
+ "info": {
+ "finished": 88,
+ "name": "total",
+ "started": 0,
+ "last_trace_started": 88
+ },
+ "stats": {"db": {"count": 1, "duration": 20},
+ "wsgi": {"count": 3, "duration": 0}}}
+
+ self.redisdb.db.scan_iter.return_value = list(results.keys())
+
+ def side_effect(*args, **kwargs):
+ return jsonutils.dumps(results[args[0]])
+
+ self.redisdb.db.get.side_effect = side_effect
+
+ base_id = "10"
+
+ result = self.redisdb.get_report(base_id)
+
+ expected_filter = self.redisdb.namespace + "10*"
+ self.redisdb.db.scan_iter.assert_called_once_with(
+ match=expected_filter)
+ self.assertEqual(expected, result)
diff --git a/osprofiler/tests/unit/test_notifier.py b/osprofiler/tests/unit/test_notifier.py
new file mode 100644
index 0000000..332e620
--- /dev/null
+++ b/osprofiler/tests/unit/test_notifier.py
@@ -0,0 +1,51 @@
+# Copyright 2014 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from osprofiler import notifier
+from osprofiler.tests import test
+
+
+class NotifierTestCase(test.TestCase):
+
+ def tearDown(self):
+ notifier.__notifier = notifier._noop_notifier
+ super(NotifierTestCase, self).tearDown()
+
+ def test_set(self):
+
+ def test(info):
+ pass
+
+ notifier.set(test)
+ self.assertEqual(notifier.get(), test)
+
+ def test_get_default_notifier(self):
+ self.assertEqual(notifier.get(), notifier._noop_notifier)
+
+ def test_notify(self):
+ m = mock.MagicMock()
+ notifier.set(m)
+ notifier.notify(10)
+
+ m.assert_called_once_with(10)
+
+ @mock.patch("osprofiler.notifier.base.get_driver")
+ def test_create(self, mock_factory):
+
+ result = notifier.create("test", 10, b=20)
+ mock_factory.assert_called_once_with("test", 10, b=20)
+ self.assertEqual(mock_factory.return_value.notify, result)
diff --git a/osprofiler/tests/unit/test_opts.py b/osprofiler/tests/unit/test_opts.py
new file mode 100644
index 0000000..c5963b9
--- /dev/null
+++ b/osprofiler/tests/unit/test_opts.py
@@ -0,0 +1,65 @@
+# Copyright 2016 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from oslo_config import fixture
+
+from osprofiler import opts
+from osprofiler.tests import test
+
+
+class ConfigTestCase(test.TestCase):
+ def setUp(self):
+ super(ConfigTestCase, self).setUp()
+ self.conf_fixture = self.useFixture(fixture.Config())
+
+ def test_options_defaults(self):
+ opts.set_defaults(self.conf_fixture.conf)
+ self.assertFalse(self.conf_fixture.conf.profiler.enabled)
+ self.assertFalse(self.conf_fixture.conf.profiler.trace_sqlalchemy)
+ self.assertEqual("SECRET_KEY",
+ self.conf_fixture.conf.profiler.hmac_keys)
+ self.assertFalse(opts.is_trace_enabled(self.conf_fixture.conf))
+ self.assertFalse(opts.is_db_trace_enabled(self.conf_fixture.conf))
+
+ def test_options_defaults_override(self):
+ opts.set_defaults(self.conf_fixture.conf, enabled=True,
+ trace_sqlalchemy=True,
+ hmac_keys="MY_KEY")
+ self.assertTrue(self.conf_fixture.conf.profiler.enabled)
+ self.assertTrue(self.conf_fixture.conf.profiler.trace_sqlalchemy)
+ self.assertEqual("MY_KEY",
+ self.conf_fixture.conf.profiler.hmac_keys)
+ self.assertTrue(opts.is_trace_enabled(self.conf_fixture.conf))
+ self.assertTrue(opts.is_db_trace_enabled(self.conf_fixture.conf))
+
+ @mock.patch("osprofiler.web.enable")
+ @mock.patch("osprofiler.web.disable")
+ def test_web_trace_disabled(self, mock_disable, mock_enable):
+ opts.set_defaults(self.conf_fixture.conf, hmac_keys="MY_KEY")
+ opts.enable_web_trace(self.conf_fixture.conf)
+ opts.disable_web_trace(self.conf_fixture.conf)
+ self.assertEqual(0, mock_enable.call_count)
+ self.assertEqual(0, mock_disable.call_count)
+
+ @mock.patch("osprofiler.web.enable")
+ @mock.patch("osprofiler.web.disable")
+ def test_web_trace_enabled(self, mock_disable, mock_enable):
+ opts.set_defaults(self.conf_fixture.conf, enabled=True,
+ hmac_keys="MY_KEY")
+ opts.enable_web_trace(self.conf_fixture.conf)
+ opts.disable_web_trace(self.conf_fixture.conf)
+ mock_enable.assert_called_once_with("MY_KEY")
+ mock_disable.assert_called_once_with()
diff --git a/osprofiler/tests/unit/test_profiler.py b/osprofiler/tests/unit/test_profiler.py
new file mode 100644
index 0000000..6edd467
--- /dev/null
+++ b/osprofiler/tests/unit/test_profiler.py
@@ -0,0 +1,562 @@
+# Copyright 2014 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import collections
+import copy
+import datetime
+import re
+
+import mock
+import six
+
+from osprofiler import profiler
+from osprofiler.tests import test
+
+
+class ProfilerGlobMethodsTestCase(test.TestCase):
+
+ def test_get_profiler_not_inited(self):
+ profiler._clean()
+ self.assertIsNone(profiler.get())
+
+ def test_get_profiler_and_init(self):
+ p = profiler.init("secret", base_id="1", parent_id="2")
+ self.assertEqual(profiler.get(), p)
+
+ self.assertEqual(p.get_base_id(), "1")
+ # NOTE(boris-42): until we make first start we don't have
+ self.assertEqual(p.get_id(), "2")
+
+ def test_start_not_inited(self):
+ profiler._clean()
+ profiler.start("name")
+
+ def test_start(self):
+ p = profiler.init("secret", base_id="1", parent_id="2")
+ p.start = mock.MagicMock()
+ profiler.start("name", info="info")
+ p.start.assert_called_once_with("name", info="info")
+
+ def test_stop_not_inited(self):
+ profiler._clean()
+ profiler.stop()
+
+ def test_stop(self):
+ p = profiler.init("secret", base_id="1", parent_id="2")
+ p.stop = mock.MagicMock()
+ profiler.stop(info="info")
+ p.stop.assert_called_once_with(info="info")
+
+
+class ProfilerTestCase(test.TestCase):
+
+ def test_profiler_get_base_id(self):
+ prof = profiler._Profiler("secret", base_id="1", parent_id="2")
+ self.assertEqual(prof.get_base_id(), "1")
+
+ @mock.patch("osprofiler.profiler.uuidutils.generate_uuid")
+ def test_profiler_get_parent_id(self, mock_generate_uuid):
+ mock_generate_uuid.return_value = "42"
+ prof = profiler._Profiler("secret", base_id="1", parent_id="2")
+ prof.start("test")
+ self.assertEqual(prof.get_parent_id(), "2")
+
+ @mock.patch("osprofiler.profiler.uuidutils.generate_uuid")
+ def test_profiler_get_base_id_unset_case(self, mock_generate_uuid):
+ mock_generate_uuid.return_value = "42"
+ prof = profiler._Profiler("secret")
+ self.assertEqual(prof.get_base_id(), "42")
+ self.assertEqual(prof.get_parent_id(), "42")
+
+ @mock.patch("osprofiler.profiler.uuidutils.generate_uuid")
+ def test_profiler_get_id(self, mock_generate_uuid):
+ mock_generate_uuid.return_value = "43"
+ prof = profiler._Profiler("secret")
+ prof.start("test")
+ self.assertEqual(prof.get_id(), "43")
+
+ @mock.patch("osprofiler.profiler.datetime")
+ @mock.patch("osprofiler.profiler.uuidutils.generate_uuid")
+ @mock.patch("osprofiler.profiler.notifier.notify")
+ def test_profiler_start(self, mock_notify, mock_generate_uuid,
+ mock_datetime):
+ mock_generate_uuid.return_value = "44"
+ now = datetime.datetime.utcnow()
+ mock_datetime.datetime.utcnow.return_value = now
+
+ info = {"some": "info"}
+ payload = {
+ "name": "test-start",
+ "base_id": "1",
+ "parent_id": "2",
+ "trace_id": "44",
+ "info": info,
+ "timestamp": now.strftime("%Y-%m-%dT%H:%M:%S.%f"),
+ }
+
+ prof = profiler._Profiler("secret", base_id="1", parent_id="2")
+ prof.start("test", info=info)
+
+ mock_notify.assert_called_once_with(payload)
+
+ @mock.patch("osprofiler.profiler.datetime")
+ @mock.patch("osprofiler.profiler.notifier.notify")
+ def test_profiler_stop(self, mock_notify, mock_datetime):
+ now = datetime.datetime.utcnow()
+ mock_datetime.datetime.utcnow.return_value = now
+ prof = profiler._Profiler("secret", base_id="1", parent_id="2")
+ prof._trace_stack.append("44")
+ prof._name.append("abc")
+
+ info = {"some": "info"}
+ prof.stop(info=info)
+
+ payload = {
+ "name": "abc-stop",
+ "base_id": "1",
+ "parent_id": "2",
+ "trace_id": "44",
+ "info": info,
+ "timestamp": now.strftime("%Y-%m-%dT%H:%M:%S.%f"),
+ }
+
+ mock_notify.assert_called_once_with(payload)
+ self.assertEqual(len(prof._name), 0)
+ self.assertEqual(prof._trace_stack, collections.deque(["1", "2"]))
+
+ def test_profiler_hmac(self):
+ hmac = "secret"
+ prof = profiler._Profiler(hmac, base_id="1", parent_id="2")
+ self.assertEqual(hmac, prof.hmac_key)
+
+
+class WithTraceTestCase(test.TestCase):
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_with_trace(self, mock_start, mock_stop):
+
+ with profiler.Trace("a", info="a1"):
+ mock_start.assert_called_once_with("a", info="a1")
+ mock_start.reset_mock()
+ with profiler.Trace("b", info="b1"):
+ mock_start.assert_called_once_with("b", info="b1")
+ mock_stop.assert_called_once_with()
+ mock_stop.reset_mock()
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_with_trace_etype(self, mock_start, mock_stop):
+
+ def foo():
+ with profiler.Trace("foo"):
+ raise ValueError("bar")
+
+ self.assertRaises(ValueError, foo)
+ mock_start.assert_called_once_with("foo", info=None)
+ mock_stop.assert_called_once_with(info={"etype": "ValueError"})
+
+
+@profiler.trace("function", info={"info": "some_info"})
+def tracede_func(i):
+ return i
+
+
+@profiler.trace("hide_args", hide_args=True)
+def trace_hide_args_func(a, i=10):
+ return (a, i)
+
+
+class TraceDecoratorTestCase(test.TestCase):
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_duplicate_trace_disallow(self, mock_start, mock_stop):
+
+ @profiler.trace("test")
+ def trace_me():
+ pass
+
+ self.assertRaises(
+ ValueError,
+ profiler.trace("test-again", allow_multiple_trace=False),
+ trace_me)
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_with_args(self, mock_start, mock_stop):
+ self.assertEqual(1, tracede_func(1))
+ expected_info = {
+ "info": "some_info",
+ "function": {
+ "name": "osprofiler.tests.unit.test_profiler.tracede_func",
+ "args": str((1,)),
+ "kwargs": str({})
+ }
+ }
+ mock_start.assert_called_once_with("function", info=expected_info)
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_without_args(self, mock_start, mock_stop):
+ self.assertEqual((1, 2), trace_hide_args_func(1, i=2))
+ expected_info = {
+ "function": {
+ "name": "osprofiler.tests.unit.test_profiler"
+ ".trace_hide_args_func"
+ }
+ }
+ mock_start.assert_called_once_with("hide_args", info=expected_info)
+ mock_stop.assert_called_once_with()
+
+
+class FakeTracedCls(object):
+
+ def method1(self, a, b, c=10):
+ return a + b + c
+
+ def method2(self, d, e):
+ return d - e
+
+ def method3(self, g=10, h=20):
+ return g * h
+
+ def _method(self, i):
+ return i
+
+
+@profiler.trace_cls("rpc", info={"a": 10})
+class FakeTraceClassWithInfo(FakeTracedCls):
+ pass
+
+
+@profiler.trace_cls("a", info={"b": 20}, hide_args=True)
+class FakeTraceClassHideArgs(FakeTracedCls):
+ pass
+
+
+@profiler.trace_cls("rpc", trace_private=True)
+class FakeTracePrivate(FakeTracedCls):
+ pass
+
+
+class FakeTraceStaticMethodBase(FakeTracedCls):
+ @staticmethod
+ def static_method(arg):
+ return arg
+
+
+@profiler.trace_cls("rpc", trace_static_methods=True)
+class FakeTraceStaticMethod(FakeTraceStaticMethodBase):
+ pass
+
+
+@profiler.trace_cls("rpc")
+class FakeTraceStaticMethodSkip(FakeTraceStaticMethodBase):
+ pass
+
+
+class FakeTraceClassMethodBase(FakeTracedCls):
+ @classmethod
+ def class_method(cls, arg):
+ return arg
+
+
+@profiler.trace_cls("rpc")
+class FakeTraceClassMethodSkip(FakeTraceClassMethodBase):
+ pass
+
+
+def py3_info(info):
+ # NOTE(boris-42): py33 I hate you.
+ info_py3 = copy.deepcopy(info)
+ new_name = re.sub("FakeTrace[^.]*", "FakeTracedCls",
+ info_py3["function"]["name"])
+ info_py3["function"]["name"] = new_name
+ return info_py3
+
+
+def possible_mock_calls(name, info):
+ # NOTE(boris-42): py33 I hate you.
+ return [mock.call(name, info=info), mock.call(name, info=py3_info(info))]
+
+
+class TraceClsDecoratorTestCase(test.TestCase):
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_args(self, mock_start, mock_stop):
+ fake_cls = FakeTraceClassWithInfo()
+ self.assertEqual(30, fake_cls.method1(5, 15))
+ expected_info = {
+ "a": 10,
+ "function": {
+ "name": ("osprofiler.tests.unit.test_profiler"
+ ".FakeTraceClassWithInfo.method1"),
+ "args": str((fake_cls, 5, 15)),
+ "kwargs": str({})
+ }
+ }
+ self.assertEqual(1, len(mock_start.call_args_list))
+ self.assertIn(mock_start.call_args_list[0],
+ possible_mock_calls("rpc", expected_info))
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_kwargs(self, mock_start, mock_stop):
+ fake_cls = FakeTraceClassWithInfo()
+ self.assertEqual(50, fake_cls.method3(g=5, h=10))
+ expected_info = {
+ "a": 10,
+ "function": {
+ "name": ("osprofiler.tests.unit.test_profiler"
+ ".FakeTraceClassWithInfo.method3"),
+ "args": str((fake_cls,)),
+ "kwargs": str({"g": 5, "h": 10})
+ }
+ }
+ self.assertEqual(1, len(mock_start.call_args_list))
+ self.assertIn(mock_start.call_args_list[0],
+ possible_mock_calls("rpc", expected_info))
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_without_private(self, mock_start, mock_stop):
+ fake_cls = FakeTraceClassHideArgs()
+ self.assertEqual(10, fake_cls._method(10))
+ self.assertFalse(mock_start.called)
+ self.assertFalse(mock_stop.called)
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_without_args(self, mock_start, mock_stop):
+ fake_cls = FakeTraceClassHideArgs()
+ self.assertEqual(40, fake_cls.method1(5, 15, c=20))
+ expected_info = {
+ "b": 20,
+ "function": {
+ "name": ("osprofiler.tests.unit.test_profiler"
+ ".FakeTraceClassHideArgs.method1"),
+ }
+ }
+
+ self.assertEqual(1, len(mock_start.call_args_list))
+ self.assertIn(mock_start.call_args_list[0],
+ possible_mock_calls("a", expected_info))
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_private_methods(self, mock_start, mock_stop):
+ fake_cls = FakeTracePrivate()
+ self.assertEqual(5, fake_cls._method(5))
+
+ expected_info = {
+ "function": {
+ "name": ("osprofiler.tests.unit.test_profiler"
+ ".FakeTracePrivate._method"),
+ "args": str((fake_cls, 5)),
+ "kwargs": str({})
+ }
+ }
+
+ self.assertEqual(1, len(mock_start.call_args_list))
+ self.assertIn(mock_start.call_args_list[0],
+ possible_mock_calls("rpc", expected_info))
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ @test.testcase.skip(
+ "Static method tracing was disabled due the bug. This test should be "
+ "skipped until we find the way to address it.")
+ def test_static(self, mock_start, mock_stop):
+ fake_cls = FakeTraceStaticMethod()
+
+ self.assertEqual(25, fake_cls.static_method(25))
+
+ expected_info = {
+ "function": {
+ # fixme(boris-42): Static methods are treated differently in
+ # Python 2.x and Python 3.x. So in PY2 we
+ # expect to see method4 because method is
+ # static and doesn't have reference to class
+ # - and FakeTraceStatic.method4 in PY3
+ "name":
+ "osprofiler.tests.unit.test_profiler"
+ ".method4" if six.PY2 else
+ "osprofiler.tests.unit.test_profiler.FakeTraceStatic"
+ ".method4",
+ "args": str((25,)),
+ "kwargs": str({})
+ }
+ }
+
+ self.assertEqual(1, len(mock_start.call_args_list))
+ self.assertIn(mock_start.call_args_list[0],
+ possible_mock_calls("rpc", expected_info))
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_static_method_skip(self, mock_start, mock_stop):
+ self.assertEqual(25, FakeTraceStaticMethodSkip.static_method(25))
+ self.assertFalse(mock_start.called)
+ self.assertFalse(mock_stop.called)
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_class_method_skip(self, mock_start, mock_stop):
+ self.assertEqual("foo", FakeTraceClassMethodSkip.class_method("foo"))
+ self.assertFalse(mock_start.called)
+ self.assertFalse(mock_stop.called)
+
+
+@six.add_metaclass(profiler.TracedMeta)
+class FakeTraceWithMetaclassBase(object):
+ __trace_args__ = {"name": "rpc",
+ "info": {"a": 10}}
+
+ def method1(self, a, b, c=10):
+ return a + b + c
+
+ def method2(self, d, e):
+ return d - e
+
+ def method3(self, g=10, h=20):
+ return g * h
+
+ def _method(self, i):
+ return i
+
+
+class FakeTraceDummy(FakeTraceWithMetaclassBase):
+ def method4(self, j):
+ return j
+
+
+class FakeTraceWithMetaclassHideArgs(FakeTraceWithMetaclassBase):
+ __trace_args__ = {"name": "a",
+ "info": {"b": 20},
+ "hide_args": True}
+
+ def method5(self, k, l):
+ return k + l
+
+
+class FakeTraceWithMetaclassPrivate(FakeTraceWithMetaclassBase):
+ __trace_args__ = {"name": "rpc",
+ "trace_private": True}
+
+ def _new_private_method(self, m):
+ return 2 * m
+
+
+class TraceWithMetaclassTestCase(test.TestCase):
+
+ def test_no_name_exception(self):
+ def define_class_with_no_name():
+ @six.add_metaclass(profiler.TracedMeta)
+ class FakeTraceWithMetaclassNoName(FakeTracedCls):
+ pass
+ self.assertRaises(TypeError, define_class_with_no_name, 1)
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_args(self, mock_start, mock_stop):
+ fake_cls = FakeTraceWithMetaclassBase()
+ self.assertEqual(30, fake_cls.method1(5, 15))
+ expected_info = {
+ "a": 10,
+ "function": {
+ "name": ("osprofiler.tests.unit.test_profiler"
+ ".FakeTraceWithMetaclassBase.method1"),
+ "args": str((fake_cls, 5, 15)),
+ "kwargs": str({})
+ }
+ }
+ self.assertEqual(1, len(mock_start.call_args_list))
+ self.assertIn(mock_start.call_args_list[0],
+ possible_mock_calls("rpc", expected_info))
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_kwargs(self, mock_start, mock_stop):
+ fake_cls = FakeTraceWithMetaclassBase()
+ self.assertEqual(50, fake_cls.method3(g=5, h=10))
+ expected_info = {
+ "a": 10,
+ "function": {
+ "name": ("osprofiler.tests.unit.test_profiler"
+ ".FakeTraceWithMetaclassBase.method3"),
+ "args": str((fake_cls,)),
+ "kwargs": str({"g": 5, "h": 10})
+ }
+ }
+ self.assertEqual(1, len(mock_start.call_args_list))
+ self.assertIn(mock_start.call_args_list[0],
+ possible_mock_calls("rpc", expected_info))
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_without_private(self, mock_start, mock_stop):
+ fake_cls = FakeTraceWithMetaclassHideArgs()
+ self.assertEqual(10, fake_cls._method(10))
+ self.assertFalse(mock_start.called)
+ self.assertFalse(mock_stop.called)
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_without_args(self, mock_start, mock_stop):
+ fake_cls = FakeTraceWithMetaclassHideArgs()
+ self.assertEqual(20, fake_cls.method5(5, 15))
+ expected_info = {
+ "b": 20,
+ "function": {
+ "name": ("osprofiler.tests.unit.test_profiler"
+ ".FakeTraceWithMetaclassHideArgs.method5")
+ }
+ }
+
+ self.assertEqual(1, len(mock_start.call_args_list))
+ self.assertIn(mock_start.call_args_list[0],
+ possible_mock_calls("a", expected_info))
+ mock_stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.profiler.stop")
+ @mock.patch("osprofiler.profiler.start")
+ def test_private_methods(self, mock_start, mock_stop):
+ fake_cls = FakeTraceWithMetaclassPrivate()
+ self.assertEqual(10, fake_cls._new_private_method(5))
+
+ expected_info = {
+ "function": {
+ "name": ("osprofiler.tests.unit.test_profiler"
+ ".FakeTraceWithMetaclassPrivate._new_private_method"),
+ "args": str((fake_cls, 5)),
+ "kwargs": str({})
+ }
+ }
+
+ self.assertEqual(1, len(mock_start.call_args_list))
+ self.assertIn(mock_start.call_args_list[0],
+ possible_mock_calls("rpc", expected_info))
+ mock_stop.assert_called_once_with()
diff --git a/osprofiler/tests/unit/test_sqlalchemy.py b/osprofiler/tests/unit/test_sqlalchemy.py
new file mode 100644
index 0000000..7534511
--- /dev/null
+++ b/osprofiler/tests/unit/test_sqlalchemy.py
@@ -0,0 +1,103 @@
+# Copyright 2014 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import contextlib
+
+import mock
+
+from osprofiler import sqlalchemy
+from osprofiler.tests import test
+
+
+class SqlalchemyTracingTestCase(test.TestCase):
+
+ @mock.patch("osprofiler.sqlalchemy.profiler")
+ def test_before_execute(self, mock_profiler):
+ handler = sqlalchemy._before_cursor_execute("sql")
+
+ handler(mock.MagicMock(), 1, 2, 3, 4, 5)
+ expected_info = {"db": {"statement": 2, "params": 3}}
+ mock_profiler.start.assert_called_once_with("sql", info=expected_info)
+
+ @mock.patch("osprofiler.sqlalchemy.profiler")
+ def test_after_execute(self, mock_profiler):
+ handler = sqlalchemy._after_cursor_execute()
+ handler(mock.MagicMock(), 1, 2, 3, 4, 5)
+ mock_profiler.stop.assert_called_once_with()
+
+ @mock.patch("osprofiler.sqlalchemy._before_cursor_execute")
+ @mock.patch("osprofiler.sqlalchemy._after_cursor_execute")
+ def test_add_tracing(self, mock_after_exc, mock_before_exc):
+ sa = mock.MagicMock()
+ engine = mock.MagicMock()
+
+ mock_before_exc.return_value = "before"
+ mock_after_exc.return_value = "after"
+
+ sqlalchemy.add_tracing(sa, engine, "sql")
+
+ mock_before_exc.assert_called_once_with("sql")
+ mock_after_exc.assert_called_once_with()
+ expected_calls = [
+ mock.call(engine, "before_cursor_execute", "before"),
+ mock.call(engine, "after_cursor_execute", "after")
+ ]
+ self.assertEqual(sa.event.listen.call_args_list, expected_calls)
+
+ @mock.patch("osprofiler.sqlalchemy._before_cursor_execute")
+ @mock.patch("osprofiler.sqlalchemy._after_cursor_execute")
+ def test_wrap_session(self, mock_after_exc, mock_before_exc):
+ sa = mock.MagicMock()
+
+ @contextlib.contextmanager
+ def _session():
+ session = mock.MagicMock()
+ # current engine object stored within the session
+ session.bind = mock.MagicMock()
+ session.bind.traced = None
+ yield session
+
+ mock_before_exc.return_value = "before"
+ mock_after_exc.return_value = "after"
+
+ session = sqlalchemy.wrap_session(sa, _session())
+
+ with session as sess:
+ pass
+
+ mock_before_exc.assert_called_once_with("db")
+ mock_after_exc.assert_called_once_with()
+ expected_calls = [
+ mock.call(sess.bind, "before_cursor_execute", "before"),
+ mock.call(sess.bind, "after_cursor_execute", "after")
+ ]
+
+ self.assertEqual(sa.event.listen.call_args_list, expected_calls)
+
+ @mock.patch("osprofiler.sqlalchemy._before_cursor_execute")
+ @mock.patch("osprofiler.sqlalchemy._after_cursor_execute")
+ def test_disable_and_enable(self, mock_after_exc, mock_before_exc):
+ sqlalchemy.disable()
+
+ sa = mock.MagicMock()
+ engine = mock.MagicMock()
+ sqlalchemy.add_tracing(sa, engine, "sql")
+ self.assertFalse(mock_after_exc.called)
+ self.assertFalse(mock_before_exc.called)
+
+ sqlalchemy.enable()
+ sqlalchemy.add_tracing(sa, engine, "sql")
+ self.assertTrue(mock_after_exc.called)
+ self.assertTrue(mock_before_exc.called)
diff --git a/osprofiler/tests/unit/test_utils.py b/osprofiler/tests/unit/test_utils.py
new file mode 100644
index 0000000..19aff30
--- /dev/null
+++ b/osprofiler/tests/unit/test_utils.py
@@ -0,0 +1,133 @@
+# Copyright 2014 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import base64
+import hashlib
+import hmac
+
+import mock
+
+from osprofiler import _utils as utils
+from osprofiler.tests import test
+
+
+class UtilsTestCase(test.TestCase):
+
+ def test_split(self):
+ self.assertEqual([1, 2], utils.split([1, 2]))
+ self.assertEqual(["A", "B"], utils.split("A, B"))
+ self.assertEqual(["A", " B"], utils.split("A, B", strip=False))
+
+ def test_split_wrong_type(self):
+ self.assertRaises(TypeError, utils.split, 1)
+
+ def test_binary_encode_and_decode(self):
+ self.assertEqual("text",
+ utils.binary_decode(utils.binary_encode("text")))
+
+ def test_binary_encode_invalid_type(self):
+ self.assertRaises(TypeError, utils.binary_encode, 1234)
+
+ def test_binary_encode_binary_type(self):
+ binary = utils.binary_encode("text")
+ self.assertEqual(binary, utils.binary_encode(binary))
+
+ def test_binary_decode_invalid_type(self):
+ self.assertRaises(TypeError, utils.binary_decode, 1234)
+
+ def test_binary_decode_text_type(self):
+ self.assertEqual("text", utils.binary_decode("text"))
+
+ def test_generate_hmac(self):
+ hmac_key = "secrete"
+ data = "my data"
+
+ h = hmac.new(utils.binary_encode(hmac_key), digestmod=hashlib.sha1)
+ h.update(utils.binary_encode(data))
+
+ self.assertEqual(h.hexdigest(), utils.generate_hmac(data, hmac_key))
+
+ def test_signed_pack_unpack(self):
+ hmac = "secret"
+ data = {"some": "data"}
+
+ packed_data, hmac_data = utils.signed_pack(data, hmac)
+
+ process_data = utils.signed_unpack(packed_data, hmac_data, [hmac])
+ self.assertIn("hmac_key", process_data)
+ process_data.pop("hmac_key")
+ self.assertEqual(data, process_data)
+
+ def test_signed_pack_unpack_many_keys(self):
+ keys = ["secret", "secret2", "secret3"]
+ data = {"some": "data"}
+ packed_data, hmac_data = utils.signed_pack(data, keys[-1])
+
+ process_data = utils.signed_unpack(packed_data, hmac_data, keys)
+ self.assertEqual(keys[-1], process_data["hmac_key"])
+
+ def test_signed_pack_unpack_many_wrong_keys(self):
+ keys = ["secret", "secret2", "secret3"]
+ data = {"some": "data"}
+ packed_data, hmac_data = utils.signed_pack(data, "password")
+
+ process_data = utils.signed_unpack(packed_data, hmac_data, keys)
+ self.assertIsNone(process_data)
+
+ def test_signed_unpack_wrong_key(self):
+ data = {"some": "data"}
+ packed_data, hmac_data = utils.signed_pack(data, "secret")
+
+ self.assertIsNone(utils.signed_unpack(packed_data, hmac_data, "wrong"))
+
+ def test_signed_unpack_no_key_or_hmac_data(self):
+ data = {"some": "data"}
+ packed_data, hmac_data = utils.signed_pack(data, "secret")
+ self.assertIsNone(utils.signed_unpack(packed_data, hmac_data, None))
+ self.assertIsNone(utils.signed_unpack(packed_data, None, "secret"))
+ self.assertIsNone(utils.signed_unpack(packed_data, " ", "secret"))
+
+ @mock.patch("osprofiler._utils.generate_hmac")
+ def test_singed_unpack_generate_hmac_failed(self, mock_generate_hmac):
+ mock_generate_hmac.side_effect = Exception
+ self.assertIsNone(utils.signed_unpack("data", "hmac_data", "hmac_key"))
+
+ def test_signed_unpack_invalid_json(self):
+ hmac = "secret"
+ data = base64.urlsafe_b64encode(utils.binary_encode("not_a_json"))
+ hmac_data = utils.generate_hmac(data, hmac)
+
+ self.assertIsNone(utils.signed_unpack(data, hmac_data, hmac))
+
+ def test_itersubclasses(self):
+
+ class A(object):
+ pass
+
+ class B(A):
+ pass
+
+ class C(A):
+ pass
+
+ class D(C):
+ pass
+
+ self.assertEqual([B, C, D], list(utils.itersubclasses(A)))
+
+ class E(type):
+ pass
+
+ self.assertEqual([], list(utils.itersubclasses(E)))
diff --git a/osprofiler/tests/unit/test_web.py b/osprofiler/tests/unit/test_web.py
new file mode 100644
index 0000000..24b3906
--- /dev/null
+++ b/osprofiler/tests/unit/test_web.py
@@ -0,0 +1,307 @@
+# Copyright 2014 Mirantis Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from webob import response as webob_response
+
+from osprofiler import _utils as utils
+from osprofiler import profiler
+from osprofiler.tests import test
+from osprofiler import web
+
+
+def dummy_app(environ, response):
+ res = webob_response.Response()
+ return res(environ, response)
+
+
+class WebTestCase(test.TestCase):
+
+ def setUp(self):
+ super(WebTestCase, self).setUp()
+ profiler._clean()
+ self.addCleanup(profiler._clean)
+
+ def test_get_trace_id_headers_no_hmac(self):
+ profiler.init(None, base_id="y", parent_id="z")
+ headers = web.get_trace_id_headers()
+ self.assertEqual(headers, {})
+
+ def test_get_trace_id_headers(self):
+ profiler.init("key", base_id="y", parent_id="z")
+ headers = web.get_trace_id_headers()
+ self.assertEqual(sorted(headers.keys()),
+ sorted(["X-Trace-Info", "X-Trace-HMAC"]))
+
+ trace_info = utils.signed_unpack(headers["X-Trace-Info"],
+ headers["X-Trace-HMAC"], ["key"])
+ self.assertIn("hmac_key", trace_info)
+ self.assertEqual("key", trace_info.pop("hmac_key"))
+ self.assertEqual({"parent_id": "z", "base_id": "y"}, trace_info)
+
+ @mock.patch("osprofiler.profiler.get")
+ def test_get_trace_id_headers_no_profiler(self, mock_get_profiler):
+ mock_get_profiler.return_value = False
+ headers = web.get_trace_id_headers()
+ self.assertEqual(headers, {})
+
+
+class WebMiddlewareTestCase(test.TestCase):
+ def setUp(self):
+ super(WebMiddlewareTestCase, self).setUp()
+ profiler._clean()
+ # it's default state of _ENABLED param, so let's set it here
+ web._ENABLED = None
+ self.addCleanup(profiler._clean)
+
+ def tearDown(self):
+ web.enable()
+ super(WebMiddlewareTestCase, self).tearDown()
+
+ def test_factory(self):
+ mock_app = mock.MagicMock()
+ local_conf = {"enabled": True, "hmac_keys": "123"}
+
+ factory = web.WsgiMiddleware.factory(None, **local_conf)
+ wsgi = factory(mock_app)
+
+ self.assertEqual(wsgi.application, mock_app)
+ self.assertEqual(wsgi.name, "wsgi")
+ self.assertTrue(wsgi.enabled)
+ self.assertEqual(wsgi.hmac_keys, [local_conf["hmac_keys"]])
+
+ def _test_wsgi_middleware_with_invalid_trace(self, headers, hmac_key,
+ mock_profiler_init,
+ enabled=True):
+ request = mock.MagicMock()
+ request.get_response.return_value = "yeah!"
+ request.headers = headers
+
+ middleware = web.WsgiMiddleware("app", hmac_key, enabled=enabled)
+ self.assertEqual("yeah!", middleware(request))
+ request.get_response.assert_called_once_with("app")
+ self.assertEqual(0, mock_profiler_init.call_count)
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_disabled(self, mock_profiler_init):
+ hmac_key = "secret"
+ pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
+ headers = {
+ "a": "1",
+ "b": "2",
+ "X-Trace-Info": pack[0],
+ "X-Trace-HMAC": pack[1]
+ }
+
+ self._test_wsgi_middleware_with_invalid_trace(headers, hmac_key,
+ mock_profiler_init,
+ enabled=False)
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_no_trace(self, mock_profiler_init):
+ headers = {
+ "a": "1",
+ "b": "2"
+ }
+ self._test_wsgi_middleware_with_invalid_trace(headers, "secret",
+ mock_profiler_init)
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_invalid_trace_headers(self, mock_profiler_init):
+ headers = {
+ "a": "1",
+ "b": "2",
+ "X-Trace-Info": "abbababababa",
+ "X-Trace-HMAC": "abbababababa"
+ }
+ self._test_wsgi_middleware_with_invalid_trace(headers, "secret",
+ mock_profiler_init)
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_no_trace_hmac(self, mock_profiler_init):
+ hmac_key = "secret"
+ pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
+ headers = {
+ "a": "1",
+ "b": "2",
+ "X-Trace-Info": pack[0]
+ }
+ self._test_wsgi_middleware_with_invalid_trace(headers, hmac_key,
+ mock_profiler_init)
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_invalid_hmac(self, mock_profiler_init):
+ hmac_key = "secret"
+ pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
+ headers = {
+ "a": "1",
+ "b": "2",
+ "X-Trace-Info": pack[0],
+ "X-Trace-HMAC": "not valid hmac"
+ }
+ self._test_wsgi_middleware_with_invalid_trace(headers, hmac_key,
+ mock_profiler_init)
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_invalid_trace_info(self, mock_profiler_init):
+ hmac_key = "secret"
+ pack = utils.signed_pack([{"base_id": "1"}, {"parent_id": "2"}],
+ hmac_key)
+ headers = {
+ "a": "1",
+ "b": "2",
+ "X-Trace-Info": pack[0],
+ "X-Trace-HMAC": pack[1]
+ }
+ self._test_wsgi_middleware_with_invalid_trace(headers, hmac_key,
+ mock_profiler_init)
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_key_passthrough(self, mock_profiler_init):
+ hmac_key = "secret2"
+ request = mock.MagicMock()
+ request.get_response.return_value = "yeah!"
+ request.url = "someurl"
+ request.host_url = "someurl"
+ request.path = "path"
+ request.query_string = "query"
+ request.method = "method"
+ request.scheme = "scheme"
+
+ pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
+
+ request.headers = {
+ "a": "1",
+ "b": "2",
+ "X-Trace-Info": pack[0],
+ "X-Trace-HMAC": pack[1]
+ }
+
+ middleware = web.WsgiMiddleware("app", "secret1,%s" % hmac_key,
+ enabled=True)
+ self.assertEqual("yeah!", middleware(request))
+ mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
+ base_id="1",
+ parent_id="2")
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_key_passthrough2(self, mock_profiler_init):
+ hmac_key = "secret1"
+ request = mock.MagicMock()
+ request.get_response.return_value = "yeah!"
+ request.url = "someurl"
+ request.host_url = "someurl"
+ request.path = "path"
+ request.query_string = "query"
+ request.method = "method"
+ request.scheme = "scheme"
+
+ pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
+
+ request.headers = {
+ "a": "1",
+ "b": "2",
+ "X-Trace-Info": pack[0],
+ "X-Trace-HMAC": pack[1]
+ }
+
+ middleware = web.WsgiMiddleware("app", "%s,secret2" % hmac_key,
+ enabled=True)
+ self.assertEqual("yeah!", middleware(request))
+ mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
+ base_id="1",
+ parent_id="2")
+
+ @mock.patch("osprofiler.web.profiler.Trace")
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware(self, mock_profiler_init, mock_profiler_trace):
+ hmac_key = "secret"
+ request = mock.MagicMock()
+ request.get_response.return_value = "yeah!"
+ request.url = "someurl"
+ request.host_url = "someurl"
+ request.path = "path"
+ request.query_string = "query"
+ request.method = "method"
+ request.scheme = "scheme"
+
+ pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
+
+ request.headers = {
+ "a": "1",
+ "b": "2",
+ "X-Trace-Info": pack[0],
+ "X-Trace-HMAC": pack[1]
+ }
+
+ middleware = web.WsgiMiddleware("app", hmac_key, enabled=True)
+ self.assertEqual("yeah!", middleware(request))
+ mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
+ base_id="1",
+ parent_id="2")
+ expected_info = {
+ "request": {
+ "path": request.path,
+ "query": request.query_string,
+ "method": request.method,
+ "scheme": request.scheme
+ }
+ }
+ mock_profiler_trace.assert_called_once_with("wsgi", info=expected_info)
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_disable_via_python(self, mock_profiler_init):
+ request = mock.MagicMock()
+ request.get_response.return_value = "yeah!"
+ web.disable()
+ middleware = web.WsgiMiddleware("app", "hmac_key", enabled=True)
+ self.assertEqual("yeah!", middleware(request))
+ self.assertEqual(mock_profiler_init.call_count, 0)
+
+ @mock.patch("osprofiler.web.profiler.init")
+ def test_wsgi_middleware_enable_via_python(self, mock_profiler_init):
+ request = mock.MagicMock()
+ request.get_response.return_value = "yeah!"
+ request.url = "someurl"
+ request.host_url = "someurl"
+ request.path = "path"
+ request.query_string = "query"
+ request.method = "method"
+ request.scheme = "scheme"
+ hmac_key = "super_secret_key2"
+
+ pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
+ request.headers = {
+ "a": "1",
+ "b": "2",
+ "X-Trace-Info": pack[0],
+ "X-Trace-HMAC": pack[1]
+ }
+
+ web.enable("super_secret_key1,super_secret_key2")
+ middleware = web.WsgiMiddleware("app", enabled=True)
+ self.assertEqual("yeah!", middleware(request))
+ mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
+ base_id="1",
+ parent_id="2")
+
+ def test_disable(self):
+ web.disable()
+ self.assertFalse(web._ENABLED)
+
+ def test_enabled(self):
+ web.disable()
+ web.enable()
+ self.assertTrue(web._ENABLED)
diff --git a/osprofiler/web.py b/osprofiler/web.py
index 8c53e9a..3a4707d 100644
--- a/osprofiler/web.py
+++ b/osprofiler/web.py
@@ -123,5 +123,8 @@
"scheme": request.scheme
}
}
- with profiler.Trace(self.name, info=info):
- return request.get_response(self.application)
+ try:
+ with profiler.Trace(self.name, info=info):
+ return request.get_response(self.application)
+ finally:
+ profiler._clean()
diff --git a/releasenotes/notes/add-reno-996dd44974d53238.yaml b/releasenotes/notes/add-reno-996dd44974d53238.yaml
new file mode 100644
index 0000000..2234c38
--- /dev/null
+++ b/releasenotes/notes/add-reno-996dd44974d53238.yaml
@@ -0,0 +1,3 @@
+---
+other:
+ - Introduce reno for deployer release notes.
diff --git a/releasenotes/source/_static/.placeholder b/releasenotes/source/_static/.placeholder
new file mode 100644
index 0000000..e69de29
diff --git a/releasenotes/source/_templates/.placeholder b/releasenotes/source/_templates/.placeholder
new file mode 100644
index 0000000..e69de29
diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py
new file mode 100644
index 0000000..6d5caab
--- /dev/null
+++ b/releasenotes/source/conf.py
@@ -0,0 +1,281 @@
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+# sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'openstackdocstheme',
+ 'reno.sphinxext',
+]
+
+# openstackdocstheme options
+repository_name = 'openstack/osprofiler'
+bug_project = 'osprofiler'
+bug_tag = ''
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'osprofiler Release Notes'
+copyright = u'2016, osprofiler Developers'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+# The full version, including alpha/beta/rc tags.
+import pkg_resources
+release = pkg_resources.get_distribution('osprofiler').version
+# The short X.Y version.
+version = release
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+# language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+html_theme = 'openstackdocs'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# " v documentation".
+# html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+# html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+# html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%Y-%m-%d %H:%M'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_domain_indices = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'osprofilerReleaseNotesDoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ # 'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ ('index', 'osprofilerReleaseNotes.tex',
+ u'osprofiler Release Notes Documentation',
+ u'osprofiler Developers', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+# latex_appendices = []
+
+# If false, no module index is generated.
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ ('index', 'osprofilerReleaseNotes',
+ u'osprofiler Release Notes Documentation',
+ [u'osprofiler Developers'], 1)
+]
+
+# If true, show URL addresses after external links.
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ ('index', 'osprofilerReleaseNotes',
+ u'osprofiler Release Notes Documentation',
+ u'osprofiler Developers', 'osprofilerReleaseNotes',
+ 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+# texinfo_no_detailmenu = False
+
+# -- Options for Internationalization output ------------------------------
+locale_dirs = ['locale/']
diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst
new file mode 100644
index 0000000..baf6cbd
--- /dev/null
+++ b/releasenotes/source/index.rst
@@ -0,0 +1,9 @@
+==========================
+ osprofiler Release Notes
+==========================
+
+ .. toctree::
+ :maxdepth: 1
+
+ unreleased
+ ocata
diff --git a/releasenotes/source/ocata.rst b/releasenotes/source/ocata.rst
new file mode 100644
index 0000000..ebe62f4
--- /dev/null
+++ b/releasenotes/source/ocata.rst
@@ -0,0 +1,6 @@
+===================================
+ Ocata Series Release Notes
+===================================
+
+.. release-notes::
+ :branch: origin/stable/ocata
diff --git a/releasenotes/source/unreleased.rst b/releasenotes/source/unreleased.rst
new file mode 100644
index 0000000..cd22aab
--- /dev/null
+++ b/releasenotes/source/unreleased.rst
@@ -0,0 +1,5 @@
+==============================
+ Current Series Release Notes
+==============================
+
+.. release-notes::
diff --git a/requirements.txt b/requirements.txt
index d59f2f3..e38a9d8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,8 @@
six>=1.9.0 # MIT
-oslo.utils>=3.4.0 # Apache-2.0
-WebOb>=1.2.3 # MIT
+oslo.messaging>=5.2.0 # Apache-2.0
+oslo.log>=3.11.0 # Apache-2.0
+oslo.utils>=3.16.0 # Apache-2.0
+WebOb>=1.6.0 # MIT
+requests>=2.10.0 # Apache-2.0
+netaddr>=0.7.13,!=0.7.16 # BSD
+oslo.concurrency>=3.8.0 # Apache-2.0
diff --git a/setup.cfg b/setup.cfg
index 1c17426..d2d655e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -5,7 +5,7 @@
README.rst
author = OpenStack
author-email = openstack-dev@lists.openstack.org
-home-page = http://www.openstack.org/
+home-page = https://docs.openstack.org/osprofiler/latest/
classifier =
Environment :: OpenStack
Intended Audience :: Developers
@@ -15,7 +15,7 @@
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
- Programming Language :: Python :: 3.4
+ Programming Language :: Python :: 3.5
[files]
packages =
@@ -33,9 +33,12 @@
all_files = 1
build-dir = doc/build
source-dir = doc/source
+warning-is-error = 1
[entry_points]
oslo.config.opts =
osprofiler = osprofiler.opts:list_opts
console_scripts =
osprofiler = osprofiler.cmd.shell:main
+paste.filter_factory =
+ osprofiler = osprofiler.web:WsgiMiddleware.factory
diff --git a/setup.py b/setup.py
index b96e399..ddd1771 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,27 @@
-#!/usr/bin/env python
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+ import multiprocessing # noqa
+except ImportError:
+ pass
+
setuptools.setup(
- setup_requires=['pbr'],
+ setup_requires=['pbr>=1.8'],
pbr=True)
diff --git a/test-requirements.txt b/test-requirements.txt
index 8ba925b..aa45596 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,11 +1,26 @@
-hacking>=0.10.2,<0.11
+hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0
-coverage>=3.6
-discover
-mock>=1.2
-python-subunit>=0.0.18
-testrepository>=0.0.18
-testtools>=1.4.0
+coverage>=3.6 # Apache-2.0
+ddt>=1.0.1 # MIT
+mock>=2.0 # BSD
+python-subunit>=0.0.18 # Apache-2.0/BSD
+testrepository>=0.0.18 # Apache-2.0/BSD
+testtools>=1.4.0 # MIT
-oslosphinx>=2.5.0,!=3.4.0 # Apache-2.0
-sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
+openstackdocstheme>=1.11.0 # Apache-2.0
+sphinx>=1.6.2 # BSD
+
+# Bandit security code scanner
+bandit>=1.1.0 # Apache-2.0
+
+python-ceilometerclient>=2.5.0 # Apache-2.0
+pymongo>=3.0.2,!=3.1 # Apache-2.0
+
+# Elasticsearch python client
+elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0
+
+# Redis python client
+redis>=2.10.0 # MIT
+
+# Build release notes
+reno>=1.8.0 # Apache-2.0
diff --git a/tools/lint.py b/tools/lint.py
index 69b88ca..d2a545b 100644
--- a/tools/lint.py
+++ b/tools/lint.py
@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
# Copyright (c) 2013 Intel Corporation.
# All Rights Reserved.
#
diff --git a/tools/patch_tox_venv.py b/tools/patch_tox_venv.py
index dc9ce83..ee8f53c 100644
--- a/tools/patch_tox_venv.py
+++ b/tools/patch_tox_venv.py
@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
# Copyright 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
diff --git a/tox.ini b/tox.ini
index 12e18de..8929336 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,7 @@
[tox]
minversion = 1.6
skipsdist = True
-envlist = py34,py27,pep8
+envlist = py35,py27,pep8
[testenv]
setenv = VIRTUAL_ENV={envdir}
@@ -16,8 +16,22 @@
commands = python setup.py testr --slowest --testr-args='{posargs}'
distribute = false
+[testenv:functional]
+basepython = python2.7
+setenv = {[testenv]setenv}
+ OS_TEST_PATH=./osprofiler/tests/functional
+deps = {[testenv]deps}
+
+[testenv:functional-py35]
+basepython = python3.5
+setenv = {[testenv:functional]setenv}
+deps = {[testenv:functional]deps}
+
[testenv:pep8]
-commands = flake8
+commands =
+ flake8
+ # Run security linter
+ bandit -r osprofiler -n5
distribute = false
[testenv:venv]
@@ -27,13 +41,18 @@
commands = python setup.py testr --coverage --testr-args='{posargs}'
[testenv:docs]
-changedir = doc/source
-commands = make html
+commands = python setup.py build_sphinx
+
+[testenv:bandit]
+commands = bandit -r osprofiler -n5
[flake8]
show-source = true
builtins = _
-exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,setup.py
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,setup.py,build,releasenotes
[hacking]
-local-check-factory = osprofiler.tests.hacking.checks.factory
+local-check-factory = osprofiler.hacking.checks.factory
+
+[testenv:releasenotes]
+commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html