支持 OSProfiler 的核心组件

OSProfiler 支持的组件的起始版本,6 个核心组件

组件 版本
Keystone Newton
Glance Juno
Nova Ocata
Neutron Newton
Swift Blueprint Not started
Cinder Juno

从 3.0.0 版本开始 OSProfiler 弃用 Python2 支持,至少得使用 Python3.6 。每个 OpenStack 都有对应的稳定版 OSProfiler ,主要用于性能测试。

Keystone

相关文件

1
2
3
4
5
keystone/conf/__init__.py      # 设置默认配置
keystone/common/profiler.py # 读取配置文件,设置通知驱动,启用中间件
keystone/common/sql/core.py # 包装数据库会话
keystone/server/flask/core.py # 初始化,调用 common/profiler.py
setup.cfg # 设置中间件 WsgiMiddleware

keystone/conf/__init__.py 设置 OSProfiler 默认配置,不启用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
def set_external_opts_defaults():
"""Update default configuration options for oslo.middleware."""
cors.set_defaults(
allow_headers=['X-Auth-Token',
'X-Openstack-Request-Id',
'X-Subject-Token',
'X-Project-Id',
'X-Project-Name',
'X-Project-Domain-Id',
'X-Project-Domain-Name',
'X-Domain-Id',
'X-Domain-Name',
'Openstack-Auth-Receipt'],
expose_headers=['X-Auth-Token',
'X-Openstack-Request-Id',
'X-Subject-Token',
'Openstack-Auth-Receipt'],
allow_methods=['GET',
'PUT',
'POST',
'DELETE',
'PATCH']
)

# configure OSprofiler options
profiler.set_defaults(CONF, enabled=False, trace_sqlalchemy=False) # 设置 OSProfiler 默认配置
...

keystone/common/profiler.py 读取配置文件启用 OSProfiler ,设置驱动,添加日志信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
def setup(name, host='0.0.0.0'):  # nosec
"""Setup OSprofiler notifier and enable profiling.

:param name: name of the service that will be profiled
:param host: hostname or host IP address that the service will be
running on. By default host will be set to 0.0.0.0, but more
specified host name / address usage is highly recommended.
"""
if CONF.profiler.enabled:
osprofiler.initializer.init_from_conf(
conf=CONF,
context={},
project="keystone",
service=name,
host=host
)
LOG.info("OSProfiler is enabled.\n"
"Traces provided from the profiler "
"can only be subscribed to using the same HMAC keys that "
"are configured in Keystone's configuration file "
"under the [profiler] section. \n To disable OSprofiler "
"set in /etc/keystone/keystone.conf:\n"
"[profiler]\n"
"enabled=false")

setup.cfg 设置 server 中间件

1
2
3
4
5
6
7
keystone.server_middleware =
cors = oslo_middleware:CORS
sizelimit = oslo_middleware:RequestBodySizeLimiter
http_proxy_to_wsgi = oslo_middleware:HTTPProxyToWSGI
osprofiler = osprofiler.web:WsgiMiddleware <--- 使用 osprofiler 的 WSGI 中间件
request_id = oslo_middleware:RequestId
debug = oslo_middleware:Debug

keystone/server/flask/core.py 服务器中间件按以下顺序处理请求

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# NOTE(morgan): ORDER HERE IS IMPORTANT! The middleware will process the
# request in this list's order.
_APP_MIDDLEWARE = (
_Middleware(namespace='keystone.server_middleware',
ep='cors',
conf={'oslo_config_project': 'keystone'}),
_Middleware(namespace='keystone.server_middleware',
ep='sizelimit',
conf={}),
_Middleware(namespace='keystone.server_middleware',
ep='http_proxy_to_wsgi',
conf={}),
_Middleware(namespace='keystone.server_middleware', # OSProfiler 提供的 WSGI 中间件
ep='osprofiler',
conf={}),
_Middleware(namespace='keystone.server_middleware',
ep='request_id',
conf={}),
)

keystone/common/sql/core.py 包装数据库引擎会话

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 使用 osprofiler 模块包装会话
def _wrap_session(sess):
if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
sess = osprofiler.sqlalchemy.wrap_session(sql, sess)
return sess

# 读
def session_for_read():
if _TESTING_USE_GLOBAL_CONTEXT_MANAGER:
reader = enginefacade.reader
else:
reader = _get_main_context_manager().reader
return _wrap_session(reader.using(_get_context()))

# 写
def session_for_write():
if _TESTING_USE_GLOBAL_CONTEXT_MANAGER:
writer = enginefacade.writer
else:
writer = _get_main_context_manager().writer
return _wrap_session(writer.using(_get_context()))

Cinder

相关文件

1
2
3
cinder/db/sqlalchemy/api.py     # 添加数据库追踪
cinder/rpc.py # 上下文序列化/反序列化
cinder/service.py # 启用 OSProfiler;添加 RPC 追踪

cinder/db/sqlalchemy/api.py 添加数据库追踪

1
2
3
4
5
6
7
8
9
10
11
def configure(conf):
main_context_manager.configure(**dict(conf.database))
# NOTE(geguileo): To avoid a cyclical dependency we import the
# group here. Dependency cycle is objects.base requires db.api,
# which requires db.sqlalchemy.api, which requires service which
# requires objects.base
CONF.import_group("profiler", "cinder.service")
if CONF.profiler.enabled:
if CONF.profiler.trace_sqlalchemy: # 添加数据库追踪
lambda eng: osprofiler_sqlalchemy.add_tracing(sqlalchemy,
eng, "db")

cinder/rpc.py 请求上下文类,对上下文进行序列化和反序列化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
class RequestContextSerializer(messaging.Serializer):

def __init__(self, base):
self._base = base

def serialize_entity(self, context, entity):
if not self._base:
return entity
return self._base.serialize_entity(context, entity)

def deserialize_entity(self, context, entity):
if not self._base:
return entity
return self._base.deserialize_entity(context, entity)

def serialize_context(self, context):
_context = context.to_dict()
if profiler is not None:
prof = profiler.get()
if prof:
trace_info = {
"hmac_key": prof.hmac_key,
"base_id": prof.get_base_id(),
"parent_id": prof.get_id()
}
_context.update({"trace_info": trace_info})
return _context

def deserialize_context(self, context):
trace_info = context.pop("trace_info", None)
if trace_info:
if profiler is not None:
profiler.init(**trace_info)

return cinder.context.RequestContext.from_dict(context)

cinder/service.py 读取配置文件启用 OSProfiler ,设置驱动,添加日志信息;使用类装饰器追踪 RPC 调用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
CONF = cfg.CONF
CONF.register_opts(service_opts)
if profiler_opts:
profiler_opts.set_defaults(CONF)


def setup_profiler(binary, host):
if (osprofiler_initializer is None or
profiler is None or
profiler_opts is None):
LOG.debug('osprofiler is not present')
return

if CONF.profiler.enabled:
osprofiler_initializer.init_from_conf(
conf=CONF,
context=context.get_admin_context().to_dict(),
project="cinder",
service=binary,
host=host
)
LOG.warning(
"OSProfiler is enabled.\nIt means that person who knows "
"any of hmac_keys that are specified in "
"/etc/cinder/cinder.conf can trace his requests. \n"
"In real life only operator can read this file so there "
"is no security issue. Note that even if person can "
"trigger profiler, only admin user can retrieve trace "
"information.\n"
"To disable OSProfiler set in cinder.conf:\n"
"[profiler]\nenabled=false")


class Service(service.Service):
"""Service object for binaries running on hosts.

A service takes a manager and enables rpc by listening to queues based
on topic. It also periodically runs tasks on the manager and reports
it state to the database services table.
"""
# Make service_id a class attribute so it can be used for clean up
service_id = None

def __init__(self, host, binary, topic, manager, report_interval=None,
periodic_interval=None, periodic_fuzzy_delay=None,
service_name=None, coordination=False, cluster=None, *args,
**kwargs):
super(Service, self).__init__()

if not rpc.initialized():
rpc.init(CONF)

self.cluster = cluster
self.host = host
self.binary = binary
self.topic = topic
self.manager_class_name = manager
self.coordination = coordination
manager_class = importutils.import_class(self.manager_class_name)
if CONF.profiler.enabled:
manager_class = profiler.trace_cls("rpc")(manager_class) # 类装饰器
...

Glance

相关文件

1
2
3
4
5
6
glance/opts.py                  # 添加 OSProfiler 选项
glance/db/sqlalchemy/api.py # 添加数据库追踪
glance/cmd/api.py # 启用 OSProfiler
glance/common/client.py # 更新 http 请求头中的追踪信息
glance/common/wsgi.py # 设置 OSProfiler 默认配置
glance/common/wsgi_app.py # 启用 OSProfiler

glance/opts.py 包含 OSProfiler 选项,作为 oslo_config 的选项,通过 Glance API 获得选项列表

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
_api_opts = [
(None, list(itertools.chain(
glance.api.middleware.context.context_opts,
glance.api.versions.versions_opts,
glance.common.config.common_opts,
glance.common.location_strategy.location_strategy_opts,
glance.common.property_utils.property_opts,
glance.common.wsgi.bind_opts,
glance.common.wsgi.eventlet_opts,
glance.common.wsgi.socket_opts,
glance.common.wsgi.wsgi_opts,
glance.common.wsgi.store_opts,
glance.image_cache.drivers.sqlite.sqlite_opts,
glance.image_cache.image_cache_opts,
glance.notifier.notifier_opts,
glance.scrubber.scrubber_opts))),
('image_format', glance.common.config.image_format_opts),
('task', glance.common.config.task_opts),
('taskflow_executor', list(itertools.chain(
glance.async_.taskflow_executor.taskflow_executor_opts,
glance.async_.flows.convert.convert_task_opts))),
('store_type_location_strategy',
glance.common.location_strategy.store_type.store_type_opts),
profiler.list_opts()[0], # 添加 OSProfiler 选项
('paste_deploy', glance.common.config.paste_deploy_opts),
('wsgi', glance.common.config.wsgi_opts),
]

glance/db/sqlalchemy/api.py 添加数据库追踪

1
2
3
4
5
6
7
8
9
10
11
12
def _create_facade_lazily():
global _LOCK, _FACADE
if _FACADE is None:
with _LOCK:
if _FACADE is None:
_FACADE = session.EngineFacade.from_config(CONF)
# 添加数据库追踪
if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
osprofiler.sqlalchemy.add_tracing(sqlalchemy,
_FACADE.get_engine(),
"db")
return _FACADE

glance/cmd/api.py 启用 OSProfiler

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
def main():
try:
config.parse_args()
config.set_config_defaults()
wsgi.set_eventlet_hub()
logging.setup(CONF, 'glance')
gmr.TextGuruMeditation.setup_autorun(version)
notifier.set_defaults()

if CONF.profiler.enabled:
osprofiler.initializer.init_from_conf( # 启用 OSProfiler
conf=CONF,
context={},
project="glance",
service="api",
host=CONF.bind_host
)
...

glance/common/wsgi.py 设置 OSProfiler 默认配置

1
profiler_opts.set_defaults(CONF)    # OSProfiler 默认配置

glance/common/wsgi_app.py 初始化应用时启用 OSProfiler

1
2
3
4
5
6
7
8
def _setup_os_profiler():
notifier.set_defaults()
if CONF.profiler.enabled:
osprofiler.initializer.init_from_conf(conf=CONF,
context={},
project='glance',
service='api',
host=CONF.bind_host)

glance/common/client.py 更新请求头中的追踪信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
class BaseClient(object):

"""A base client class"""
...

@handle_redirects
def _do_request(self, method, url, body, headers):
"""
Connects to the server and issues a request. Handles converting
any returned HTTP error status codes to OpenStack/Glance exceptions
and closing the server connection. Returns the result data, or
raises an appropriate exception.

:param method: HTTP method ("GET", "POST", "PUT", etc...)
:param url: urlparse.ParsedResult object with URL information
:param body: data to send (as string, filelike or iterable),
or None (default)
:param headers: mapping of key/value pairs to add as headers

:note

If the body param has a read attribute, and method is either
POST or PUT, this method will automatically conduct a chunked-transfer
encoding and use the body as a file object or iterable, transferring
chunks of data using the connection's send() method. This allows large
objects to be transferred efficiently without buffering the entire
body in memory.
"""
if url.query:
path = url.path + "?" + url.query
else:
path = url.path

try:
connection_type = self.get_connection_type()
headers = self._encode_headers(headers or {})
headers.update(osprofiler.web.get_trace_id_headers()) # 更新请求头
...

Nova

相关文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
nova/config.py             # 根据配置文件设置 osprofiler
nova/service.py # 根据配置文件设置驱动,启用 WSGI 中间件
nova/profiler.py # 重写 WSGI 中间件、类追踪装饰器
nova/manager.py # 元类
nova/rpc.py # 追踪上下文序列化/反序列化
nova/utils.py # spawn/spawn_n 装饰器
nova/db/sqlalchemy/api.py # 追踪数据库调用

# 继承抽象基类
nova/compute/manager.py
nova/conductor/manager.py
nova/scheduler/manager.py

# 调用类装饰器 @profiler.trace_cls("")
nova/compute/api.py # compute_api
nova/compute/rpcapi.py # rpc
nova/conductor/manager.py # rpc
nova/conductor/rpcapi.py # rpc
nova/scheduler/rpcapi.py # rpc
nova/image/glance.py # nova_image
nova/network/neutron.py # neutron_api
nova/virt/libvirt/volume/volume.py # volume_api
nova/virt/libvirt/vif.py # vif_driver

nova/config.py 设置 OSProfiler 默认配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
def parse_args(argv, default_config_files=None, configure_db=True,
init_rpc=True):
log.register_options(CONF)

# NOTE(sean-k-mooney): this filter addresses bug #1825584
# https://bugs.launchpad.net/nova/+bug/1825584
# eventlet monkey-patching breaks AMQP heartbeat on uWSGI
rabbit_logger = logging.getLogger('oslo.messaging._drivers.impl_rabbit')
rabbit_logger.addFilter(rabbit_heartbeat_filter)

set_lib_defaults()
if profiler:
profiler.set_defaults(CONF) # 设置默认配置

CONF(argv[1:],
project='nova',
version=version.version_string(),
default_config_files=default_config_files)

if init_rpc:
rpc.init(CONF)

if configure_db:
sqlalchemy_api.configure(CONF)

nova/service.py 根据配置文件启用 OSProfiler

1
2
3
4
5
6
7
8
9
def setup_profiler(binary, host):
if osprofiler and CONF.profiler.enabled:
osprofiler.initializer.init_from_conf(
conf=CONF,
context=context.get_admin_context().to_dict(),
project="nova",
service=binary,
host=host)
LOG.info("OSProfiler is enabled.")

nova/profiler.py 重写 OSProfiler 装饰器类

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
# 重写 WSGI 中间件
class WsgiMiddleware(object):

def __init__(self, application, **kwargs):
self.application = application

@classmethod
def factory(cls, global_conf, **local_conf):
if profiler_web: # 调用 osprofiler
return profiler_web.WsgiMiddleware.factory(global_conf,
**local_conf)

def filter_(app):
return cls(app, **local_conf)

return filter_

@webob.dec.wsgify
def __call__(self, request):
return request.get_response(self.application)

# 获取元数据
def get_traced_meta():
if profiler and 'profiler' in CONF and CONF.profiler.enabled:
return profiler.TracedMeta
else:
# NOTE(rpodolyaka): if we do not return a child of type, then Python
# fails to build a correct MRO when osprofiler is not installed
class NoopMeta(type):
pass
return NoopMeta

# 包装 osprofiler 类装饰器
def trace_cls(name, **kwargs):
"""Wrap the OSProfiler trace_cls decorator so that it will not try to
patch the class unless OSProfiler is present and enabled in the config

:param name: The name of action. E.g. wsgi, rpc, db, etc..
:param kwargs: Any other keyword args used by profiler.trace_cls
"""

def decorator(cls):
if profiler and 'profiler' in CONF and CONF.profiler.enabled:
trace_decorator = profiler.trace_cls(name, kwargs)
return trace_decorator(cls)
return cls

return decorator

nova/manager.py 使用元类实现类支持追踪,nova 组件内部的模块都继承该类(ComputeManagerConductorManagerSchedulerManager

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# 元类
class ManagerMeta(profiler.get_traced_meta(), type(PeriodicTasks)):
"""Metaclass to trace all children of a specific class.

This metaclass wraps every public method (not starting with _ or __)
of the class using it. All children classes of the class using ManagerMeta
will be profiled as well.

Adding this metaclass requires that the __trace_args__ attribute be added
to the class we want to modify. That attribute is a dictionary
with one mandatory key: "name". "name" defines the name
of the action to be traced (for example, wsgi, rpc, db).

The OSprofiler-based tracing, although, will only happen if profiler
instance was initiated somewhere before in the thread, that can only happen
if profiling is enabled in nova.conf and the API call to Nova API contained
specific headers.
"""

# 继承
class Manager(base.Base, PeriodicTasks, metaclass=ManagerMeta):
__trace_args__ = {"name": "rpc"} # 必须
...

nova/rpc.py 实现追踪上下文的序列化(并更新上下文)和反序列化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
class ProfilerRequestContextSerializer(RequestContextSerializer):
def serialize_context(self, context):
_context = super(ProfilerRequestContextSerializer,
self).serialize_context(context)

prof = profiler.get() # 获取 osprofiler 实例
if prof:
# FIXME(DinaBelova): we'll add profiler.get_info() method
# to extract this info -> we'll need to update these lines
trace_info = {
"hmac_key": prof.hmac_key,
"base_id": prof.get_base_id(),
"parent_id": prof.get_id()
}
_context.update({"trace_info": trace_info}) # 添加追踪信息

return _context

def deserialize_context(self, context):
trace_info = context.pop("trace_info", None)
if trace_info:
profiler.init(**trace_info) # 初始新的化 osprofiler 实例

return super(ProfilerRequestContextSerializer,
self).deserialize_context(context)

nova/utils.py 实现 spawn 和 spawn_n 装饰器

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
# 返回追踪信息
def _serialize_profile_info():
if not profiler:
return None
prof = profiler.get() # 获取实例
trace_info = None
if prof:
# FIXME(DinaBelova): we'll add profiler.get_info() method
# to extract this info -> we'll need to update these lines
trace_info = {
"hmac_key": prof.hmac_key,
"base_id": prof.get_base_id(),
"parent_id": prof.get_id()
}
return trace_info

# 装饰器
def spawn(func, *args, **kwargs):
"""Passthrough method for eventlet.spawn.

This utility exists so that it can be stubbed for testing without
interfering with the service spawns.

It will also grab the context from the threadlocal store and add it to
the store on the new thread. This allows for continuity in logging the
context when using this method to spawn a new thread.
"""
_context = common_context.get_current() # 当前线程
profiler_info = _serialize_profile_info() # 追踪信息

@functools.wraps(func)
def context_wrapper(*args, **kwargs):
# NOTE: If update_store is not called after spawn it won't be
# available for the logger to pull from threadlocal storage.
if _context is not None:
_context.update_store()
if profiler_info and profiler:
profiler.init(**profiler_info) # 初始化 osprofiler 实例
return func(*args, **kwargs)

return eventlet.spawn(context_wrapper, *args, **kwargs)

# 装饰器
def spawn_n(func, *args, **kwargs):
"""Passthrough method for eventlet.spawn_n.

This utility exists so that it can be stubbed for testing without
interfering with the service spawns.

It will also grab the context from the threadlocal store and add it to
the store on the new thread. This allows for continuity in logging the
context when using this method to spawn a new thread.
"""
_context = common_context.get_current()
profiler_info = _serialize_profile_info()

@functools.wraps(func)
def context_wrapper(*args, **kwargs):
# NOTE: If update_store is not called after spawn_n it won't be
# available for the logger to pull from threadlocal storage.
if _context is not None:
_context.update_store()
if profiler_info and profiler:
profiler.init(**profiler_info)
func(*args, **kwargs)

eventlet.spawn_n(context_wrapper, *args, **kwargs)

nova/db/sqlalchemy/api.py 添加数据库追踪

1
2
3
4
5
6
7
8
9
10
11
12
# 追踪数据库调用
def configure(conf):
main_context_manager.configure(**_get_db_conf(conf.database))
api_context_manager.configure(**_get_db_conf(conf.api_database))

if profiler_sqlalchemy and CONF.profiler.enabled \
and CONF.profiler.trace_sqlalchemy:

main_context_manager.append_on_engine_create(
lambda eng: profiler_sqlalchemy.add_tracing(sa, eng, "db")) # 添加追踪
api_context_manager.append_on_engine_create(
lambda eng: profiler_sqlalchemy.add_tracing(sa, eng, "db")) # 添加追踪

Neutron

相关文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
neutron/manager.py          # 元类
neutron/common/profiler.py # 根据配置文件设置驱动,启用 WSGI 中间件
neutron/common/utils.py # spawn/spawn_n 装饰器

# 调用 neutron/common/profiler.py 设置 OSProfiler
neutron/service.py
neutron/server/__init__.py
neutron/plugins/ml2/drivers/openvswitch/agent/main.py
neutron/plugins/ml2/drivers/linuxbridge/agent/linuxbridge_neutron_agent.py
neutron/plugins/ml2/drivers/mec_sriov/agent/sriov_nic_agent.py

# 调用类装饰器 @profiler.trace_cls("")
neutron/agent/l3/agent.py # l3-agent
neutron/plugins/ml2/drivers/openvswitch/agent/ovs_dvr_neutron_agent.py # ovs_dvr_agent
neutron/plugins/ml2/drivers/agent/_common_agent.py # rpc
neutron/plugins/ml2/drivers/mec_sriov/agent/sriov_nic_agent.py # rpc

# 调用函数装饰器 @profiler.trace("")
neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py # rpc
neutron/plugins/ml2/rpc.py # rpc

# cProfile
conf/profiling.py
neutron/profiling/profiled_decorator.py

neutron/manager.py 和 Nova 同样的实现思路,使用元类实现类追踪

1
2
3
4
5
6
7
8
# 元类
class ManagerMeta(profiler.TracedMeta, type(periodic_task.PeriodicTasks)):
pass

# 继承
class Manager(periodic_task.PeriodicTasks, metaclass=ManagerMeta):
__trace_args__ = {"name": "rpc"}
...

neutron/common/profiler.py 启用 OSProfiler

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
def setup(name, host='0.0.0.0'):  # nosec
"""Setup OSprofiler notifier and enable profiling.

:param name: name of the service, that will be profiled
:param host: host (either host name or host address) the service will be
running on. By default host will be set to 0.0.0.0, but more
specified host name / address usage is highly recommended.
"""
if CONF.profiler.enabled:
osprofiler.initializer.init_from_conf( # 启用 OSProfiler
conf=CONF,
context=context.get_admin_context().to_dict(),
project="neutron",
service=name,
host=host
)
LOG.info("OSProfiler is enabled.\n"
"Traces provided from the profiler "
"can only be subscribed to using the same HMAC keys that "
"are configured in Neutron's configuration file "
"under the [profiler] section.\n To disable OSprofiler "
"set in /etc/neutron/neutron.conf:\n"
"[profiler]\n"
"enabled=false")

neutron/common/utils.py 返回追踪信息,实现 spawn 和 spawn_n 装饰器

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
# 返回追踪信息
def collect_profiler_info():
p = profiler.get()
if p:
return {
"hmac_key": p.hmac_key,
"base_id": p.get_base_id(),
"parent_id": p.get_id(),
}

# 装饰器
def spawn(func, *args, **kwargs):
"""As eventlet.spawn() but with osprofiler initialized in the new threads

osprofiler stores the profiler instance in thread local storage, therefore
in new threads (including eventlet threads) osprofiler comes uninitialized
by default. This spawn() is a stand-in replacement for eventlet.spawn()
but we re-initialize osprofiler in threads spawn()-ed.
"""

profiler_info = collect_profiler_info()

@functools.wraps(func)
def wrapper(*args, **kwargs):
if profiler_info:
profiler.init(**profiler_info)
return func(*args, **kwargs)

return eventlet.spawn(wrapper, *args, **kwargs)

# 装饰器
def spawn_n(func, *args, **kwargs):
"""See spawn() above"""

profiler_info = collect_profiler_info()

@functools.wraps(func)
def wrapper(*args, **kwargs):
if profiler_info:
profiler.init(**profiler_info)
return func(*args, **kwargs)

return eventlet.spawn_n(wrapper, *args, **kwargs)

neutron/server/__init__.py 启用 OSProfiler

1
2
3
4
def get_application():
_init_configuration()
profiler.setup('neutron-server', cfg.CONF.host) # 启用 OSProfiler
return config.load_paste_app('neutron')

OSProfiler 使用

实验环境

CentOS 7 安装 Openstack Rocky 版本 - 环境搭建 一系列文章的基础上进行实验,整体架构如下:

修改控制节点上相关组件的配置文件,启用 OSProfiler

Controller

  • Keystone
  • Glance
  • Nova*
  • Neutron*
  • Cinder
1
2
3
4
5
6
7
8
9
10
11
[profiler]
enabled = True

trace_sqlalchemy = True
trace_wsgi_transport = True
trace_message_store = True
trace_management_store = True

hmac_keys = 123

connection_string = mongodb://10.112.116.249:27017

使用 MongoDB 作为后端,如果用 Elasticsearch 或 Jaeger 会产生错误,暂时没有找到解决方案,可能是版本不兼容。

在 Gateway 上使用 Docker 部署 MongoDB ,存储追踪信息。

追踪服务组件

显式指定追踪的调用命令

Keystone

身份认证服务,列出所有用户

1
2
3
4
5
6
7
8
9
10
11
# 清除临时环境变量
unset OS_AUTH_URL OS_PASSWORD

# 相关权限认证
. admin-openrc

# 列出所有用户
openstack user list --os-profile 123

# 导出 html 文件
osprofiler trace show --html fdfa1e7a-0863-470b-8ae2-169b21b0fbe5 --connection-string "mongodb://10.112.116.249:27017" --out "test1.html"

Glance

镜像服务,列出可用镜像

1
2
3
4
5
6
7
8
9
10
11
# 清除临时环境变量
unset OS_AUTH_URL OS_PASSWORD

# 相关权限认证
. demo-openrc

# 列出可用镜像
openstack image list --os-profile 123

# 导出 html 文件
osprofiler trace show --html 04c4f7b5-e22f-40f7-8248-162247642cb5 --connection-string "mongodb://10.112.116.249:27017" --out "test2.html"

Nova

计算服务,查询实例状态

1
2
3
4
5
6
7
8
9
10
11
# 清除临时环境变量
unset OS_AUTH_URL OS_PASSWORD

# 相关权限认证
. demo-openrc

# 查看实例
openstack server list --os-profile 123

# 导出 html 文件
osprofiler trace show --html 8cce121c-adf1-4061-96eb-950bb4a75db8 --connection-string "mongodb://10.112.116.249:27017" --out "test3.html"

Cinder

卷服务,查看卷状态

1
2
3
4
5
6
7
8
9
10
11
# 清除临时环境变量
unset OS_AUTH_URL OS_PASSWORD

# 相关权限认证
. demo-openrc

# 查看卷
openstack volume list --os-profile 123

# 导出 html 文件
osprofiler trace show --html 71760b3d-df04-4fec-bc26-dfc9b909f518 --connection-string "mongodb://10.112.116.249:27017" --out "test4.html"

元数据分析

trace point 可以理解为 span

字段 格式 含义
info <dict> 在调用 profiler 的 start()stop() 方法时传递的用户信息,数据库语句、请求参数等
name `<point_name>-(start stop)`
service <service_name> public / api / osapi_compute
timestamp <timestamp> 时间戳
trace_id <uuid> 当前追踪点id
project <project_name> 服务组件
parent_id <uuid> 父级追踪点id
base_id <uuid> 所有属于一条追踪链的追踪点都拥有相同的id

OSProfiler 源码分析

用 git 拉取源码 openstack/osprofiler

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
├─devstack    # DevStack 脚本,安装和配置 osprofiler
├─doc # 文档
├─osprofiler # 主要代码
│ │ exc.py # cmd 错误类
│ │ initializer.py # (读取配置文件)初始化
│ │ notifier.py # 通知消息
│ │ opts.py # 配置选项
│ │ profiler.py # osprofiler 实例,函数/类装饰器,元数据类
│ │ sqlalchemy.py # 追踪数据库调用
│ │ web.py # 追踪 WSGI 调用
│ │ _utils.py # 工具函数
│ │ __init__.py
│ ├─cmd # cmd 接口
│ │ cliutils.py # 参数装饰、绑定
│ │ commands.py # 显示和保存指定跟踪(html/json/dot),列出所有跟踪
│ │ shell.py # 处理命令行
│ │ template.html # html 模板
│ │ __init__.py
│ ├─drivers # 驱动
│ │ base.py # 基类
│ │ elasticsearch_driver.py
│ │ jaeger.py
│ │ loginsight.py
│ │ messaging.py
│ │ mongodb.py
│ │ redis_driver.py
│ │ sqlalchemy_driver.py
│ │ __init__.py
│ ├─hacking # 针对 osprofiler 编写的测试
│ └─tests # 测试
├─playbooks # Ansible 剧本
├─releasenotes # 发行说明
└─tools # 代码风格检查,tox 虚拟环境

_utils.py(工具函数)

  • 私有模块
  • 公有函数
1
2
3
4
5
6
7
8
9
def split(text, strip=True):                    # 分割逗号分隔的文本
def binary_encode(text, encoding="utf-8"): # 将文本转换为二进制编码
def binary_decode(data, encoding="utf-8"): # 将二进制编码转换为文本
def generate_hmac(data, hmac_key): # 用 key 产生 HMAC
def signed_pack(data, hmac_key): # 用 key 打包和签名数据
def signed_unpack(data, hmac_data, hmac_keys): # 解包数据并验证签名
def itersubclasses(cls, _seen=None): # 判断是否为子类
def import_modules_from_package(package): # 从包导入模块并加入系统模块
def shorten_id(span_id): # UUID 转换为 64 位 ID

notifier.py(通知消息)

  • 默认使用基本的驱动程序
  • 根据参数设置驱动
1
2
3
4
5
6
def _noop_notifier(info, context=None):         # 使用基本的驱动程序
def notify(info): # 传递信息
def get(): # 返回可调用函数
def set(notifier): # 设置可调用函数
def create(connection_string, *args, **kwargs): # 根据参数设置驱动
def clear_notifier_cache(): # 清除缓存

profiler.py(osprofiler 实例,函数/类装饰器,元数据类)

  • 函数装饰器
  • 类装饰器
  • 元数据类
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
def clean():                                            # 清除全局 ThreadLocal 对象
def _ensure_no_multiple_traced(traceable_attrs): # 确保不重复跟踪
def init(hmac_key, base_id=None, parent_id=None): # 初始化 osprofiler 线程实例
def get(): # 获取 osprofiler 实例
def start(name, info=None): # 启动 osprofiler
def stop(info=None): # 停止 osprofiler

# 函数追踪装饰器
def trace(name, info=None, hide_args=False, hide_result=True,
allow_multiple_trace=True):

# 类追踪装饰器
def trace_cls(name, info=None, hide_args=False, hide_result=True,
trace_private=False, allow_multiple_trace=True,
trace_class_methods=False, trace_static_methods=False):

class TracedMeta(type): # 元数据
class Trace(object): # 封装 osprofiler 线程实例,使用 with 语句调用
class _Profiler(object): # 私有类(测试用)

sqlalchemy.py(追踪数据库调用)

  • 监听调用(调用前、调用后、调用错误)
  • 包装数据库会话连接
1
2
3
4
5
6
7
8
9
def add_tracing(sqlalchemy, engine, name, hide_result=True):    # 追踪数据库调用

# 包装会话
@contextlib.contextmanager
def wrap_session(sqlalchemy, sess):

def _before_cursor_execute(name): # 传递语句及参数
def _after_cursor_execute(hide_result=True): # 传递执行结果
def handle_error(exception_context): # 传递错误信息

web.py(追踪 WSGI 调用)

  • 签名和打包请求头
  • WSGI 中间件类
1
2
def get_trace_id_headers():     # 签名请求头并添加到字典
class WsgiMiddleware(object): # WSGI 中间件类

initializer.py(读取配置文件初始化)

  • 读取配置文件
  • 设置通知驱动
  • 启用中间件
1
2
# 服务配置文件,请求上下文,项目名称,服务名称,主机名称/IP,通知参数
def init_from_conf(conf, context, project, service, host, **kwargs):

opts.py(配置选项)

  • 设置默认配置
  • 判断配置选项
  • 列出配置选项
1
2
3
4
5
6
7
8
9
10
11
# 默认配置
def set_defaults(conf, enabled=None, trace_sqlalchemy=None, hmac_keys=None,
connection_string=None, es_doc_type=None,
es_scroll_time=None, es_scroll_size=None,
socket_timeout=None, sentinel_service_name=None):

def is_trace_enabled(conf=None):
def is_db_trace_enabled(conf=None):
def enable_web_trace(conf=None):
def disable_web_trace(conf=None):
def list_opts():

Keystone 启用 OSProfiler

  1. 数据库会话包装
  2. flask WSGI 中间件包装
1
2
3
4
5
common/profiler.py    # 读取配置文件,设置通知驱动,启用中间件
common/sql/core.py # 包装数据库会话
conf/__init__.py # 默认配置
server/flask/core.py # 初始化,调用 common/profiler.py
setup.cfg # 设置中间件 WsgiMiddleware

common/sql/core.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 使用 osprofiler 模块包装会话
def _wrap_session(sess):
if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy:
sess = osprofiler.sqlalchemy.wrap_session(sql, sess)
return sess

# 读
def session_for_read():
if _TESTING_USE_GLOBAL_CONTEXT_MANAGER:
reader = enginefacade.reader
else:
reader = _get_main_context_manager().reader
return _wrap_session(reader.using(_get_context()))

# 写
def session_for_write():
if _TESTING_USE_GLOBAL_CONTEXT_MANAGER:
writer = enginefacade.writer
else:
writer = _get_main_context_manager().writer
return _wrap_session(writer.using(_get_context()))

setup.cfg 设置 server 中间件

1
2
3
4
5
6
7
keystone.server_middleware =
cors = oslo_middleware:CORS
sizelimit = oslo_middleware:RequestBodySizeLimiter
http_proxy_to_wsgi = oslo_middleware:HTTPProxyToWSGI
osprofiler = osprofiler.web:WsgiMiddleware <--- 使用 osprofiler 的 WSGI 中间件
request_id = oslo_middleware:RequestId
debug = oslo_middleware:Debug

Nova 启用 OSProfiler

  1. 追踪数据库调用
  2. 类追踪装饰器 @profiler.trace_cls
  3. 公有方法装饰器
  • 组件之间 REST API 并遵循 AMQP 协议
  • 组件内部 RPC
1
2
3
4
5
6
7
config.py             # 根据配置文件设置 osprofiler
service.py # 根据配置文件设置驱动,启用 WSGI 中间件
profiler.py # 重写 WSGI 中间件类、类追踪装饰器x
manager.py # 公有方法装饰器(抽象基类)
rpc.py # 追踪上下文序列化/反序列化
utils.py # spawn/spawn_n 装饰器,传递上下文
db/sqlalchemy/api.py # 追踪数据库调用

db/sqlalchemy/api.py

1
2
3
4
5
6
7
8
9
10
11
12
# 追踪数据库调用
def configure(conf):
main_context_manager.configure(**_get_db_conf(conf.database))
api_context_manager.configure(**_get_db_conf(conf.api_database))

if profiler_sqlalchemy and CONF.profiler.enabled \
and CONF.profiler.trace_sqlalchemy:

main_context_manager.append_on_engine_create(
lambda eng: profiler_sqlalchemy.add_tracing(sa, eng, "db")) # 添加追踪
api_context_manager.append_on_engine_create(
lambda eng: profiler_sqlalchemy.add_tracing(sa, eng, "db")) # 添加追踪

使用 osprofiler 类装饰器(compute/api.py)

1
2
3
4
@profiler.trace_cls("compute_api")
class API(base.Base):
"""API for interacting with the compute manager."""
...

公有方法装饰器(抽象类),nova 组件内部的模块都继承 Manager 类,支持追踪

  • ComputeManagerConductorManagerSchedulerManager
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# 公有方法装饰器
class ManagerMeta(profiler.get_traced_meta(), type(PeriodicTasks)):
"""Metaclass to trace all children of a specific class.

This metaclass wraps every public method (not starting with _ or __)
of the class using it. All children classes of the class using ManagerMeta
will be profiled as well.

Adding this metaclass requires that the __trace_args__ attribute be added
to the class we want to modify. That attribute is a dictionary
with one mandatory key: "name". "name" defines the name
of the action to be traced (for example, wsgi, rpc, db).

The OSprofiler-based tracing, although, will only happen if profiler
instance was initiated somewhere before in the thread, that can only happen
if profiling is enabled in nova.conf and the API call to Nova API contained
specific headers.
"""

# 抽象基类
class Manager(base.Base, PeriodicTasks, metaclass=ManagerMeta):
__trace_args__ = {"name": "rpc"} # 必须
...

线程间传递上下文

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# 追踪请求上下文
class ProfilerRequestContextSerializer(RequestContextSerializer):
def serialize_context(self, context):
_context = super(ProfilerRequestContextSerializer,
self).serialize_context(context)

prof = profiler.get() # 获取 osprofiler 实例
if prof:
# FIXME(DinaBelova): we'll add profiler.get_info() method
# to extract this info -> we'll need to update these lines
trace_info = {
"hmac_key": prof.hmac_key,
"base_id": prof.get_base_id(),
"parent_id": prof.get_id()
}
_context.update({"trace_info": trace_info}) # 添加追踪信息

return _context

def deserialize_context(self, context):
trace_info = context.pop("trace_info", None)
if trace_info:
profiler.init(**trace_info) # 初始化 osprofiler 实例

return super(ProfilerRequestContextSerializer,
self).deserialize_context(context)

参阅