# Copyright 2012 OpenStack Foundation
# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import importlib
import inspect
import sys
from debtcollector import removals
from oslo_log import log as logging
import testtools
from tempest.lib import auth
from tempest.lib.common.utils import misc
from tempest.lib import exceptions
from tempest.lib.services import compute
from tempest.lib.services import identity
from tempest.lib.services import image
from tempest.lib.services import network
from tempest.lib.services import object_storage
from tempest.lib.services import placement
from tempest.lib.services import volume
LOG = logging.getLogger(__name__)
[docs]
def tempest_modules():
"""Dict of service client modules available in Tempest.
Provides a dict of stable service modules available in Tempest, with
``service_version`` as key, and the module object as value.
"""
return {
'compute': compute,
'placement': placement,
'identity.v2': identity.v2,
'identity.v3': identity.v3,
'image.v2': image.v2,
'network': network,
'object-storage': object_storage,
'volume.v2': volume.v2,
'volume.v3': volume.v3
}
[docs]
def available_modules():
"""Set of service client modules available in Tempest and plugins
Set of stable service clients from Tempest and service clients exposed
by plugins. This set of available modules can be used for automatic
configuration.
:raise PluginRegistrationException: if a plugin exposes a service_version
already defined by Tempest or another plugin.
Examples::
from tempest import config
params = {}
for service_version in available_modules():
service = service_version.split('.')[0]
params[service] = config.service_client_config(service)
service_clients = ServiceClients(creds, identity_uri,
client_parameters=params)
"""
extra_service_versions = set([])
_tempest_modules = set(tempest_modules())
plugin_services = ClientsRegistry().get_service_clients()
name_conflicts = []
for plugin_name in plugin_services:
plug_service_versions = set([x['service_version'] for x in
plugin_services[plugin_name]])
# If a plugin exposes a duplicate service_version raise an exception
if plug_service_versions:
if not plug_service_versions.isdisjoint(extra_service_versions):
detailed_error = (
'Plugin %s is trying to register a service %s already '
'claimed by another one' % (plugin_name,
extra_service_versions &
plug_service_versions))
name_conflicts.append(exceptions.PluginRegistrationException(
name=plugin_name, detailed_error=detailed_error))
extra_service_versions |= plug_service_versions
if name_conflicts:
LOG.error(
'Failed to list available modules due to name conflicts: %s',
name_conflicts)
raise testtools.MultipleExceptions(*name_conflicts)
return _tempest_modules | extra_service_versions
@misc.singleton
class ClientsRegistry(object):
"""Registry of all service clients available from plugins"""
def __init__(self):
self._service_clients = {}
def register_service_client(self, plugin_name, service_client_data):
if plugin_name in self._service_clients:
detailed_error = 'Clients for plugin %s already registered'
raise exceptions.PluginRegistrationException(
name=plugin_name,
detailed_error=detailed_error % plugin_name)
self._service_clients[plugin_name] = service_client_data
LOG.debug("Successfully registered plugin %s in the service client "
"registry with configuration: %s", plugin_name,
service_client_data)
def get_service_clients(self):
return self._service_clients
[docs]
class ClientsFactory(object):
"""Builds service clients for a service client module
This class implements the logic of feeding service client parameters
to service clients from a specific module. It allows setting the
parameters once and obtaining new instances of the clients without the
need of passing any parameter.
ClientsFactory can be used directly, or consumed via the `ServiceClients`
class, which manages the authorization part.
"""
def __init__(self, module_path, client_names, auth_provider, **kwargs):
"""Initialises the client factory
:param module_path: Path to module that includes all service clients.
All service client classes must be exposed by a single module.
If they are separated in different modules, defining __all__
in the root module can help, similar to what is done by service
clients in tempest.
:param client_names: List or set of names of the service client
classes.
:param auth_provider: The auth provider used to initialise client.
:param kwargs: Parameters to be passed to all clients. Parameters
values can be overwritten when clients are initialised, but
parameters cannot be deleted.
:raise ImportError: if the specified module_path cannot be imported
Example::
# Get credentials and an auth_provider
clients = ClientsFactory(
module_path='my_service.my_service_clients',
client_names=['ServiceClient1', 'ServiceClient2'],
auth_provider=auth_provider,
service='my_service',
region='region1')
my_api_client = clients.MyApiClient()
my_api_client_region2 = clients.MyApiClient(region='region2')
"""
# Import the module. If it's not importable, the raised exception
# provides good enough information about what happened
_module = importlib.import_module(module_path)
# If any of the classes is not in the module we fail
for class_name in client_names:
# TODO(andreaf) This always passes all parameters to all clients.
# In future to allow clients to specify the list of parameters
# that they accept based out of a list of standard ones.
# Obtain the class
klass = self._get_class(_module, class_name)
final_kwargs = copy.copy(kwargs)
# Set the function as an attribute of the factory
setattr(self, class_name, self._get_partial_class(
klass, auth_provider, final_kwargs))
def _get_partial_class(self, klass, auth_provider, kwargs):
# Define a function that returns a new class instance by
# combining default kwargs with extra ones
def partial_class(alias=None, **later_kwargs):
"""Returns a callable the initialises a service client
Builds a callable that accepts kwargs, which are passed through
to the __init__ of the service client, along with a set of defaults
set in factory at factory __init__ time.
Original args in the service client can only be passed as kwargs.
It accepts one extra parameter 'alias' compared to the original
service client. When alias is provided, the returned callable will
also set an attribute called with a name defined in 'alias', which
contains the instance of the service client.
:param alias: str Name of the attribute set on the factory once
the callable is invoked which contains the initialised
service client. If None, no attribute is set.
:param later_kwargs: kwargs passed through to the service client
__init__ on top of defaults set at factory level.
"""
kwargs.update(later_kwargs)
_client = klass(auth_provider=auth_provider, **kwargs)
if alias:
setattr(self, alias, _client)
return _client
return partial_class
@classmethod
def _get_class(cls, module, class_name):
klass = getattr(module, class_name, None)
if not klass:
msg = 'Invalid class name, %s is not found in %s'
raise AttributeError(msg % (class_name, module))
if not inspect.isclass(klass):
msg = 'Expected a class, got %s of type %s instead'
raise TypeError(msg % (klass, type(klass)))
return klass
[docs]
class ServiceClients(object):
"""Service client provider class
The ServiceClients object provides a useful means for tests to access
service clients configured for a specified set of credentials.
It hides some of the complexity from the authorization and configuration
layers.
Examples::
# johndoe is a tempest.lib.auth.Credentials type instance
johndoe_clients = clients.ServiceClients(johndoe, identity_uri)
# List servers in default region
johndoe_servers_client = johndoe_clients.compute.ServersClient()
johndoe_servers = johndoe_servers_client.list_servers()
# List servers in Region B
johndoe_servers_client_B = johndoe_clients.compute.ServersClient(
region='B')
johndoe_servers = johndoe_servers_client_B.list_servers()
"""
# NOTE(andreaf) This class does not depend on tempest configuration
# and its meant for direct consumption by external clients such as tempest
# plugins. Tempest provides a wrapper class, `clients.Manager`, that
# initialises this class using values from tempest CONF object. The wrapper
# class should only be used by tests hosted in Tempest.
@removals.removed_kwarg('client_parameters')
def __init__(self, credentials, identity_uri, region=None, scope=None,
disable_ssl_certificate_validation=True, ca_certs=None,
trace_requests='', client_parameters=None, proxy_url=None):
"""Service Clients provider
Instantiate a `ServiceClients` object, from a set of credentials and an
identity URI. The identity version is inferred from the credentials
object. Optionally auth scope can be provided.
A few parameters can be given a value which is applied as default
for all service clients: region, dscv, ca_certs, trace_requests.
Parameters dscv, ca_certs and trace_requests all apply to the auth
provider as well as any service clients provided by this manager.
Any other client parameter should be set via ClientsRegistry.
Client parameter used to be set via client_parameters, but this is
deprecated, and it is actually already not honoured
anymore: https://launchpad.net/bugs/1680915.
The list of available parameters is defined in the service clients
interfaces. For reference, most clients will accept 'region',
'service', 'endpoint_type', 'build_timeout' and 'build_interval', which
are all inherited from RestClient.
The `config` module in Tempest exposes an helper function
`service_client_config` that can be used to extract from configuration
a dictionary ready to be injected in kwargs.
Exceptions are:
- Token clients for 'identity' must be given an 'auth_url' parameter
- Volume client for 'volume' accepts 'default_volume_size'
- Servers client from 'compute' accepts 'enable_instance_password'
If Tempest configuration is used, parameters will be loaded in the
Registry automatically for all service client (Tempest stable ones
and plugins).
Examples::
identity_params = config.service_client_config('identity')
params = {
'identity': identity_params,
'compute': {'region': 'region2'}}
manager = lib_manager.Manager(
my_creds, identity_uri, client_parameters=params)
:param credentials: An instance of `auth.Credentials`
:param identity_uri: URI of the identity API. This should be a
mandatory parameter, and it will so soon.
:param region: Default value of region for service clients.
:param scope: default scope for tokens produced by the auth provider
:param disable_ssl_certificate_validation: Applies to auth and to all
service clients.
:param ca_certs: Applies to auth and to all service clients.
:param trace_requests: Applies to auth and to all service clients.
:param client_parameters: Dictionary with parameters for service
clients. Keys of the dictionary are the service client service
name, as declared in `service_clients.available_modules()` except
for the version. Values are dictionaries of parameters that are
going to be passed to all clients in the service client module.
:param proxy_url: Applies to auth and to all service clients, set a
proxy url for the clients to use.
"""
self._registered_services = set([])
self.credentials = credentials
self.identity_uri = identity_uri
if not identity_uri:
raise exceptions.InvalidCredentials(
'ServiceClients requires a non-empty identity_uri.')
self.region = region
# Check if passed or default credentials are valid
if not self.credentials.is_valid():
raise exceptions.InvalidCredentials(credentials)
# Get the identity classes matching the provided credentials
# TODO(andreaf) Define a new interface in Credentials to get
# the API version from an instance
identity = [(k, auth.IDENTITY_VERSION[k][1]) for k in
auth.IDENTITY_VERSION.keys() if
isinstance(self.credentials, auth.IDENTITY_VERSION[k][0])]
# Zero matches or more than one are both not valid.
if len(identity) != 1:
msg = "Zero or %d ambiguous auth provider found. identity: %s, " \
"credentials: %s" % (len(identity), identity, credentials)
raise exceptions.InvalidCredentials(msg)
self.auth_version, auth_provider_class = identity[0]
self.dscv = disable_ssl_certificate_validation
self.ca_certs = ca_certs
self.trace_requests = trace_requests
self.proxy_url = proxy_url
if self.credentials.project_id or self.credentials.project_name:
scope = 'project'
elif self.credentials.system:
scope = 'system'
elif self.credentials.domain_id or self.credentials.domain_name:
scope = 'domain'
else:
scope = 'project'
# Creates an auth provider for the credentials
self.auth_provider = auth_provider_class(
self.credentials, self.identity_uri, scope=scope,
disable_ssl_certificate_validation=self.dscv,
ca_certs=self.ca_certs, trace_requests=self.trace_requests,
proxy_url=proxy_url)
# Setup some defaults for client parameters of registered services
client_parameters = client_parameters or {}
self.parameters = {}
# Parameters are provided for unversioned services
all_modules = available_modules()
unversioned_services = set(
[x.split('.')[0] for x in all_modules])
for service in unversioned_services:
self.parameters[service] = self._setup_parameters(
client_parameters.pop(service, {}))
# Check that no client parameters was supplied for unregistered clients
if client_parameters:
raise exceptions.UnknownServiceClient(
services=list(client_parameters.keys()))
# Register service clients from the registry (__tempest__ and plugins)
clients_registry = ClientsRegistry()
plugin_service_clients = clients_registry.get_service_clients()
registration_errors = []
for plugin in plugin_service_clients:
service_clients = plugin_service_clients[plugin]
# Each plugin returns a list of service client parameters
for service_client in service_clients:
# NOTE(andreaf) If a plugin cannot register, stop the
# registration process, log some details to help
# troubleshooting, and re-raise
try:
self.register_service_client_module(**service_client)
except Exception:
registration_errors.append(sys.exc_info())
LOG.exception(
'Failed to register service client from plugin %s '
'with parameters %s', plugin, service_client)
if registration_errors:
raise testtools.MultipleExceptions(*registration_errors)
def register_service_client_module(self, name, service_version,
module_path, client_names, **kwargs):
"""Register a service client module
Initiates a client factory for the specified module, using this
class auth_provider, and accessible via a `name` attribute in the
service client.
:param name: Name used to access the client
:param service_version: Name of the service complete with version.
Used to track registered services. When a plugin implements it,
it can be used by other plugins to obtain their configuration.
:param module_path: Path to module that includes all service clients.
All service client classes must be exposed by a single module.
If they are separated in different modules, defining __all__
in the root module can help, similar to what is done by service
clients in tempest.
:param client_names: List or set of names of service client classes.
:param kwargs: Extra optional parameters to be passed to all clients.
ServiceClient provides defaults for region, dscv, ca_certs, http
proxies and trace_requests.
:raise ServiceClientRegistrationException: if the provided name is
already in use or if service_version is already registered.
:raise ImportError: if module_path cannot be imported.
"""
if hasattr(self, name):
using_name = getattr(self, name)
detailed_error = 'Module name already in use: %s' % using_name
raise exceptions.ServiceClientRegistrationException(
name=name, service_version=service_version,
module_path=module_path, client_names=client_names,
detailed_error=detailed_error)
if service_version in self.registered_services:
detailed_error = 'Service %s already registered.' % service_version
raise exceptions.ServiceClientRegistrationException(
name=name, service_version=service_version,
module_path=module_path, client_names=client_names,
detailed_error=detailed_error)
params = dict(region=self.region,
disable_ssl_certificate_validation=self.dscv,
ca_certs=self.ca_certs,
trace_requests=self.trace_requests,
proxy_url=self.proxy_url)
params.update(kwargs)
# Instantiate the client factory
_factory = ClientsFactory(module_path=module_path,
client_names=client_names,
auth_provider=self.auth_provider,
**params)
# Adds the client factory to the service_client
setattr(self, name, _factory)
# Add the name of the new service in self.SERVICES for discovery
self._registered_services.add(service_version)
@property
def registered_services(self):
return self._registered_services
def _setup_parameters(self, parameters):
"""Setup default values for client parameters
Region by default is the region passed as an __init__ parameter.
Checks that no parameter for an unknown service is provided.
"""
_parameters = {}
# Use region from __init__
if self.region:
_parameters['region'] = self.region
# Update defaults with specified parameters
_parameters.update(parameters)
# If any parameter is left, parameters for an unknown service were
# provided as input. Fail rather than ignore silently.
return _parameters
