b6d644cba70b902314c31a50a7b355d2fe79ad93
[ccsdk/cds.git] / ms / command-executor / src / main / python / command_executor_handler.py
1 #
2 # Copyright (C) 2019 - 2020 Bell Canada.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #      http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 #
16 from builtins import Exception, open, dict
17 from subprocess import CalledProcessError, PIPE, TimeoutExpired
18 from google.protobuf.json_format import MessageToJson
19 import tempfile
20 import logging
21 import os
22 import sys
23 import re
24 import subprocess
25 import virtualenv
26 import venv
27 import utils
28 import proto.CommandExecutor_pb2 as CommandExecutor_pb2
29 from zipfile import ZipFile
30 import io
31 import time
32 import prometheus_client as prometheus
33
34 REQUIREMENTS_TXT = "requirements.txt"
35
36
37 class CommandExecutorHandler():
38     BLUEPRINTS_DEPLOY_DIR = '/opt/app/onap/blueprints/deploy/'
39     TOSCA_META_FILE = 'TOSCA-Metadata/TOSCA.meta'
40     PROMETHEUS_METRICS_UPLOAD_CBA_LABEL = 'upload_cba'
41     PROMETHEUS_METRICS_PREP_ENV_LABEL = 'prepare_env'
42     PROMETHEUS_METRICS_EXEC_COMMAND_LABEL = 'execute_command'
43
44     def __init__(self, request):
45         self.request = request
46         self.logger = logging.getLogger(self.__class__.__name__)
47         self.blueprint_name = utils.get_blueprint_name(request)
48         self.blueprint_version = utils.get_blueprint_version(request)
49         self.uuid = utils.get_blueprint_uuid(request)
50         self.request_id = utils.get_blueprint_requestid(request)
51         self.sub_request_id = utils.get_blueprint_subRequestId(request)
52         self.blueprint_name_version = utils.blueprint_name_version(request) # for legacy support
53         self.blueprint_name_version_uuid = utils.blueprint_name_version_uuid(request)
54         self.execution_timeout = utils.get_blueprint_timeout(request)
55         # onap/blueprints/deploy will be ephemeral now
56         # if the command matches “/opt/app/onap/blueprints/deploy/$cba_name/$cba_version/stuff_that_is_not_cba_uuid/”
57         # then prepend the $cba_uuid before “stuff_that_is_not_cba_uuid”
58         self.blueprint_dir = self.BLUEPRINTS_DEPLOY_DIR + self.blueprint_name_version_uuid
59         self.blueprint_tosca_meta_file = self.blueprint_dir + '/' + self.TOSCA_META_FILE
60         self.extra = utils.getExtraLogData(request)
61         self.installed = self.blueprint_dir + '/.installed'
62         self.prometheus_histogram = self.get_prometheus_histogram()
63         self.prometheus_counter = self.get_prometheus_counter()
64         self.start_prometheus_server()
65
66     def get_prometheus_histogram(self):
67         histogram = getattr(prometheus.REGISTRY, '_command_executor_histogram', None)
68         if not histogram:
69             histogram = prometheus.Histogram('cds_ce_execution_duration_seconds',
70                              'How many times CE actions (upload, prepare env and execute) got executed and how long it took to complete for each CBA python script.',
71                              ['step', 'blueprint_name', 'blueprint_version', 'script_name'])
72             prometheus.REGISTRY._command_executor_histogram = histogram
73         return histogram
74
75     def get_prometheus_counter(self):
76         counter = getattr(prometheus.REGISTRY, '_command_executor_counter', None)
77         if not counter:
78             counter = prometheus.Counter('cds_ce_execution_error_total',
79                               'How many times CE actions (upload, prepare env and execute) got executed and failed for each CBA python script',
80                               ['step', 'blueprint_name', 'blueprint_version', 'script_name'])
81             prometheus.REGISTRY._command_executor_counter = counter
82         return counter
83
84     def start_prometheus_server(self):
85         self.logger.info("PROMETHEUS_METRICS_ENABLED: {}".format(os.environ.get('PROMETHEUS_METRICS_ENABLED')), extra=self.extra)
86         if (os.environ.get('PROMETHEUS_METRICS_ENABLED')):
87            if not "PROMETHEUS_PORT" in os.environ:
88               err_msg = "ERROR: failed to start prometheus server, PROMETHEUS_PORT env variable is not found."
89               self.logger.error(err_msg, extra=self.extra)
90               return utils.build_ret_data(False, results_log=[], error=err_msg)
91
92            server_started = getattr(prometheus.REGISTRY, '_command_executor_prometheus_server_started', None)
93            if not server_started:
94                self.logger.info("PROMETHEUS_PORT: {}".format(os.environ.get('PROMETHEUS_PORT')), extra=self.extra)
95                prometheus.start_http_server(int(os.environ.get('PROMETHEUS_PORT')))
96                prometheus.REGISTRY._command_executor_prometheus_server_started = True
97
98     def is_installed(self):
99         return os.path.exists(self.installed)
100
101     def blueprint_dir_exists(self):
102         return os.path.exists(self.blueprint_dir)
103
104     # used to validate if the blueprint actually had a chace of getting uploaded
105     def blueprint_tosca_meta_file_exists(self):
106         return os.path.exists(self.blueprint_tosca_meta_file)
107
108     def err_exit(self, msg):
109         self.logger.error(msg, extra=self.extra)
110         return utils.build_ret_data(False, error=msg)
111     
112     def is_valid_archive_type(self, archiveType):
113         return archiveType=="CBA_ZIP" or archiveType=="CBA_GZIP"
114
115     # Handle uploading blueprint request
116     # accept UploadBlueprintInput (CommandExecutor.proto) struct
117     # create dir blueprintName/BlueprintVersion/BlueprintUUID, and extract binData as either ZIP file or GZIP
118     # based on archiveType field...
119     def uploadBlueprint(self, request):
120         start_time = time.time()
121         archive_type = request.archiveType
122         compressed_cba_stream = io.BytesIO(request.binData)
123         self.logger.info("uploadBlueprint request {}".format(request), extra=self.extra)
124         if not self.is_valid_archive_type(archive_type):
125             self.prometheus_counter.labels(self.PROMETHEUS_METRICS_UPLOAD_CBA_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
126             return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, False, ["Archive type {} is not valid.".format(archive_type)])
127         
128         # create the BP dir self.blueprint_dir
129         try:
130             os.makedirs(name=self.blueprint_dir, mode=0o755, exist_ok=True)
131         except OSError as ex:
132             self.prometheus_counter.labels(self.PROMETHEUS_METRICS_UPLOAD_CBA_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
133             err_msg = "Failed to create blueprint dir: {} exception message: {}".format(self.blueprint_dir, ex.strerror)
134             self.logger.error(err_msg, extra=self.extra)
135             return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, False, [err_msg])
136         if archive_type=="CBA_ZIP":
137             self.logger.info("Extracting ZIP data to dir {}".format(self.blueprint_dir), extra=self.extra)
138             try:
139                 with ZipFile(compressed_cba_stream,'r') as zipfile:
140                     zipfile.extractall(self.blueprint_dir)                    
141                 self.logger.info("Done extracting ZIP data to dir {}".format(self.blueprint_dir), extra=self.extra)
142             except (IOError, zipfile.error) as e:
143                 self.prometheus_counter.labels(self.PROMETHEUS_METRICS_UPLOAD_CBA_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
144                 err_msg = "Error extracting ZIP data to dir {} exception: {}".format(self.blueprint_dir, e)
145                 self.logger.error(err_msg, extra=self.extra)
146                 return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, False, [err_msg])
147         # TODO with an actual test gzip cba...
148         elif archive_type=="CBA_GZIP":
149             self.prometheus_counter.labels(self.PROMETHEUS_METRICS_UPLOAD_CBA_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
150             self.logger.error("CBA_GZIP TODO", extra=self.extra)
151             return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, False, ["Error extracting GZIP data to {} GZIP todo!".format(self.blueprint_dir)])
152         # Finally, everything is ok!
153         self.prometheus_histogram.labels(self.PROMETHEUS_METRICS_UPLOAD_CBA_LABEL, self.blueprint_name, self.blueprint_version, None).observe(time.time() - start_time)
154         return utils.build_grpc_blueprint_upload_response(self.request_id, self.sub_request_id, True, [])
155
156     def prepare_env(self, request):
157         results_log = []
158         start_time = time.time()
159
160         self.logger.info("prepare_env request {}".format(request), extra=self.extra)
161         # validate that the blueprint name in the request exists, if not, notify the caller
162         if not self.blueprint_dir_exists():
163             self.prometheus_counter.labels(self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
164             err_msg = "CBA directory {} not found on cmd-exec. CBA will be uploaded by BP proc.".format(self.blueprint_name_version_uuid)
165             self.logger.info(err_msg, extra=self.extra)
166             return utils.build_ret_data(False, results_log=results_log, error=err_msg, reupload_cba=True)
167         if not self.blueprint_tosca_meta_file_exists():
168             self.prometheus_counter.labels(self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
169             err_msg = "CBA directory {} exists on cmd-exec, but TOSCA meta file is not found!!! Returning (null) as UUID. CBA will be uploaded by BP proc.".format(self.blueprint_name_version_uuid)
170             self.logger.info(err_msg, extra=self.extra)
171             return utils.build_ret_data(False, results_log=results_log, error=err_msg, reupload_cba=True)
172         self.logger.info("CBA directory {} exists on cmd-exec.".format(self.blueprint_name_version_uuid), extra=self.extra)
173
174         if not self.is_installed():
175             create_venv_status = self.create_venv()
176             if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]:
177                 self.prometheus_counter.labels(self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
178                 return self.err_exit("ERROR: failed to prepare environment for request {} due to error in creating virtual Python env. Original error {}".format(self.blueprint_name_version_uuid, create_venv_status[utils.ERR_MSG_KEY]))
179
180             activate_venv_status = self.activate_venv()
181             if not activate_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]:
182                 self.prometheus_counter.labels(self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
183                 return self.err_exit("ERROR: failed to prepare environment for request {} due Python venv_activation. Original error {}".format(self.blueprint_name_version_uuid, activate_venv_status[utils.ERR_MSG_KEY]))
184
185             try:
186                 with open(self.installed, "w+") as f:
187                     if not self.install_packages(request, CommandExecutor_pb2.pip, f, results_log):
188                         self.prometheus_counter.labels(self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
189                         err_msg = "ERROR: failed to prepare environment for request {} during pip package install.".format(self.blueprint_name_version_uuid)
190                         return utils.build_ret_data(False, results_log=results_log, error=err_msg)
191                     f.write("\r\n") # TODO: is \r needed?
192                     results_log.append("\n")
193                     if not self.install_packages(request, CommandExecutor_pb2.ansible_galaxy, f, results_log):
194                         self.prometheus_counter.labels(self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
195                         err_msg = "ERROR: failed to prepare environment for request {} during Ansible install.".format(self.blueprint_name_version_uuid)
196                         return utils.build_ret_data(False, results_log=results_log, error=err_msg)
197             except Exception as ex:
198                 self.prometheus_counter.labels(self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
199                 err_msg = "ERROR: failed to prepare environment for request {} during installing packages. Exception: {}".format(self.blueprint_name_version_uuid, ex)
200                 self.logger.error(err_msg, extra=self.extra)
201                 return utils.build_ret_data(False, error=err_msg)
202         else:
203             try:
204                 with open(self.installed, "r") as f:
205                     results_log.append(f.read())
206             except Exception as ex:
207                 self.prometheus_counter.labels(self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc()
208                 err_msg="ERROR: failed to prepare environment during reading 'installed' file {}. Exception: {}".format(self.installed, ex)
209                 return utils.build_ret_data(False, error=err_msg)
210
211         # deactivate_venv(blueprint_id)
212         self.prometheus_histogram.labels(self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).observe(time.time() - start_time)
213         return utils.build_ret_data(True, results_log=results_log)
214
215     def execute_command(self, request):
216         start_time = time.time()
217         # STDOUT/STDERR output of the process
218         results_log = []
219         # encoded payload returned by the process
220         result = {}
221
222         self.logger.info("execute_command request {}".format(request), extra=self.extra)
223         # workaround for when packages are not specified, we may not want to go through the install step
224         # can just call create_venv from here.
225         if not self.is_installed():
226             self.create_venv()
227         try:
228             if not self.is_installed():
229                 create_venv_status = self.create_venv
230                 if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]:
231                     self.prometheus_counter.labels(self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL, self.blueprint_name, self.blueprint_version, request.command).inc()
232                     err_msg = "{} - Failed to execute command during venv creation. Original error: {}".format(self.blueprint_name_version_uuid, create_venv_status[utils.ERR_MSG_KEY])
233                     return utils.build_ret_data(False, error=err_msg)
234             activate_response = self.activate_venv()
235             if not activate_response[utils.CDS_IS_SUCCESSFUL_KEY]:
236                 orig_error = activate_response[utils.ERR_MSG_KEY]
237                 err_msg = "{} - Failed to execute command during environment activation. Original error: {}".format(self.blueprint_name_version_uuid, orig_error)
238                 return utils.build_ret_data(False, error=err_msg)
239             # touch blueprint dir to indicate this CBA was used recently
240             os.utime(self.blueprint_dir)
241
242             cmd = "cd " + self.blueprint_dir
243
244             ### if properties are defined we add them to the command
245             properties = ""
246             if request.properties is not None and len(request.properties) > 0:
247                 properties = " " + re.escape(MessageToJson(request.properties)).replace('"','\\"')
248
249             # compatibility hack
250             # check if the path for the request.command does not contain UUID, then add it after cba_name/cba_version path.
251             updated_request_command = request.command
252             if self.blueprint_name_version in updated_request_command and self.blueprint_name_version_uuid not in updated_request_command:
253                 updated_request_command = updated_request_command.replace(self.blueprint_name_version, self.blueprint_name_version_uuid)
254
255             ### TODO: replace with os.environ['VIRTUAL_ENV']?
256             if "ansible-playbook" in updated_request_command:
257                 cmd = cmd + "; " + updated_request_command + " -e 'ansible_python_interpreter=" + self.blueprint_dir + "/bin/python'"
258             else:
259                 cmd = cmd + "; " + updated_request_command + properties
260
261             ### extract the original header request into sys-env variables
262             ### OriginatorID
263             originator_id = request.originatorId
264             ### CorrelationID
265             correlation_id = request.correlationId
266             request_id_map = {'CDS_REQUEST_ID':self.request_id, 'CDS_SUBREQUEST_ID':self.sub_request_id, 'CDS_ORIGINATOR_ID': originator_id, 'CDS_CORRELATION_ID': correlation_id}
267             updated_env =  { **os.environ, **request_id_map }
268             self.logger.info("Running blueprint {} with timeout: {}".format(self.blueprint_name_version_uuid, self.execution_timeout), extra=self.extra)
269
270             with tempfile.TemporaryFile(mode="w+") as tmp:
271                 try:
272                     completed_subprocess = subprocess.run(cmd, stdout=tmp, stderr=subprocess.STDOUT, shell=True,
273                                                 env=updated_env, timeout=self.execution_timeout)
274                 except TimeoutExpired:
275                     self.prometheus_counter.labels(self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL, self.blueprint_name, self.blueprint_version, request.command).inc()
276                     timeout_err_msg = "Running command {} failed due to timeout of {} seconds.".format(self.blueprint_name_version_uuid, self.execution_timeout)
277                     self.logger.error(timeout_err_msg, extra=self.extra)
278                     utils.parse_cmd_exec_output(tmp, self.logger, result, results_log, self.extra)
279                     return utils.build_ret_data(False, results_log=results_log, error=timeout_err_msg)
280
281                 utils.parse_cmd_exec_output(tmp, self.logger, result, results_log, self.extra)
282                 rc = completed_subprocess.returncode
283         except Exception as e:
284             self.prometheus_counter.labels(self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL, self.blueprint_name, self.blueprint_version, request.command).inc()
285             err_msg = "{} - Failed to execute command. Error: {}".format(self.blueprint_name_version_uuid, e)
286             result.update(utils.build_ret_data(False, results_log=results_log, error=err_msg))
287             return result
288
289         # deactivate_venv(blueprint_id)
290         #Since return code is only used to check if it's zero (success), we can just return success flag instead.
291         is_execution_successful = rc == 0
292         result.update(utils.build_ret_data(is_execution_successful, results_log=results_log))
293         self.prometheus_histogram.labels(self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL, self.blueprint_name, self.blueprint_version, request.command).observe(time.time() - start_time)
294
295         return result
296
297     def install_packages(self, request, type, f, results):
298         success = self.install_python_packages('UTILITY', results)
299         if not success:
300             self.logger.error("Error installing 'UTILITY (cds_utils) package to CBA python environment!!!", extra=self.extra)
301             return False
302
303         for package in request.packages:
304             if package.type == type:
305                 f.write("Installed %s packages:\r\n" % CommandExecutor_pb2.PackageType.Name(type))
306                 for p in package.package:
307                     f.write("   %s\r\n" % p)
308                     if package.type == CommandExecutor_pb2.pip:
309                         success = self.install_python_packages(p, results)
310                     else:
311                         success = self.install_ansible_packages(p, results)
312                     if not success:
313                         f.close()
314                         os.remove(self.installed)
315                         return False
316         return True
317
318     def install_python_packages(self, package, results):
319         self.logger.info(
320             "{} - Install Python package({}) in Python Virtual Environment".format(self.blueprint_name_version_uuid, package), extra=self.extra)
321
322         if REQUIREMENTS_TXT == package:
323             command = ["pip", "install", "--user", "-r", self.blueprint_dir + "/Environments/" + REQUIREMENTS_TXT]
324         elif package == 'UTILITY':
325             py_ver_maj = sys.version_info.major
326             py_ver_min = sys.version_info.minor
327             command = ["cp", "-r", "./cds_utils", "{}/lib/python{}.{}/site-packages/".format(self.blueprint_dir, py_ver_maj,py_ver_min)]
328         else:
329             command = ["pip", "install", "--user", package]
330
331         env = dict(os.environ)
332         if "https_proxy" in os.environ:
333             env['https_proxy'] = os.environ['https_proxy']
334             self.logger.info("Using https_proxy: {}".format(env['https_proxy']), extra=self.extra)
335         try:
336             results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode())
337             results.append("\n")
338             self.logger.info("install_python_packages {} succeeded".format(package), extra=self.extra)
339             return True
340         except CalledProcessError as e:
341             results.append(e.stderr.decode())
342             self.logger.error("install_python_packages {} failed".format(package), extra=self.extra)
343             return False
344
345     def install_ansible_packages(self, package, results):
346         self.logger.info(
347             "{} - Install Ansible Role package({}) in Python Virtual Environment".format(self.blueprint_name_version_uuid, package), extra=self.extra)
348         command = ["ansible-galaxy", "install", package, "-p", self.blueprint_dir + "/Scripts/ansible/roles"]
349
350         env = dict(os.environ)
351         if "http_proxy" in os.environ:
352             # ansible galaxy uses https_proxy environment variable, but requires it to be set with http proxy value.
353             env['https_proxy'] = os.environ['http_proxy']
354         try:
355             results.append(subprocess.run(command, check=True, stdout=PIPE, stderr=PIPE, env=env).stdout.decode())
356             results.append("\n")
357             return True
358         except CalledProcessError as e:
359             results.append(e.stderr.decode())
360             return False
361
362     # Returns a map with 'status' and 'err_msg'.
363     # 'status' True indicates success.
364     # 'err_msg' indicates an error occurred. The presence of err_msg may not be fatal,
365     # status should be set to False for fatal errors.
366     def create_venv(self):
367         self.logger.info("{} - Create Python Virtual Environment".format(self.blueprint_name_version_uuid), extra=self.extra)
368         try:
369             bin_dir = self.blueprint_dir + "/bin"
370             # venv doesn't populate the activate_this.py script, hence we use from virtualenv
371             venv.create(self.blueprint_dir, with_pip=True, system_site_packages=True)
372             virtualenv.writefile(os.path.join(bin_dir, "activate_this.py"), virtualenv.ACTIVATE_THIS)
373             self.logger.info("{} - Creation of Python Virtual Environment finished.".format(self.blueprint_name_version_uuid), extra=self.extra)
374             return utils.build_ret_data(True)
375         except Exception as err:
376             err_msg = "{} - Failed to provision Python Virtual Environment. Error: {}".format(self.blueprint_name_version_uuid, err)
377             self.logger.info(err_msg, extra=self.extra)
378             return utils.build_ret_data(False, error=err_msg)
379
380     # return map cds_is_successful and err_msg. Status is True on success. err_msg may existence doesn't necessarily indicate fatal condition.
381     # the 'status' should be set to False to indicate error.
382     def activate_venv(self):
383         self.logger.info("{} - Activate Python Virtual Environment".format(self.blueprint_name_version_uuid), extra=self.extra)
384
385         # Fix: The python generated activate_this.py script concatenates the env bin dir to PATH on every call
386         #      eventually this process PATH variable was so big (128Kb) that no child process could be spawn
387         #      This script will remove all duplicates; while keeping the order of the PATH folders
388         fixpathenvvar = "os.environ['PATH']=os.pathsep.join(list(dict.fromkeys(os.environ['PATH'].split(':'))))"
389
390         path = "%s/bin/activate_this.py" % self.blueprint_dir
391         try:
392             with open(path) as activate_this_script:
393                 exec (activate_this_script.read(), {'__file__': path})
394             exec (fixpathenvvar)
395             self.logger.info("Running with PATH : {}".format(os.environ['PATH']), extra=self.extra)
396             return utils.build_ret_data(True)
397         except Exception as err:
398             err_msg ="{} - Failed to activate Python Virtual Environment. Error: {}".format(self.blueprint_name_version_uuid, err)
399             self.logger.info( err_msg, extra=self.extra)
400             return utils.build_ret_data(False, error=err_msg)
401
402     def deactivate_venv(self):
403         self.logger.info("{} - Deactivate Python Virtual Environment".format(self.blueprint_name_version_uuid), extra=self.extra)
404         command = ["deactivate"]
405         try:
406             subprocess.run(command, check=True)
407         except Exception as err:
408             self.logger.info(
409                 "{} - Failed to deactivate Python Virtual Environment. Error: {}".format(self.blueprint_name_version_uuid, err), extra=self.extra)
410
411