From 50ffbc20b6e236581becbe66c9c33af8c0550451 Mon Sep 17 00:00:00 2001 From: liangke Date: Thu, 15 Mar 2018 17:00:43 +0800 Subject: [PATCH] Fix framework logging format Update the logging format. Add context middleware to set MDC attributes. Allow framework to transmit request headers,So backend plugin could get "TransactionID" from framework. Change-Id: Ib918489a4a3e2069046471b42810fe4d1cbcb374 Issue-ID: MULTICLOUD-151 Signed-off-by: liangke --- multivimbroker/multivimbroker/forwarder/views.py | 15 +++-- multivimbroker/multivimbroker/middleware.py | 64 ++++++++++++++++++++++ multivimbroker/multivimbroker/pub/config/config.py | 5 ++ multivimbroker/multivimbroker/pub/config/log.yml | 8 +-- multivimbroker/multivimbroker/pub/utils/syscomm.py | 8 +++ multivimbroker/multivimbroker/settings.py | 1 + multivimbroker/requirements.txt | 2 +- 7 files changed, 93 insertions(+), 10 deletions(-) create mode 100644 multivimbroker/multivimbroker/middleware.py diff --git a/multivimbroker/multivimbroker/forwarder/views.py b/multivimbroker/multivimbroker/forwarder/views.py index cea4134..c77fe94 100644 --- a/multivimbroker/multivimbroker/forwarder/views.py +++ b/multivimbroker/multivimbroker/forwarder/views.py @@ -20,6 +20,7 @@ from rest_framework.views import APIView from rest_framework.views import Response from rest_framework.views import status from multivimbroker.forwarder.base import BaseHandler +from multivimbroker.pub.utils.syscomm import originHeaders class BaseServer(BaseHandler, APIView): @@ -48,18 +49,21 @@ class Identity(BaseServer): def get(self, request, vimid): - return self.send(vimid, request.get_full_path(), request.body, "GET") + return self.send(vimid, request.get_full_path(), request.body, "GET", + headers=originHeaders(request)) def post(self, request, vimid): - return self.send(vimid, request.get_full_path(), request.body, "POST") + return self.send(vimid, request.get_full_path(), request.body, "POST", + headers=originHeaders(request)) class Registry(BaseServer): def post(self, request, vimid): - return self.send(vimid, request.get_full_path(), request.body, "POST") + return self.send(vimid, request.get_full_path(), request.body, "POST", + headers=originHeaders(request)) class UnRegistry(BaseServer): @@ -67,14 +71,15 @@ class UnRegistry(BaseServer): def delete(self, request, vimid): return self.send(vimid, request.get_full_path(), request.body, - "DELETE") + "DELETE", headers=originHeaders(request)) class Extension(BaseServer): def get(self, request, vimid): - return self.send(vimid, request.get_full_path(), request.body, "GET") + return self.send(vimid, request.get_full_path(), request.body, "GET", + headers=originHeaders(request)) class VIMTypes(BaseServer): diff --git a/multivimbroker/multivimbroker/middleware.py b/multivimbroker/multivimbroker/middleware.py new file mode 100644 index 0000000..5b320a3 --- /dev/null +++ b/multivimbroker/multivimbroker/middleware.py @@ -0,0 +1,64 @@ +# Copyright (c) 2017-2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + +import uuid +from onaplogging.mdcContext import MDC +from multivimbroker.pub.config.config import SERVICE_NAME +from multivimbroker.pub.config.config import FORWARDED_FOR_FIELDS + + +class LogContextMiddleware(object): + + # the last IP behind multiple proxies, if no exist proxies + # get local host ip. + def _getLastIp(self, request): + + ip = "" + try: + for field in FORWARDED_FOR_FIELDS: + if field in request.META: + if ',' in request.META[field]: + parts = request.META[field].split(',') + ip = parts[-1].strip().split(":")[0] + else: + ip = request.META[field].split(":")[0] + + if ip == "": + ip = request.META.get("HTTP_HOST").split(":")[0] + + except Exception: + pass + + return ip + + def process_request(self, request): + + # Fetch TRANSACTIONID Id and pass to plugin server + ReqeustID = request.META.get("HTTP_X_TRANSACTIONID", None) + if ReqeustID is None: + ReqeustID = uuid.uuid3(uuid.NAMESPACE_URL, SERVICE_NAME) + request.META["HTTP_X_TRANSACTIONID"] = ReqeustID + MDC.put("requestID", ReqeustID) + # generate the unique id + InovocationID = uuid.uuid3(uuid.NAMESPACE_DNS, SERVICE_NAME) + MDC.put("invocationID", InovocationID) + MDC.put("serviceName", SERVICE_NAME) + # access ip + MDC.put("serviceIP", self._getLastIp(request)) + + return None + + def process_response(self, request, response): + + MDC.clear() + return response diff --git a/multivimbroker/multivimbroker/pub/config/config.py b/multivimbroker/multivimbroker/pub/config/config.py index b849f44..192c743 100644 --- a/multivimbroker/multivimbroker/pub/config/config.py +++ b/multivimbroker/multivimbroker/pub/config/config.py @@ -28,6 +28,11 @@ AAI_SCHEMA_VERSION = "v13" AAI_USERNAME = 'AAI' AAI_PASSWORD = 'AAI' +# [MDC] +SERVICE_NAME = "multicloud-broker" +FORWARDED_FOR_FIELDS = ["HTTP_X_FORWARDED_FOR", "HTTP_X_FORWARDED_HOST", + "HTTP_X_FORWARDED_SERVER"] + # [IMAGE LOCAL PATH] ROOT_PATH = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/multivimbroker/multivimbroker/pub/config/log.yml b/multivimbroker/multivimbroker/pub/config/log.yml index 12da69f..09be40d 100644 --- a/multivimbroker/multivimbroker/pub/config/log.yml +++ b/multivimbroker/multivimbroker/pub/config/log.yml @@ -12,14 +12,14 @@ handlers: class: "logging.handlers.RotatingFileHandler" filename: "/var/log/onap/multicloud/multivimbroker/multivimbroker.log" formatter: "mdcFormat" - maxBytes: 1024*1024*50 + maxBytes: 52428800 backupCount: 10 formatters: standard: - format: "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s" + format: "%(asctime)s|||||%(name)s||%(thread)||%(funcName)s||%(levelname)s||%(message)s" mdcFormat: - format: "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:[%(mdc)s]: %(message)s" - mdcfmt: "{requestID}" + format: "%(asctime)s|||||%(name)s||%(thread)s||%(funcName)s||%(levelname)s||%(message)s||||%(mdc)s \t" + mdcfmt: "{requestID} {invocationID} {serviceName} {serviceIP}" datefmt: "%Y-%m-%d %H:%M:%S" (): onaplogging.mdcformatter.MDCFormatter diff --git a/multivimbroker/multivimbroker/pub/utils/syscomm.py b/multivimbroker/multivimbroker/pub/utils/syscomm.py index 0b2b557..bd4bbb5 100644 --- a/multivimbroker/multivimbroker/pub/utils/syscomm.py +++ b/multivimbroker/multivimbroker/pub/utils/syscomm.py @@ -36,6 +36,14 @@ def getHeadersKeys(response): return [header for header in response.keys() if header not in hopbyhop] +# trim out 'HTTP_' prefix part and replace "_" wiht "-". +def originHeaders(request): + regex = re.compile('^HTTP_') + return dict((regex.sub('', header).replace("_", "-"), value) + for (header, value) in request.META.items() + if header.startswith('HTTP_')) + + def findMultivimDriver(vim=None): json_file = os.path.join(os.path.dirname(__file__), '../config/provider-plugin.json') diff --git a/multivimbroker/multivimbroker/settings.py b/multivimbroker/multivimbroker/settings.py index be887b9..8d1fba5 100644 --- a/multivimbroker/multivimbroker/settings.py +++ b/multivimbroker/multivimbroker/settings.py @@ -51,6 +51,7 @@ MIDDLEWARE_CLASSES = [ 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'multivimbroker.middleware.LogContextMiddleware', ] ROOT_URLCONF = 'multivimbroker.urls' diff --git a/multivimbroker/requirements.txt b/multivimbroker/requirements.txt index 1378612..ef24d49 100644 --- a/multivimbroker/requirements.txt +++ b/multivimbroker/requirements.txt @@ -24,4 +24,4 @@ mock==2.0.0 unittest_xml_reporting==1.12.0 # for onap logging -onappylog>=1.0.5 \ No newline at end of file +onappylog>=1.0.6 \ No newline at end of file -- 2.16.6