<url>http://maven.apache.org</url>
<modules>
<module>reference</module>
+ <module>pylog</module>
</modules>
</project>
--- /dev/null
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- /dev/null
+# ONAP python logging package
+- python-package onappylog extend python standard logging library which
+could be used in any python project to log MDC(Mapped Diagnostic Contex)
+and easy to reload logging at runtime.
+
+-----
+
+## install package
+```bash
+ pip install onappylog
+```
+
+## Usage
+
+### 1. MDC monkey patch
+
+Import the MDC monkey patch making logRecord to store context in local thread.
+
+```python
+ from onaplogging import monkey; monkey.patch_loggingMDC()
+```
+Import the MDC format to be used to configure mdc output format.
+Please replace your old logging format with mdc format in configuration.
+
+
+```python
+ from onaplogging import mdcformatter
+```
+the mdc format example
+```python
+'mdcFormater':{
+ '()': mdcformatter.MDCFormatter, # Use MDCFormatter instance to convert logging string
+ 'format': '%(mdc)s and other %-style key ', # Add '%(mdc)s' here.
+ 'mdcfmt': '{key1} {key2}', # Define your mdc keys here.
+ 'datefmt': '%Y-%m-%d %H:%M:%S' # date format
+ }
+```
+
+Import MDC to store context in python file with logger
+code.
+
+```python
+from onaplogging.mdcContext import MDC
+# add mdc
+MDC.put("key1", "value1")
+MDC.put("key2", "value2")
+
+# origin code
+logger.info("msg")
+logger.debug("debug")
+
+```
+
+### 2. Reload logging at runtime
+
+It's thread safe to reload logging. If you want to use this feature,
+must use yaml file to configure logging.
+
+
+import the yaml monkey patch and load logging yaml file
+
+```python
+ from onaplogging import monkey,monkey.patch_loggingYaml()
+ # yaml config
+ config.yamlConfig(filepath=<yaml filepath>, watchDog=True)
+```
+
+Notice that the watchDog is opening,So your logging could be reloaded at runtime.
+if you modify yaml file to change handler、filter or format,
+the logger in program will be reloaded to use new configuration.
+
+Set watchDog to **false**, If you don't need to reloaded logging.
+
+
+
+
+Yaml configure exmaple
+
+```yaml
+version: 1
+
+disable_existing_loggers: True
+
+loggers:
+vio:
+ level: DEBUG
+ handlers: [vioHandler]
+ propagate: False
+handlers:
+vioHandler:
+ class: logging.handlers.RotatingFileHandler
+ level: DEBUG
+ filename: /var/log/bt.log
+ mode: a
+ maxBytes: 1024*1024*50
+ backupCount: 10
+ formatter: mdcFormatter
+formatters:
+ mdcFormatter:
+ format: "%(asctime)s:[%(name)s] %(created)f %(module)s %(funcName)s %(pathname)s %(process)d %(levelno)s :[ %(threadName)s %(thread)d]: [%(mdc)s]: [%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s"
+ mdcfmt: "{key1} {key2} {key3} {key4} dwdawdwa "
+ datefmt: "%Y-%m-%d %H:%M:%S"
+ (): onaplogging.mdcformatter.MDCFormatter
+ standard:
+ format: '%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d]
+ [%(levelname)s]:%(message)s '
+ datefmt: "%Y-%m-%d %H:%M:%S"
+
+```
+
+
+### 3. reference
+
+[What's MDC?](https://logging.apache.org/log4j/2.x/manual/thread-context.html)
+
+[Onap Logging Guidelines](https://wiki.onap.org/pages/viewpage.action?pageId=20087036)
+
+[Python Standard Logging Library](https://docs.python.org/2/library/logging.html)
--- /dev/null
+<!--
+ Copyright (c) 2017-2018 VMware, Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at:
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>pylog</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <fileSets>
+ <fileSet>
+ <directory>onaplogging</directory>
+ <outputDirectory>/onaplogging</outputDirectory>
+ <includes>
+ <include>**/*.py</include>
+ <include>**/*.json</include>
+ <include>**/*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>tests</directory>
+ <outputDirectory>/tests</outputDirectory>
+ <includes>
+ <include>**/*.py</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>.</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>*.py</include>
+ <include>*.txt</include>
+ <include>*.sh</include>
+ <include>*.ini</include>
+ <include>*.md</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <baseDirectory>pylog</baseDirectory>
+ <!--baseDirectory>multivimdriver-broker/multivimbroker</baseDirectory-->
+</assembly>
--- /dev/null
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- /dev/null
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+import os
+import yaml
+import traceback
+from logging import config
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+
+__all__ = ['patch_loggingYaml']
+
+
+def _yaml2Dict(filename):
+
+ with open(filename, 'rt') as f:
+ return yaml.load(f.read())
+
+
+class FileEventHandlers(FileSystemEventHandler):
+
+ def __init__(self, filepath):
+
+ FileSystemEventHandler.__init__(self)
+ self.filepath = filepath
+ self.currentConfig = None
+
+ def on_modified(self, event):
+ try:
+ if event.src_path == self.filepath:
+ newConfig = _yaml2Dict(self.filepath)
+ print ("reload logging configure file %s" % event.src_path)
+ config.dictConfig(newConfig)
+ self.currentConfig = newConfig
+
+ except Exception as e:
+ traceback.print_exc(e)
+ print ("Reuse the old configuration to avoid this "
+ "exception terminate program")
+ if self.currentConfig:
+ config.dictConfig(self.currentConfig)
+
+
+def _yamlConfig(filepath=None, watchDog=None):
+
+ """
+ load logging configureation from yaml file and monitor file status
+
+ :param filepath: logging yaml configure file absolute path
+ :param watchDog: monitor yaml file identifier status
+ :return:
+ """
+ if os.path.isfile(filepath) is False:
+ raise OSError("wrong file")
+
+ dirpath = os.path.dirname(filepath)
+ event_handler = None
+
+ try:
+ dictConfig = _yaml2Dict(filepath)
+ # The watchdog could monitor yaml file status,if be modified
+ # will send a notify then we could reload logging configuration
+ if watchDog:
+ observer = Observer()
+ event_handler = FileEventHandlers(filepath)
+ observer.schedule(event_handler=event_handler, path=dirpath,
+ recursive=False)
+ observer.setDaemon(True)
+ observer.start()
+
+ config.dictConfig(dictConfig)
+
+ if event_handler:
+ # here we keep the correct configuration for reusing
+ event_handler.currentConfig = dictConfig
+
+ except Exception as e:
+ traceback.print_exc(e)
+
+
+def patch_loggingYaml():
+
+ # The patch to add yam config forlogginf and runtime
+ # reload logging when modify yaml file
+ config.yamlConfig = _yamlConfig
--- /dev/null
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+
+import logging
+import threading
+import os
+import sys
+import functools
+
+
+__all__ = ['patch_loggingMDC', 'MDC']
+
+_replace_func_name = ['info', 'critical', 'fatal', 'debug',
+ 'error', 'warn', 'warning', 'findCaller']
+
+
+class MDCContext(threading.local):
+ """
+ A Thread local instance to storage mdc values
+ """
+ def __init__(self):
+
+ super(MDCContext, self).__init__()
+ self._localDict = {}
+
+ def get(self, key):
+
+ return self._localDict.get(key, None)
+
+ def put(self, key, value):
+
+ self._localDict[key] = value
+
+ def remove(self, key):
+
+ if key in self.localDict:
+ del self._localDict[key]
+
+ def clear(self):
+
+ self._localDict.clear()
+
+ def result(self):
+
+ return self._localDict
+
+ def isEmpty(self):
+
+ return self._localDict == {} or self._localDict is None
+
+
+MDC = MDCContext()
+
+
+def fetchkeys(func):
+
+ @functools.wraps(func)
+ def replace(*args, **kwargs):
+ kwargs['extra'] = _getmdcs(extra=kwargs.get('extra', None))
+ func(*args, **kwargs)
+ return replace
+
+
+def _getmdcs(extra=None):
+ """
+ Put mdc dict in logging record extra filed with key 'mdc'
+ :param extra: dict
+ :return: mdc dict
+ """
+ if MDC.isEmpty():
+ return
+
+ mdc = MDC.result()
+
+ if extra is not None:
+ for key in extra:
+ # make sure extra key dosen't override mdckey
+ if key in mdc or key == 'mdc':
+ raise KeyError("Attempt to overwrite %r in MDC" % key)
+ else:
+ extra = {}
+
+ extra['mdc'] = mdc
+ del mdc
+ return extra
+
+
+@fetchkeys
+def info(self, msg, *args, **kwargs):
+
+ if self.isEnabledFor(logging.INFO):
+ self._log(logging.INFO, msg, args, **kwargs)
+
+
+@fetchkeys
+def debug(self, msg, *args, **kwargs):
+
+ if self.isEnabledFor(logging.DEBUG):
+ self._log(logging.DEBUG, msg, args, **kwargs)
+
+
+@fetchkeys
+def warning(self, msg, *args, **kwargs):
+ if self.isEnabledFor(logging.WARNING):
+ self._log(logging.WARNING, msg, args, **kwargs)
+
+
+@fetchkeys
+def exception(self, msg, *args, **kwargs):
+
+ kwargs['exc_info'] = 1
+ self.error(msg, *args, **kwargs)
+
+
+@fetchkeys
+def critical(self, msg, *args, **kwargs):
+
+ if self.isEnabledFor(logging.CRITICAL):
+ self._log(logging.CRITICAL, msg, args, **kwargs)
+
+
+@fetchkeys
+def error(self, msg, *args, **kwargs):
+ if self.isEnabledFor(logging.ERROR):
+ self._log(logging.ERROR, msg, args, **kwargs)
+
+
+def findCaller(self):
+
+ f = logging.currentframe()
+ if f is not None:
+ f = f.f_back
+ rv = "(unkown file)", 0, "(unknow function)"
+ while hasattr(f, "f_code"):
+ co = f.f_code
+ filename = os.path.normcase(co.co_filename)
+ # jump through local 'replace' func frame
+ if filename == logging._srcfile or co.co_name == "replace":
+ f = f.f_back
+ continue
+ rv = (co.co_filename, f.f_lineno, co.co_name)
+ break
+
+ return rv
+
+
+def patch_loggingMDC():
+ """
+ The patch to add MDC ability in logging Record instance at runtime
+ """
+ localModule = sys.modules[__name__]
+ for attr in dir(logging.Logger):
+ if attr in _replace_func_name:
+ newfunc = getattr(localModule, attr, None)
+ if newfunc:
+ setattr(logging.Logger, attr, newfunc)
--- /dev/null
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+import logging
+
+
+class MDCFormatter(logging.Formatter):
+ """
+ A custom MDC formatter to prepare Mapped Diagnostic Context
+ to enrich log message.
+ """
+
+ def __init__(self, fmt=None, mdcfmt=None, datefmt=None):
+ """
+ :param fmt: build-in format string contains standard
+ Python %-style mapping keys
+ :param mdcFmt: mdc format with '{}'-style mapping keys
+ :param datefmt: Date format to use
+ """
+
+ super(MDCFormatter, self).__init__(fmt=fmt, datefmt=datefmt)
+ self._tmpfmt = self._fmt
+ if mdcfmt:
+ self._mdcFmt = mdcfmt
+ else:
+ self._mdcFmt = '{reqeustID}'
+
+ def _mdcfmtKey(self):
+ """
+ maximum barce match algorithm to find the mdc key
+ :return: key in brace and key not in brace,such as ({key}, key)
+ """
+
+ left = '{'
+ right = '}'
+ target = self._mdcFmt
+ st = []
+ keys = []
+ for index, v in enumerate(target):
+ if v == left:
+ st.append(index)
+ elif v == right:
+
+ if len(st) == 0:
+ continue
+
+ elif len(st) == 1:
+ start = st.pop()
+ end = index
+ keys.append(target[start:end + 1])
+ elif len(st) > 0:
+ st.pop()
+
+ keys = filter(lambda x: x[1:-1].strip('\n \t ') != "", keys)
+ words = None
+ if keys:
+ words = map(lambda x: x[1:-1], keys)
+
+ return keys, words
+
+ def _replaceStr(self, keys):
+
+ fmt = self._mdcFmt
+ for i in keys:
+ fmt = fmt.replace(i, i[1:-1] + "=" + i)
+
+ return fmt
+
+ def format(self, record):
+ """
+ Find mdcs in log record extra field, if key form mdcFmt dosen't
+ contains mdcs, the values will be empty.
+ :param record: the logging record instance
+ :return: string
+ for example:
+ the mdcs dict in logging record is
+ {'key1':'value1','key2':'value2'}
+ the mdcFmt is" '{key1} {key3}'
+ the output of mdc message: 'key1=value1 key3='
+
+ """
+ mdcIndex = self._fmt.find('%(mdc)s')
+ if mdcIndex == -1:
+ return super(MDCFormatter, self).format(record)
+
+ mdcFmtkeys, mdcFmtWords = self._mdcfmtKey()
+ if mdcFmtWords is None:
+
+ self._fmt = self._fmt.replace("%(mdc)s", "")
+ return super(MDCFormatter, self).format(record)
+
+ mdc = record.__dict__.get('mdc', None)
+ res = {}
+ for i in mdcFmtWords:
+ if mdc and i in mdc:
+ res[i] = mdc[i]
+ else:
+ res[i] = ""
+
+ del mdc
+ try:
+ mdcstr = self._replaceStr(keys=mdcFmtkeys).format(**res)
+ self._fmt = self._fmt.replace("%(mdc)s", mdcstr)
+ s = super(MDCFormatter, self).format(record)
+ return s
+
+ except KeyError as e:
+ print ("The mdc key %s format is wrong" % e.message)
+ except Exception:
+ raise
+
+ finally:
+ # reset fmt format
+ self._fmt = self._tmpfmt
--- /dev/null
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+
+from mdcContext import patch_loggingMDC
+from logWatchDog import patch_loggingYaml
+
+
+__all__ = ["patch_all"]
+
+
+def patch_all(mdc=True, yaml=True):
+
+ if mdc is True:
+ patch_loggingMDC()
+
+ if yaml is True:
+ patch_loggingYaml()
--- /dev/null
+<?xml version="1.0"?>
+<!--
+ Copyright (c) 2017-2018 VMware, Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at:
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-->
+<project
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.onap.logging-analytics</groupId>
+ <artifactId>logging-analytics</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>logging-pylog</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+ <name>logging-pylog</name>
+ <description>onap python logging library</description>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
--- /dev/null
+PyYAML>=3.10
+watchdog>=0.8.3
--- /dev/null
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+
+from setuptools import setup, find_packages
+
+setup(
+
+ name='onappylog',
+ keywords=("yaml", "logging", "mdc", "onap"),
+ description='onap python logging library',
+ long_description="python-package onappylog could be used in any python project to record MDC information and reload logging at runtime",
+ version="1.0.5",
+ license="MIT Licence",
+ author='ke liang',
+ author_email="lokyse@163.com",
+ packages=find_packages(),
+ platforms=['all'],
+ install_requires=[
+ "PyYAML>=3.10",
+ "watchdog>=0.8.3"
+ ],
+ classifiers=[
+ 'Development Status :: 4 - Beta',
+ 'Intended Audience :: Developers',
+ 'Programming Language :: Python :: 2.7'
+ ]
+)
--- /dev/null
+# Copyright (c) 2018-2019 VMware, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+
+import unittest
+
+
+class TestExample(unittest.TestCase):
+
+ def test_mdcFormat(self):
+ return
--- /dev/null
+[tox]
+envlist =py,py27,pep8
+skipsdist = true
+skip_missing_interpreters = true
+
+[tox:jenkins]
+downloadcache = ~/cache/pip
+
+[testenv]
+deps = -r{toxinidir}/requirements.txt
+ pytest
+ coverage
+ pytest-cov
+setenv = PYTHONPATH={toxinidir}/
+
+commands =
+ /usr/bin/find . -type f -name "*.py[c|o]" -delete
+ py.test
+
+[testenv:pep8]
+deps=flake8
+commands=flake8
+
+[flake8]
+show-source = true
+exclude = env,venv,.venv,.git,.tox,dist,doc,*egg,build
+
+
+[testenv:py27]
+commands =
+ {[testenv]commands}
+
+[testenv:cover]
+commands = py.test --cov onaplogging
Code Review / logging-analytics.git / commitdiff