Department of Physics and Astronomy

The Forbes Group

Logging with Python

$\newcommand{\vect}[1]{\mathbf{#1}} \newcommand{\uvect}[1]{\hat{#1}} \newcommand{\abs}[1]{\lvert#1\rvert} \newcommand{\norm}[1]{\lVert#1\rVert} \newcommand{\I}{\mathrm{i}} \newcommand{\ket}[1]{\left|#1\right\rangle} \newcommand{\bra}[1]{\left\langle#1\right|} \newcommand{\braket}[1]{\langle#1\rangle} \newcommand{\op}[1]{\mathbf{#1}} \newcommand{\mat}[1]{\mathbf{#1}} \newcommand{\d}{\mathrm{d}} \newcommand{\pdiff}[3][]{\frac{\partial^{#1} #2}{\partial {#3}^{#1}}} \newcommand{\diff}[3][]{\frac{\d^{#1} #2}{\d {#3}^{#1}}} \newcommand{\ddiff}[3][]{\frac{\delta^{#1} #2}{\delta {#3}^{#1}}} \DeclareMathOperator{\erf}{erf} \DeclareMathOperator{\Tr}{Tr} \DeclareMathOperator{\order}{O} \DeclareMathOperator{\diag}{diag} \DeclareMathOperator{\sgn}{sgn} \DeclareMathOperator{\sech}{sech} $

In this post, we discuss how to use the Python logging facilities in your code.

TL;DR

In [15]:
h.stream.fileno()
Out[15]:
1
In [3]:
import logging
logger = logging.getLogger()
h = logging.StreamHandler(os.fdopen(1, "w"))
In [15]:
logger.log?
In [13]:
import logging
import time
from contextlib import contextmanager


# Not exactly sure why this works, but here we add a handler
# to send output to the main console.
# https://stackoverflow.com/a/39331977/1088938
logger = logging.getLogger()
handler = None
for h in logger.handlers:
    if hasattr(h, 'stream') and h.stream.fileno() == 1:
        handler = h
        break
if not handler:
    handler = logging.StreamHandler(os.fdopen(1, "w"))
    logger.addHandler(handler)

formatter = logging.Formatter(
    fmt="[{levelname[0]} {asctime} {name}] {message}\n       {filename}:{lineno}", 
    datefmt="%H:%M:%S",
    style="{")

class MyFormatter(logging.Formatter):
    def __init__(self):
        logging.Formatter.__init__(
            self,
            fmt="[{levelname[0]} {asctime} {name}] {message}",
            datefmt="%H:%M:%S",
            style="{")
        
    def format(self, record):
        if record.levelno >= logging.WARNING:
            _fmt = ("\n" + " "*14).join([self._fmt, "{filename}:{lineno}"])
        else:
            _fmt = self._fmt

        self._fmt = self._style._fmt = _fmt
        return logging.Formatter.format(self, record)
            
handler.setFormatter(MyFormatter())
handler.setLevel('DEBUG')
logger.setLevel('DEBUG')


class A(object):
    def __init__(self):
        self._level = 0
        self.logger.info("Initialized")
        
    @property
    def logger(self):
        """Return the logger."""
        # Get the logger each time so that it will have all
        # properties set by the user.
        return logging.getLogger(self.__class__.__name__)
    
    @contextmanager
    def task(self, msg):
        indent = " "*self._level
        msg = indent + msg
        self.logger.info(msg + "...")
        try:
            self._level += 2
            yield
            self.logger.info(msg + ". Done.")
        except:
            self.logger.error(msg + ". Failed!")
            raise
        finally:
            self._level -= 2            
        
    def run(self):
        with self.task("Running"):
            time.sleep(1.0)
            with self.task("Sprinting"):
                time.sleep(0.1)
            raise Exception()


def debug_logger(logger):
    # Example from https://www.electricmonk.nl/log/2017/08/06/understanding-pythons-logging-module/
    while logger is not None:
        print("level: {}, name: {}, handlers: {}".format(
            logger.level, logger.name, logger.handlers))
        logger = logger.parent


#logging.getLogger().addHandler(logging.StreamHandler(os.fdopen(1, "w")))

a = A()
a.run()
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-13-a14d51296448> in <module>()
     90 
     91 a = A()
---> 92 a.run()

<ipython-input-13-a14d51296448> in run(self)
     76             with self.task("Sprinting"):
     77                 time.sleep(0.1)
---> 78             raise Exception()
     79 
     80 

Exception: 
In [9]:
handler.formatter._style._fmt
Out[9]:
'[{levelname[0]} {asctime} {name}] {message}'
In [14]:
import IPython
ip = IPython.get_ipython()
In [4]:
f = os.fdopen(1, "w")
f.write("JOIJQW")
f.write("JOIJQW")
Out[4]:
6
In [12]:
import sys
sys.stderr.write("Hi")
Hi
In [42]:
logging.getLogger().handlers
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/data/apps/anaconda/envs/jupyter/lib/python3.6/site-packages/IPython/core/formatters.py in __call__(self, obj)
    700                 type_pprinters=self.type_printers,
    701                 deferred_pprinters=self.deferred_printers)
--> 702             printer.pretty(obj)
    703             printer.flush()
    704             return stream.getvalue()

/data/apps/anaconda/envs/jupyter/lib/python3.6/site-packages/IPython/lib/pretty.py in pretty(self, obj)
    398                         if cls is not object \
    399                                 and callable(cls.__dict__.get('__repr__')):
--> 400                             return _repr_pprint(obj, self, cycle)
    401 
    402             return _default_pprint(obj, self, cycle)

/data/apps/anaconda/envs/jupyter/lib/python3.6/site-packages/IPython/lib/pretty.py in _repr_pprint(obj, p, cycle)
    693     """A pprint that just redirects to the normal repr function."""
    694     # Find newlines and replace them with p.break_()
--> 695     output = repr(obj)
    696     for idx,output_line in enumerate(output.splitlines()):
    697         if idx:

/data/apps/anaconda/envs/jupyter/lib/python3.6/logging/__init__.py in __repr__(self)
   1003         name = getattr(self.stream, 'name', '')
   1004         if name:
-> 1005             name += ' '
   1006         return '<%s %s(%s)>' % (self.__class__.__name__, name, level)
   1007 

TypeError: unsupported operand type(s) for +=: 'int' and 'str'
In [13]:
%config Application.log_level="INFO"

Recomendations

In [ ]: