Package list logbook / 323a8b0f-75b0-42a3-ad20-dac8b486870f/main logbook / more.py
323a8b0f-75b0-42a3-ad20-dac8b486870f/main

Tree @323a8b0f-75b0-42a3-ad20-dac8b486870f/main (Download .tar.gz)

more.py @323a8b0f-75b0-42a3-ad20-dac8b486870f/mainraw · history · blame

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
# -*- coding: utf-8 -*-
"""
    logbook.more
    ~~~~~~~~~~~~

    Fancy stuff for logbook.

    :copyright: (c) 2010 by Armin Ronacher, Georg Brandl.
    :license: BSD, see LICENSE for more details.
"""
import re
import os
import platform

from collections import defaultdict
from functools import partial

from logbook.base import (
    RecordDispatcher, dispatch_record, NOTSET, ERROR, NOTICE)
from logbook.handlers import (
    Handler, StringFormatter, StringFormatterHandlerMixin, StderrHandler)
from logbook._termcolors import colorize
from logbook.helpers import PY2, string_types, iteritems, u
from logbook.ticketing import TicketingHandler as DatabaseHandler
from logbook.ticketing import BackendBase

try:
    import riemann_client.client
    import riemann_client.transport
except ImportError:
    riemann_client = None
    #from riemann_client.transport import TCPTransport, UDPTransport, BlankTransport


if PY2:
    from urllib import urlencode
    from urlparse import parse_qsl
else:
    from urllib.parse import parse_qsl, urlencode

_ws_re = re.compile(r'(\s+)', re.UNICODE)
TWITTER_FORMAT_STRING = u(
    '[{record.channel}] {record.level_name}: {record.message}')
TWITTER_ACCESS_TOKEN_URL = 'https://twitter.com/oauth/access_token'
NEW_TWEET_URL = 'https://api.twitter.com/1/statuses/update.json'


class CouchDBBackend(BackendBase):
    """Implements a backend that writes into a CouchDB database.
    """
    def setup_backend(self):
        from couchdb import Server

        uri = self.options.pop('uri', u(''))
        couch = Server(uri)
        db_name = self.options.pop('db')
        self.database = couch[db_name]

    def record_ticket(self, record, data, hash, app_id):
        """Records a log record as ticket.
        """
        db = self.database

        ticket = record.to_dict()
        ticket["time"] = ticket["time"].isoformat() + "Z"
        ticket_id, _ = db.save(ticket)

        db.save(ticket)


class TwitterFormatter(StringFormatter):
    """Works like the standard string formatter and is used by the
    :class:`TwitterHandler` unless changed.
    """
    max_length = 140

    def format_exception(self, record):
        return u('%s: %s') % (record.exception_shortname,
                              record.exception_message)

    def __call__(self, record, handler):
        formatted = StringFormatter.__call__(self, record, handler)
        rv = []
        length = 0
        for piece in _ws_re.split(formatted):
            length += len(piece)
            if length > self.max_length:
                if length - len(piece) < self.max_length:
                    rv.append(u('…'))
                break
            rv.append(piece)
        return u('').join(rv)


class TaggingLogger(RecordDispatcher):
    """A logger that attaches a tag to each record.  This is an alternative
    record dispatcher that does not use levels but tags to keep log
    records apart.  It is constructed with a descriptive name and at least
    one tag.  The tags are up for you to define::

        logger = TaggingLogger('My Logger', ['info', 'warning'])

    For each tag defined that way, a method appears on the logger with
    that name::

        logger.info('This is a info message')

    To dispatch to different handlers based on tags you can use the
    :class:`TaggingHandler`.

    The tags themselves are stored as list named ``'tags'`` in the
    :attr:`~logbook.LogRecord.extra` dictionary.
    """

    def __init__(self, name=None, tags=None):
        RecordDispatcher.__init__(self, name)
        # create a method for each tag named
        for tag in (tags or ()):
            setattr(self, tag, partial(self.log, tag))

    def log(self, tags, msg, *args, **kwargs):
        if isinstance(tags, string_types):
            tags = [tags]
        exc_info = kwargs.pop('exc_info', None)
        extra = kwargs.pop('extra', {})
        extra['tags'] = list(tags)
        frame_correction = kwargs.pop('frame_correction', 0)
        return self.make_record_and_handle(NOTSET, msg, args, kwargs,
                                           exc_info, extra, frame_correction)


class TaggingHandler(Handler):
    """A handler that logs for tags and dispatches based on those.

    Example::

        import logbook
        from logbook.more import TaggingHandler

        handler = TaggingHandler(dict(
            info=OneHandler(),
            warning=AnotherHandler()
        ))
    """

    def __init__(self, handlers, filter=None, bubble=False):
        Handler.__init__(self, NOTSET, filter, bubble)
        assert isinstance(handlers, dict)
        self._handlers = dict(
            (tag, isinstance(handler, Handler) and [handler] or handler)
            for (tag, handler) in iteritems(handlers))

    def emit(self, record):
        for tag in record.extra.get('tags', ()):
            for handler in self._handlers.get(tag, ()):
                handler.handle(record)


class TwitterHandler(Handler, StringFormatterHandlerMixin):
    """A handler that logs to twitter.  Requires that you sign up an
    application on twitter and request xauth support.  Furthermore the
    oauth2 library has to be installed.

    If you don't want to register your own application and request xauth
    credentials, there are a couple of leaked consumer key and secret
    pairs from application explicitly whitelisted at Twitter
    (`leaked secrets <http://bit.ly/leaked-secrets>`_).
    """
    default_format_string = TWITTER_FORMAT_STRING
    formatter_class = TwitterFormatter

    def __init__(self, consumer_key, consumer_secret, username,
                 password, level=NOTSET, format_string=None, filter=None,
                 bubble=False):
        Handler.__init__(self, level, filter, bubble)
        StringFormatterHandlerMixin.__init__(self, format_string)
        self.consumer_key = consumer_key
        self.consumer_secret = consumer_secret
        self.username = username
        self.password = password

        try:
            import oauth2
        except ImportError:
            raise RuntimeError('The python-oauth2 library is required for '
                               'the TwitterHandler.')

        self._oauth = oauth2
        self._oauth_token = None
        self._oauth_token_secret = None
        self._consumer = oauth2.Consumer(consumer_key,
                                         consumer_secret)
        self._client = oauth2.Client(self._consumer)

    def get_oauth_token(self):
        """Returns the oauth access token."""
        if self._oauth_token is None:
            resp, content = self._client.request(
                TWITTER_ACCESS_TOKEN_URL + '?', 'POST',
                body=urlencode({
                    'x_auth_username':  self.username.encode('utf-8'),
                    'x_auth_password':  self.password.encode('utf-8'),
                    'x_auth_mode':      'client_auth'
                }),
                headers={'Content-Type': 'application/x-www-form-urlencoded'}
            )
            if resp['status'] != '200':
                raise RuntimeError('unable to login to Twitter')
            data = dict(parse_qsl(content))
            self._oauth_token = data['oauth_token']
            self._oauth_token_secret = data['oauth_token_secret']
        return self._oauth.Token(self._oauth_token,
                                 self._oauth_token_secret)

    def make_client(self):
        """Creates a new oauth client auth a new access token."""
        return self._oauth.Client(self._consumer, self.get_oauth_token())

    def tweet(self, status):
        """Tweets a given status.  Status must not exceed 140 chars."""
        client = self.make_client()
        resp, content = client.request(
            NEW_TWEET_URL, 'POST',
            body=urlencode({'status': status.encode('utf-8')}),
            headers={'Content-Type': 'application/x-www-form-urlencoded'})
        return resp['status'] == '200'

    def emit(self, record):
        self.tweet(self.format(record))


class SlackHandler(Handler, StringFormatterHandlerMixin):

    """A handler that logs to slack.  Requires that you sign up an
    application on slack and request an api token.  Furthermore the
    slacker library has to be installed.
    """

    def __init__(self, api_token, channel, level=NOTSET, format_string=None, filter=None,
                 bubble=False):

        Handler.__init__(self, level, filter, bubble)
        StringFormatterHandlerMixin.__init__(self, format_string)
        self.api_token = api_token

        try:
            from slacker import Slacker
        except ImportError:
            raise RuntimeError('The slacker library is required for '
                               'the SlackHandler.')

        self.channel = channel
        self.slack = Slacker(api_token)

    def emit(self, record):
        self.slack.chat.post_message(channel=self.channel, text=self.format(record))


class JinjaFormatter(object):
    """A formatter object that makes it easy to format using a Jinja 2
    template instead of a format string.
    """

    def __init__(self, template):
        try:
            from jinja2 import Template
        except ImportError:
            raise RuntimeError('The jinja2 library is required for '
                               'the JinjaFormatter.')
        self.template = Template(template)

    def __call__(self, record, handler):
        return self.template.render(record=record, handler=handler)


class ExternalApplicationHandler(Handler):
    """This handler invokes an external application to send parts of
    the log record to.  The constructor takes a list of arguments that
    are passed to another application where each of the arguments is a
    format string, and optionally a format string for data that is
    passed to stdin.

    For example it can be used to invoke the ``say`` command on OS X::

        from logbook.more import ExternalApplicationHandler
        say_handler = ExternalApplicationHandler(['say', '{record.message}'])

    Note that the above example is blocking until ``say`` finished, so it's
    recommended to combine this handler with the
    :class:`logbook.ThreadedWrapperHandler` to move the execution into
    a background thread.

    .. versionadded:: 0.3
    """

    def __init__(self, arguments, stdin_format=None,
                 encoding='utf-8', level=NOTSET, filter=None,
                 bubble=False):
        Handler.__init__(self, level, filter, bubble)
        self.encoding = encoding
        self._arguments = list(arguments)
        if stdin_format is not None:
            stdin_format = stdin_format
        self._stdin_format = stdin_format
        import subprocess
        self._subprocess = subprocess

    def emit(self, record):
        args = [arg.format(record=record)
                for arg in self._arguments]
        if self._stdin_format is not None:
            stdin_data = (self._stdin_format.format(record=record)
                          .encode(self.encoding))
            stdin = self._subprocess.PIPE
        else:
            stdin = None
        c = self._subprocess.Popen(args, stdin=stdin)
        if stdin is not None:
            c.communicate(stdin_data)
        c.wait()


class ColorizingStreamHandlerMixin(object):
    """A mixin class that does colorizing.

    .. versionadded:: 0.3
    .. versionchanged:: 1.0.0
       Added Windows support if `colorama`_ is installed.

    .. _`colorama`: https://pypi.org/pypi/colorama
    """
    _use_color = None

    def force_color(self):
        """Force colorizing the stream (`should_colorize` will return True)
        """
        self._use_color = True

    def forbid_color(self):
        """Forbid colorizing the stream (`should_colorize` will return False)
        """
        self._use_color = False

    def should_colorize(self, record):
        """Returns `True` if colorizing should be applied to this
        record.  The default implementation returns `True` if the
        stream is a tty. If we are executing on Windows, colorama must be
        installed.
        """
        if os.name == 'nt':
            try:
                import colorama
            except ImportError:
                return False
        if self._use_color is not None:
            return self._use_color
        isatty = getattr(self.stream, 'isatty', None)
        return isatty and isatty()

    def get_color(self, record):
        """Returns the color for this record."""
        if record.level >= ERROR:
            return 'red'
        elif record.level >= NOTICE:
            return 'yellow'
        return 'lightgray'

    def format(self, record):
        rv = super(ColorizingStreamHandlerMixin, self).format(record)
        if self.should_colorize(record):
            color = self.get_color(record)
            if color:
                rv = colorize(color, rv)
        return rv


class ColorizedStderrHandler(ColorizingStreamHandlerMixin, StderrHandler):
    """A colorizing stream handler that writes to stderr.  It will only
    colorize if a terminal was detected.  Note that this handler does
    not colorize on Windows systems.

    .. versionadded:: 0.3
    .. versionchanged:: 1.0
       Added Windows support if `colorama`_ is installed.

    .. _`colorama`: https://pypi.org/pypi/colorama
    """
    def __init__(self, *args, **kwargs):
        StderrHandler.__init__(self, *args, **kwargs)

        # Try import colorama so that we work on Windows. colorama.init is a
        # noop on other operating systems.
        try:
            import colorama
        except ImportError:
            pass
        else:
            colorama.init()


# backwards compat.  Should go away in some future releases
from logbook.handlers import (
    FingersCrossedHandler as FingersCrossedHandlerBase)


class FingersCrossedHandler(FingersCrossedHandlerBase):
    def __init__(self, *args, **kwargs):
        FingersCrossedHandlerBase.__init__(self, *args, **kwargs)
        from warnings import warn
        warn(PendingDeprecationWarning('fingers crossed handler changed '
             'location.  It\'s now a core component of Logbook.'))


class ExceptionHandler(Handler, StringFormatterHandlerMixin):
    """An exception handler which raises exceptions of the given `exc_type`.
    This is especially useful if you set a specific error `level` e.g. to treat
    warnings as exceptions::

        from logbook.more import ExceptionHandler

        class ApplicationWarning(Exception):
            pass

        exc_handler = ExceptionHandler(ApplicationWarning, level='WARNING')

    .. versionadded:: 0.3
    """
    def __init__(self, exc_type, level=NOTSET, format_string=None,
                 filter=None, bubble=False):
        Handler.__init__(self, level, filter, bubble)
        StringFormatterHandlerMixin.__init__(self, format_string)
        self.exc_type = exc_type

    def handle(self, record):
        if self.should_handle(record):
            raise self.exc_type(self.format(record))
        return False


class DedupHandler(Handler):
    """A handler that deduplicates log messages.

    It emits each unique log record once, along with the number of times it was
    emitted.
    Example:::

        with logbook.more.DedupHandler():
            logbook.error('foo')
            logbook.error('bar')
            logbook.error('foo')

    The expected output:::

       message repeated 2 times: foo
       message repeated 1 times: bar
    """
    def __init__(self,
                 format_string='message repeated {count} times: {message}',
                 *args, **kwargs):
        Handler.__init__(self, bubble=False, *args, **kwargs)
        self._format_string = format_string
        self.clear()

    def clear(self):
        self._message_to_count = defaultdict(int)
        self._unique_ordered_records = []

    def pop_application(self):
        Handler.pop_application(self)
        self.flush()

    def pop_thread(self):
        Handler.pop_thread(self)
        self.flush()

    def pop_context(self):
        Handler.pop_context(self)
        self.flush()

    def pop_greenlet(self):
        Handler.pop_greenlet(self)
        self.flush()

    def handle(self, record):
        if record.message not in self._message_to_count:
            self._unique_ordered_records.append(record)
        self._message_to_count[record.message] += 1
        return True

    def flush(self):
        for record in self._unique_ordered_records:
            record.message = self._format_string.format(
                message=record.message,
                count=self._message_to_count[record.message])
            # record.dispatcher is the logger who created the message,
            # it's sometimes supressed (by logbook.info for example)
            if record.dispatcher is not None:
                dispatch = record.dispatcher.call_handlers
            else:
                dispatch = dispatch_record
            dispatch(record)
        self.clear()


class RiemannHandler(Handler):

    """
    A handler that sends logs as events to Riemann.
    """

    def __init__(self,
                 host,
                 port,
                 message_type="tcp",
                 ttl=60,
                 flush_threshold=10,
                 bubble=False,
                 filter=None,
                 level=NOTSET):
        """
        :param host: riemann host
        :param port: riemann port
        :param message_type: selects transport. Currently available 'tcp' and 'udp'
        :param ttl: defines time to live in riemann
        :param flush_threshold: count of events after which we send to riemann
        """
        if riemann_client is None:
            raise NotImplementedError("The Riemann handler requires the riemann_client package") # pragma: no cover
        Handler.__init__(self, level, filter, bubble)
        self.host = host
        self.port = port
        self.ttl = ttl
        self.queue = []
        self.flush_threshold = flush_threshold
        if message_type == "tcp":
            self.transport = riemann_client.transport.TCPTransport
        elif message_type == "udp":
            self.transport = riemann_client.transport.UDPTransport
        elif message_type == "test":
            self.transport = riemann_client.transport.BlankTransport
        else:
            msg = ("Currently supported message types for RiemannHandler are: {0}. \
                    {1} is not supported."
                   .format(",".join(["tcp", "udp", "test"]), message_type))
            raise RuntimeError(msg)

    def record_to_event(self, record):
        from time import time
        tags = ["log", record.level_name]
        msg = str(record.exc_info[1]) if record.exc_info else record.msg
        channel_name = str(record.channel) if record.channel else "unknown"
        if any([record.level_name == keywords
                for keywords in ["ERROR", "EXCEPTION"]]):
            state = "error"
        else:
            state = "ok"
        return {"metric_f": 1.0,
                "tags": tags,
                "description": msg,
                "time": int(time()),
                "ttl": self.ttl,
                "host": platform.node(),
                "service": "{0}.{1}".format(channel_name, os.getpid()),
                "state": state
                }

    def _flush_events(self):
        with riemann_client.client.QueuedClient(self.transport(self.host, self.port)) as cl:
            for event in self.queue:
                cl.event(**event)
            cl.flush()
        self.queue = []

    def emit(self, record):
        self.queue.append(self.record_to_event(record))

        if len(self.queue) == self.flush_threshold:
            self._flush_events()