# -*- coding: utf-8 -*-
"""
logbook.ticketing
~~~~~~~~~~~~~~~~~
Implements long handlers that write to remote data stores and assign
each logging message a ticket id.
:copyright: (c) 2010 by Armin Ronacher, Georg Brandl.
:license: BSD, see LICENSE for more details.
"""
from time import time
import json
from logbook.base import NOTSET, level_name_property, LogRecord
from logbook.handlers import Handler, HashingHandlerMixin
from logbook.helpers import cached_property, b, PY2, u
class Ticket(object):
"""Represents a ticket from the database."""
level_name = level_name_property()
def __init__(self, db, row):
self.db = db
self.__dict__.update(row)
@cached_property
def last_occurrence(self):
"""The last occurrence."""
rv = self.get_occurrences(limit=1)
if rv:
return rv[0]
def get_occurrences(self, order_by='-time', limit=50, offset=0):
"""Returns the occurrences for this ticket."""
return self.db.get_occurrences(self.ticket_id, order_by, limit, offset)
def solve(self):
"""Marks this ticket as solved."""
self.db.solve_ticket(self.ticket_id)
self.solved = True
def delete(self):
"""Deletes the ticket from the database."""
self.db.delete_ticket(self.ticket_id)
# Silence DeprecationWarning
__hash__ = None
def __eq__(self, other):
equal = True
for key in self.__dict__.keys():
if getattr(self, key) != getattr(other, key):
equal = False
break
return equal
def __ne__(self, other):
return not self.__eq__(other)
class Occurrence(LogRecord):
"""Represents an occurrence of a ticket."""
def __init__(self, db, row):
self.update_from_dict(json.loads(row['data']))
self.db = db
self.time = row['time']
self.ticket_id = row['ticket_id']
self.occurrence_id = row['occurrence_id']
class BackendBase(object):
"""Provides an abstract interface to various databases."""
def __init__(self, **options):
self.options = options
self.setup_backend()
def setup_backend(self):
"""Setup the database backend."""
raise NotImplementedError()
def record_ticket(self, record, data, hash, app_id):
"""Records a log record as ticket."""
raise NotImplementedError()
def count_tickets(self):
"""Returns the number of tickets."""
raise NotImplementedError()
def get_tickets(self, order_by='-last_occurrence_time',
limit=50, offset=0):
"""Selects tickets from the database."""
raise NotImplementedError()
def solve_ticket(self, ticket_id):
"""Marks a ticket as solved."""
raise NotImplementedError()
def delete_ticket(self, ticket_id):
"""Deletes a ticket from the database."""
raise NotImplementedError()
def get_ticket(self, ticket_id):
"""Return a single ticket with all occurrences."""
raise NotImplementedError()
def get_occurrences(self, ticket, order_by='-time', limit=50, offset=0):
"""Selects occurrences from the database for a ticket."""
raise NotImplementedError()
class SQLAlchemyBackend(BackendBase):
"""Implements a backend that is writing into a database SQLAlchemy can
interface.
This backend takes some additional options:
`table_prefix`
an optional table prefix for all tables created by
the logbook ticketing handler.
`metadata`
an optional SQLAlchemy metadata object for the table creation.
`autocreate_tables`
can be set to `False` to disable the automatic
creation of the logbook tables.
"""
def setup_backend(self):
from sqlalchemy import create_engine, MetaData
from sqlalchemy.orm import sessionmaker, scoped_session
engine_or_uri = self.options.pop('uri', None)
metadata = self.options.pop('metadata', None)
table_prefix = self.options.pop('table_prefix', 'logbook_')
if hasattr(engine_or_uri, 'execute'):
self.engine = engine_or_uri
else:
# Pool recycle keeps connections from going stale,
# which happens in MySQL Databases
# Pool size is more custom for out stack
self.engine = create_engine(engine_or_uri, convert_unicode=True,
pool_recycle=360, pool_size=1000)
# Create session factory using session maker
session = sessionmaker()
# Bind to the engined
session.configure(bind=self.engine)
# Scoped session is a thread safe solution for
# interaction with the Database
self.session = scoped_session(session)
if metadata is None:
metadata = MetaData()
self.table_prefix = table_prefix
self.metadata = metadata
self.create_tables()
if self.options.get('autocreate_tables', True):
self.metadata.create_all(bind=self.engine)
def create_tables(self):
"""Creates the tables required for the handler on the class and
metadata.
"""
import sqlalchemy as db
def table(name, *args, **kwargs):
return db.Table(self.table_prefix + name, self.metadata,
*args, **kwargs)
self.tickets = table('tickets',
db.Column('ticket_id', db.Integer,
primary_key=True),
db.Column('record_hash', db.String(40),
unique=True),
db.Column('level', db.Integer),
db.Column('channel', db.String(120)),
db.Column('location', db.String(512)),
db.Column('module', db.String(256)),
db.Column('last_occurrence_time', db.DateTime),
db.Column('occurrence_count', db.Integer),
db.Column('solved', db.Boolean),
db.Column('app_id', db.String(80)))
self.occurrences = table('occurrences',
db.Column('occurrence_id',
db.Integer, primary_key=True),
db.Column('ticket_id', db.Integer,
db.ForeignKey(self.table_prefix +
'tickets.ticket_id')),
db.Column('time', db.DateTime),
db.Column('data', db.Text),
db.Column('app_id', db.String(80)))
def _order(self, q, table, order_by):
if order_by[0] == '-':
return q.order_by(table.c[order_by[1:]].desc())
return q.order_by(table.c[order_by])
def record_ticket(self, record, data, hash, app_id):
"""Records a log record as ticket."""
# Can use the session instead engine.connection and transaction
s = self.session
try:
q = self.tickets.select(self.tickets.c.record_hash == hash)
row = s.execute(q).fetchone()
if row is None:
row = s.execute(self.tickets.insert().values(
record_hash=hash,
level=record.level,
channel=record.channel or u(''),
location=u('%s:%d') % (record.filename, record.lineno),
module=record.module or u('<unknown>'),
occurrence_count=0,
solved=False,
app_id=app_id
))
ticket_id = row.inserted_primary_key[0]
else:
ticket_id = row['ticket_id']
s.execute(self.occurrences.insert()
.values(ticket_id=ticket_id,
time=record.time,
app_id=app_id,
data=json.dumps(data)))
s.execute(
self.tickets.update()
.where(self.tickets.c.ticket_id == ticket_id)
.values(occurrence_count=self.tickets.c.occurrence_count + 1,
last_occurrence_time=record.time,
solved=False))
s.commit()
except Exception:
s.rollback()
raise
# Closes the session and removes it from the pool
s.remove()
def count_tickets(self):
"""Returns the number of tickets."""
return self.engine.execute(self.tickets.count()).fetchone()[0]
def get_tickets(self, order_by='-last_occurrence_time', limit=50,
offset=0):
"""Selects tickets from the database."""
return [Ticket(self, row) for row in self.engine.execute(
self._order(self.tickets.select(), self.tickets, order_by)
.limit(limit).offset(offset)).fetchall()]
def solve_ticket(self, ticket_id):
"""Marks a ticket as solved."""
self.engine.execute(self.tickets.update()
.where(self.tickets.c.ticket_id == ticket_id)
.values(solved=True))
def delete_ticket(self, ticket_id):
"""Deletes a ticket from the database."""
self.engine.execute(self.occurrences.delete()
.where(self.occurrences.c.ticket_id == ticket_id))
self.engine.execute(self.tickets.delete()
.where(self.tickets.c.ticket_id == ticket_id))
def get_ticket(self, ticket_id):
"""Return a single ticket with all occurrences."""
row = self.engine.execute(self.tickets.select().where(
self.tickets.c.ticket_id == ticket_id)).fetchone()
if row is not None:
return Ticket(self, row)
def get_occurrences(self, ticket, order_by='-time', limit=50, offset=0):
"""Selects occurrences from the database for a ticket."""
return [Occurrence(self, row) for row in
self.engine.execute(self._order(
self.occurrences.select()
.where(self.occurrences.c.ticket_id == ticket),
self.occurrences, order_by)
.limit(limit).offset(offset)).fetchall()]
class MongoDBBackend(BackendBase):
"""Implements a backend that writes into a MongoDB database."""
class _FixedTicketClass(Ticket):
@property
def ticket_id(self):
return self._id
class _FixedOccurrenceClass(Occurrence):
def __init__(self, db, row):
self.update_from_dict(json.loads(row['data']))
self.db = db
self.time = row['time']
self.ticket_id = row['ticket_id']
self.occurrence_id = row['_id']
# TODO: Update connection setup once PYTHON-160 is solved.
def setup_backend(self):
from pymongo import ASCENDING, DESCENDING
from pymongo.connection import Connection
try:
from pymongo.uri_parser import parse_uri
except ImportError:
from pymongo.connection import _parse_uri as parse_uri
from pymongo.errors import AutoReconnect
_connection = None
uri = self.options.pop('uri', u(''))
_connection_attempts = 0
parsed_uri = parse_uri(uri, Connection.PORT)
if type(parsed_uri) is tuple:
# pymongo < 2.0
database = parsed_uri[1]
else:
# pymongo >= 2.0
database = parsed_uri['database']
# Handle auto reconnect signals properly
while _connection_attempts < 5:
try:
if _connection is None:
_connection = Connection(uri)
database = _connection[database]
break
except AutoReconnect:
_connection_attempts += 1
time.sleep(0.1)
self.database = database
# setup correct indexes
database.tickets.ensure_index([('record_hash', ASCENDING)],
unique=True)
database.tickets.ensure_index([('solved', ASCENDING),
('level', ASCENDING)])
database.occurrences.ensure_index([('time', DESCENDING)])
def _order(self, q, order_by):
from pymongo import ASCENDING, DESCENDING
col = '%s' % (order_by[0] == '-' and order_by[1:] or order_by)
if order_by[0] == '-':
return q.sort(col, DESCENDING)
return q.sort(col, ASCENDING)
def _oid(self, ticket_id):
from pymongo.objectid import ObjectId
return ObjectId(ticket_id)
def record_ticket(self, record, data, hash, app_id):
"""Records a log record as ticket."""
db = self.database
ticket = db.tickets.find_one({'record_hash': hash})
if not ticket:
doc = {
'record_hash': hash,
'level': record.level,
'channel': record.channel or u(''),
'location': u('%s:%d') % (record.filename,
record.lineno),
'module': record.module or u('<unknown>'),
'occurrence_count': 0,
'solved': False,
'app_id': app_id,
}
ticket_id = db.tickets.insert(doc)
else:
ticket_id = ticket['_id']
db.tickets.update({'_id': ticket_id}, {
'$inc': {
'occurrence_count': 1
},
'$set': {
'last_occurrence_time': record.time,
'solved': False
}
})
# We store occurrences in a seperate collection so that
# we can make it a capped collection optionally.
db.occurrences.insert({
'ticket_id': self._oid(ticket_id),
'app_id': app_id,
'time': record.time,
'data': json.dumps(data),
})
def count_tickets(self):
"""Returns the number of tickets."""
return self.database.tickets.count()
def get_tickets(self, order_by='-last_occurrence_time', limit=50,
offset=0):
"""Selects tickets from the database."""
query = (self._order(self.database.tickets.find(), order_by)
.limit(limit).skip(offset))
return [self._FixedTicketClass(self, obj) for obj in query]
def solve_ticket(self, ticket_id):
"""Marks a ticket as solved."""
self.database.tickets.update({'_id': self._oid(ticket_id)},
{'solved': True})
def delete_ticket(self, ticket_id):
"""Deletes a ticket from the database."""
self.database.occurrences.remove({'ticket_id': self._oid(ticket_id)})
self.database.tickets.remove({'_id': self._oid(ticket_id)})
def get_ticket(self, ticket_id):
"""Return a single ticket with all occurrences."""
ticket = self.database.tickets.find_one({'_id': self._oid(ticket_id)})
if ticket:
return Ticket(self, ticket)
def get_occurrences(self, ticket, order_by='-time', limit=50, offset=0):
"""Selects occurrences from the database for a ticket."""
collection = self.database.occurrences
occurrences = self._order(collection.find(
{'ticket_id': self._oid(ticket)}
), order_by).limit(limit).skip(offset)
return [self._FixedOccurrenceClass(self, obj) for obj in occurrences]
class TicketingBaseHandler(Handler, HashingHandlerMixin):
"""Baseclass for ticketing handlers. This can be used to interface
ticketing systems that do not necessarily provide an interface that
would be compatible with the :class:`BackendBase` interface.
"""
def __init__(self, hash_salt, level=NOTSET, filter=None, bubble=False):
Handler.__init__(self, level, filter, bubble)
self.hash_salt = hash_salt
def hash_record_raw(self, record):
"""Returns the unique hash of a record."""
hash = HashingHandlerMixin.hash_record_raw(self, record)
if self.hash_salt is not None:
hash_salt = self.hash_salt
if not PY2 or isinstance(hash_salt, unicode):
hash_salt = hash_salt.encode('utf-8')
hash.update(b('\x00') + hash_salt)
return hash
class TicketingHandler(TicketingBaseHandler):
"""A handler that writes log records into a remote database. This
database can be connected to from different dispatchers which makes
this a nice setup for web applications::
from logbook.ticketing import TicketingHandler
handler = TicketingHandler('sqlite:////tmp/myapp-logs.db')
:param uri: a backend specific string or object to decide where to log to.
:param app_id: a string with an optional ID for an application. Can be
used to keep multiple application setups apart when logging
into the same database.
:param hash_salt: an optional salt (binary string) for the hashes.
:param backend: A backend class that implements the proper database
handling.
Backends available are: :class:`SQLAlchemyBackend`,
:class:`MongoDBBackend`.
"""
#: The default backend that is being used when no backend is specified.
#: Unless overriden by a subclass this will be the
#: :class:`SQLAlchemyBackend`.
default_backend = SQLAlchemyBackend
def __init__(self, uri, app_id='generic', level=NOTSET,
filter=None, bubble=False, hash_salt=None, backend=None,
**db_options):
if hash_salt is None:
hash_salt = u('apphash-') + app_id
TicketingBaseHandler.__init__(self, hash_salt, level, filter, bubble)
if backend is None:
backend = self.default_backend
db_options['uri'] = uri
self.set_backend(backend, **db_options)
self.app_id = app_id
def set_backend(self, cls, **options):
self.db = cls(**options)
def process_record(self, record, hash):
"""Subclasses can override this to tamper with the data dict that
is sent to the database as JSON.
"""
return record.to_dict(json_safe=True)
def record_ticket(self, record, data, hash):
"""Record either a new ticket or a new occurrence for a
ticket based on the hash.
"""
self.db.record_ticket(record, data, hash, self.app_id)
def emit(self, record):
"""Emits a single record and writes it to the database."""
hash = self.hash_record(record).encode('utf-8')
data = self.process_record(record, hash)
self.record_ticket(record, data, hash)