Python Notes

Wednesday, October 20, 2004

MetaTemplate: Generic data templates in pure Python

I'm happy to announce that the metatemplate module is now sufficiently mature for a beta-quality release. I've been using it for my own projects over the past few weeks. The latest changes involved cleaning the code, and the addition of a new __attr__ private member to the template which allows to map alternative names to Python names. This feature was needed to allow template attributes to be stored on external resources (such as INI files) using alternative names that can include characters that are invalid in standard Python symbol names.

This module can be used to declare standard data structures containing attributes and arbitrarily nested templates. Attributes are stored in the order thay are declared in the source code, which allows the template to be used to build structures that reflect this ordering. Some applications include ini-style configuration handling, data entry forms, document structure templates, and generic data records with ordered members.

""" -- template metaclass that can be used to customize any
class to store user-defined attributes in the original definition


import sys
import itertools
from inspect import isclass, isdatadescriptor
from types import StringType, IntType, FloatType, ListType, DictType

# Debug constants (for testing purposes)

debug_generic_attribute = 0
debug_typed_attribute = 0
debug_auto_instantiation = 0
debug_iterator = 0

# AbstractAttribute is the ancestor of all classes that can be used
# in the metatemplate framework. Abstract attributes are named.

class AbstractAttribute(object):
name = ''

# GenericAttribute is the ancestor of all simple elements that are
# used as attributes of templates
# When referred from a instance, the __get__ method returns the value
# associated with the attribute, using the default value if needed.
# If called from the class, the __get__ method returns the property
# object itself. This is used for some internal checks.

class GenericAttribute(AbstractAttribute):
""" Generic attributes for generic containers """
def __init__(self, default = None, name = None):
self._seqno = next_attribute_id()
self.value = default = name
def __repr__(self):
return "" % (
def __get__(self, instance, owner):
if debug_generic_attribute:
print "GET self:[%s], instance:[%s], owner:[%s]" % (self, instance, owner)
if instance:
attrdict = instance.__dict__.setdefault('__attr__', {})
return attrdict.get(, self.value)
return self
def __set__(self, instance, value):
if debug_generic_attribute:
print "SET self:[%s], instance:[%s], value:[%s]" % (self, instance, value)
attrdict = instance.__dict__.setdefault('__attr__', {})
attrdict[] = value

class TypedAttribute(GenericAttribute):
""" Typed attributes for generic containers """
def __init__(self, default = None, name = None, mytype = None):
GenericAttribute.__init__(self, default, name)
if mytype:
if isclass(mytype):
self.mytype = mytype
raise TypeError("Argument expects None "
"or a valid type/class")
self.mytype = type(default)
def __repr__(self):
return "" % (, self.mytype.__name__)
def __set__(self, instance, value):
if debug_typed_attribute:
print "SET self:[%s], instance:[%s], value:[%s]" % (self, instance, value)
if not isinstance(value, self.mytype):
if isinstance(value, StringType):
# tries to convert a string to the correct target
# type; needed when reading values from files.
value = self.mytype(value)
raise TypeError, "Expected %s attribute" % self.mytype.__name__
attrdict = instance.__dict__.setdefault('__attr__', {})
attrdict[] = value

# auxiliary functions

def stripindent(docstring):
stripindent - reformats a multiline, triple-quoted string, removing
extra leading spaces that are used for indentation purposes.
# shamelessly taken from PEP 257:
if not docstring:
return ''
# Convert tabs to spaces (following the normal Python rules)
# and split into a list of lines:
lines = docstring.expandtabs().splitlines()
# Determine minimum indentation (first line doesn't count):
indent = sys.maxint
for line in lines[1:]:
stripped = line.lstrip()
if stripped:
indent = min(indent, len(line) - len(stripped))
# Remove indentation (first line is special):
trimmed = [lines[0].strip()]
if indent < sys.maxint:
for line in lines[1:]:
# Strip off trailing and leading blank lines:
while trimmed and not trimmed[-1]:
while trimmed and not trimmed[0]:
# Return a single string:
return '\n'.join(trimmed)


next_attribute_id = itertools.count().next

def getfields(dct):
takes a dictionary of class attributes and returns a decorated list
containing all valid field instances and their relative position.

for fname, fobj in dct.items():
if isinstance(fobj,GenericAttribute):
yield (fobj._seqno, (fname, fobj))
elif isclass(fobj) and issubclass(fobj,AbstractAttribute):
yield (fobj._seqno, (fname, fobj))
elif (fname[0] != '_'):
# conventional attributes from basic types are just stored
# as GenericAttributes, and put at the end of the list,
# in alphabetical order
basic_types = (StringType, IntType, FloatType, ListType, DictType)
if isinstance(fobj, basic_types):
yield (sys.maxint, (fname, GenericAttribute(fobj)))
yield (0, (fname, fobj))
yield (0, (fname, fobj))

def makefieldsdict(dct, bases):
# build the field list and sort it
fields = list(getfields(dct))
# undecorate the list and build a dict that will be returned later
sorted_field_list = [field[1] for field in fields]
field_dict = dict(sorted_field_list)
# finds all nested instances and classes that are templates
attribute_list = [field for field in sorted_field_list
if (isinstance(field[1],AbstractAttribute) or
(isclass(field[1]) and
# check baseclasses for attributes inherited but not overriden
# !!WARNING: this code does not checks correctly for multiple
# base classes if there are name clashes between overriden
# members. This is not recommended anyway.
inherited = []
for baseclass in bases:
base_field_list = getattr(baseclass, '__fields__', None)
# looks for a valid __fields__ attribute in an ancestor
if isinstance(base_field_list, ListType):
fnames = [f[0] for f in attribute_list]
for fname, fobj in base_field_list:
# checks for overriden attributes
if (fname in fnames):
# overriden - inherited list contains the new value
newobj = field_dict[fname]
inherited.append((fname, newobj))
# remove attribute and quick check field names list
attribute_list.remove((fname, field_dict[fname]))
# copy the original entry into the inherited list
inherited.append((fname, fobj))

# Templates have two private members named __fields__ and
# __attr__. The former stores the ordered field definitions,
# while the later is a dict indexed by the alternative attribute
# name. There are situations where each structure is more
# convenient. The ideal situation would be to have a ordered
# dict, but this is not the case right now...

# stores the ordered field list in the new class directory
all_fields = inherited + attribute_list
field_dict['__fields__'] = all_fields

# generates a dict indexed by the alternative attribute name; for
# each key it stores a tuple containing the field name (used for
# binding in the template) and the object definition
attr_dict = {}
for field in all_fields:
fname, fobj = field
if isinstance(fobj, AbstractAttribute):
if not = fname
attr_dict[] = field
field_dict['__attr__'] = attr_dict
return field_dict

# MetaTemplate metaclass
# Most of the hard work is done outside the class by the auxiliary
# functions makefieldsdict() and getfields()

class MetaTemplate(type):
def __new__(cls, name, bases, dct):
# works out the attribute ordering; keeps inherited order
newdct = makefieldsdict(dct, bases)
# creates the class using only the processed field list
newclass = type.__new__(cls, name, bases, newdct)
newclass._seqno = next_attribute_id() = name
return newclass

# GenericTemplate superclass

class GenericTemplate(AbstractAttribute):
__metaclass__ = MetaTemplate

def __init__(self):
""" instantiates all nested classes upon creation """

# builds a copy of the field list. this is needed to allow
# customizations of the instance not to be reflected in the
# original class field list.
self.__fields__ = list(self.__class__.__fields__)

# auto instantiates nested classes and attributes
if debug_auto_instantiation:
print "AutoInstantiation <%s>: fieldlist = %s" % \, self.__fields__)
for fname, fobj in self.__fields__:
if isclass(fobj) and issubclass(fobj,GenericTemplate):
# found a nested class
if debug_auto_instantiation:
print "AutoInstantiation <%s>: field[%s] is a "\ "Container Subclass" % (, fname)
fobj = fobj()
setattr(self, fname, fobj)
elif isinstance(fobj, AbstractAttribute):
# found an attribute instance
if debug_auto_instantiation:
print "AutoInstantiation <%s>: field[%s] is an "\ "Attribute Instance" % (, fname)
#setattr(fobj, 'name', fname)
if debug_auto_instantiation:
print "AutoInstantiation <%s>: field[%s] is "\ "unknown" % (, fname)

def iterall(cls, preorder=True, posorder=False, interface=None, _iterlevel=0):
Generic recursive iterator for nested templates

This iterator handles all the recursion needed to navigate deep
nested structures. It 'flattens' the structure, returning a
simple sequence of attributes that can be processed
automatically by sequential code.

This iterator was implemented originally for testing purposes.
It's a class method, because some information can only be
acessed by the class (example: alternative attribute names).

preorder is a flag. If True, nested structures will be returned
*before* descending into its component attributes.

posorder is a flag. If True, nested structures will be returned
*after* descending into its component attributes.

if both preorder and posorder are true, nested structures will
be returned both *before* and *after* the component attributes
are returned.

interface is a filter. It allows to retrieve only the members
that expose a particular interface. It doesnt descend on nested
classes that don't expose that interface.

_iterlevel is a simple-minded watchdog. It was included for
debugging during development, to stop code infinite recursion
in some weird cases, and will probably be removed from released
if not interface:
interface = GenericTemplate
if debug_iterator:
if _iterlevel > 5:
print "[4] Recursion limit exceeded"
if debug_iterator:
print "[1] entry code:",
if preorder:
yield cls
for fname, fobj in cls.__fields__:
obj = getattr(cls, fname)
if debug_iterator:
print "[2] yield:",, fname,
if isclass(obj) and issubclass(obj, interface):
if debug_iterator:
print "[3] found nested class: ", obj, [x[0] for x in obj.__fields__]
for member in obj.iterall(preorder=preorder, posorder=posorder,
interface=interface, _iterlevel=_iterlevel+1):
yield member
elif isinstance(obj, GenericAttribute):
yield obj
if posorder:
yield cls
iterall = classmethod(iterall)

def iterfields(self):
"""Simple iterator: returns ordered fields"""
for fname, fobj in self.__fields__:
yield getattr(self, fname)

def __repr__(self):
return "<%s '%s'>" % (self.__class__.__name__,,)


Post a Comment

<< Home