# -*- coding: utf8 -*-
'''
CMF ELEMENT MODULE
***************************
***  CMF ELEMENT MODULE
***************************
$RCSfile: cmf_element.py,v $
$Revision: 930 $
$Author: markus $
$Date: 2008-05-13 13:03:35 +0200 (Di, 13. Mai 2008) $

Classes defining the basic data structure of an XML element as defined in a CMF file.
'''


from types import *
import string
from biocase.wrapper.errorclasses    import *
from biocase.wrapper.typeconverter   import XMLtransformerClass, DBAttributeClass
from biocase.tools.various_functions     import flatten, multiplyLoL, isTrue
import xml.etree.ElementTree as ET
from biocase.tools.xmlutils              import *

import biocase.configuration
cfg = biocase.configuration.Cfg()

import logging
log = logging.getLogger("pywrapper.cmf")


NS21 = "http://www.biocase.org/schemas/conceptmapping/2.1"


# ---------------------------------------------------------------------------        
def matchFilter(filterObj, dbval):
    '''Boolean function returning true for full filled filter conditions.
    Using parameter dbval to compare the filter object with.'''
    if filterObj == None:
        return 1
    elif filterObj.type == 'equals':
        # compare database value with CMF filter value (always a string!)
        if dbval == filterObj.arg2:
            return 1
    elif filterObj.type == 'notequals':
        if dbval <> filterObj.arg2:
            return 1
    elif filterObj.type == 'in':
        if dbval in filterObj.arg2:
            return 1
    return 0

# **********************
# ***  MAPPING OBJECT
# **********************

# ------------------------------------------------------------------------------------
class TotalMatchedRecords:
    '''dummy class to represent the mapping directive "TotalMatchedRecords"'''
    count = 0
    def __repr__(self):
        return "<i>TotalMatchedRecords</i>"

# ------------------------------------------------------------------------------------
class CMFMappingClass:
    '''Class representing a single DB mapping to a CMF element.
    It contains a list of items to be concatenated for output.
    Each item can be a literal (static value) or a database attribute object.
    If there are several items used for a mapping which have to be concatenated,
    the mapping is not searchable anymore.
    Conditional mapping is possible by using a filter object for a mapping.'''
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def __init__(self, items=[]):
        self.items = [] # a mixed list of literals (strings), DBAttributeClass objects or TotalMatchedRecords objects
        self.filterObj = None
        for item in items:
            self.addItem(item)

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def __repr__(self):
        tmp = []
        for item in self.items:
            if type(item) == InstanceType:
                tmp.append(item)
            else:
                tmp.append('"%s"'%item)
        return string.join([unicode(item) for item in tmp], " + ")

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getItems(self):
        '''Returns the list of items to be concatenated.'''
        return self.items
        
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def addItem(self, item):
        '''Adds an item to the list of items to be concatenated.
        Item should be a string or DBAttribute object.'''
        if not isinstance(item, (StringType, UnicodeType, DBAttributeClass, TotalMatchedRecords)):
            raise MappingObjectTypeNotSupportError()
        self.items.append(item)

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def setItems(self, itemList):
        '''Sets the list of items to be concatenated.
        List should contain only strings or DBAttribute objects.'''
        for item in itemList:
            self.addItem(item)

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def isSearchable(self):
        '''Returns true if this mapping object is searchable.'''
        if len(self.items) == 1 and isinstance(self.items[0], DBAttributeClass) and self.items[0].nosearch == 0:
            return 1
        else:
            return 0

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasLiterals(self):
        '''Returns true if this mapping object contains literal values.'''
        for item in self.items:
            if not isinstance(item, DBAttributeClass):
                return 1
        return 0

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasDBAttributes(self):
        '''Returns true if this mapping object contains db objects.'''
        for item in self.items:
            if isinstance(item, DBAttributeClass):
                return 1
        return 0

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasTotalMatchedRecords(self):
        '''Returns true if this mapping object contains TotalMatchedRecords objects.'''
        for item in self.items:
            if isinstance(item, TotalMatchedRecords):
                return 1
        return 0

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasItems(self):
        '''Returns true if this mapping object contains literals or db objects.'''
        if len(self.items) > 0:
            return True
        return False

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getDBAttributes(self):
        '''Returns a list of DBAttribute objects found in concatItems.'''
        result = []
        for item in self.items:
            if isinstance(item, DBAttributeClass):
                result.append(item)
        return result

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getValues(self, rs):
        '''Returns a list of values to be concatenated for a single record.'''
        values = []
        for item in self.items:
            if isinstance(item, DBAttributeClass):
                values.append(rs[item.rsindex])
            elif isinstance(item, TotalMatchedRecords):
                values.append( TotalMatchedRecords.count )
            else:
                values.append(item)
        return values



# **********************
# ***  ATTRIBUTE OBJECT
# **********************

# ------------------------------------------------------------------------------------
class CMFAttributeClass(XMLtransformerClass):
    '''Attribute class to represent relevant information and basic behavior for XML attributes. Initialized with a dictionary.'''
    encoding = 'latin-1'
    NodeType = 'Attribute'
    
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def __init__(self, CMFBaseObj, path2CMFElemObj={}, initDict={}, parentObject=None):
        XMLtransformerClass.__init__(self)
        # save reference to CMFBase object
        self.CMFBaseObj = CMFBaseObj
        # save reference to parental object!
        self.parent     = parentObject
        # the attributes name
        self.Name         = initDict.get('name', None)
        # the W3C simple type of the attribute
        self.Type        = initDict.get('type', None)
        # min number of this element. integer, 0  = optional
        self.RepeatMin    = int(initDict.get('repmin', 0))
        #
        # the controlled vocabulary allowed
        self.Voc        = initDict.get('voc', [])
        # the controlled vocabulary used in the DB. uses the same index as self.Voc
        self.VocDB        = initDict.get('vocdb', [])
        # xpath to this element
        self.xpath        = initDict.get('xpath', None)
        # List of Mapping Objects (see above)
        self.mappings    = initDict.get('mappings', [])
        # global filter object
        # database comparison condition for filtering the result recordset without DB mappings at this CMF element.
        # SQL COP/LOP objects used.
        self.filterObj  = initDict.get('filter', None)
        #
        # add this element/attribute to the path-dictionary
        path2CMFElemObj[self.xpath] = self

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def __repr__(self):
        return "%s: Name=%s, Type=%s, Voc=%s, DBVoc=%s, Path=%s, Num Mapped=%i, Filter=%s" %(self.__class__.NodeType, self.Name, self.Type, str(self.Voc), str(self.VocDB), self.xpath, len(self.mappings), unicode(self.filterObj) )

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def fullRepr(self):
        '''Give the full documentation of all child nodes also.'''
        return "%s: Name=%s, Type=%s, Voc=%s, DBVoc=%s, Path=%s, Num Mapped=%i, Filter=%s" %(self.__class__.NodeType, self.Name, self.Type, str(self.Voc), str(self.VocDB), self.xpath, len(self.mappings), unicode(self.filterObj) )

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def isAttribute(self):
        '''Checks whether this node is a attribute.'''
        return 1

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def isSearchable(self):
        '''Returns true if this CMF attribute object is searchable.
        That is if any of its mappings is searchable.'''
        for mapObj in self.mappings:
            if mapObj.isSearchable():
                return 1
        return 0

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def isRepNode(self):
        '''See CMFElement Class for details. Attributes are never repeatable.'''
        return False
    
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getDBAttributes(self):
        '''Returns a list of all database attribute objects.'''
        dbaos = []
        for mapObj in self.mappings:
            dbaos += mapObj.getDBAttributes()
        # include filter objects
        f = self.getFilter()
        if f is not None:
            dbaos += f.getDBValObjs()
        return dbaos
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getSearchableDBAttrObjs(self):
        '''Returns a list of searchable database attribute objects.
        This means only dbaos that are not being concatenated and are the sole item of a mapping object.'''
        dbaos = []
        for mapObj in self.mappings:
            if mapObj.isSearchable():
                dbaos += mapObj.getItems()
        return dbaos

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def omitNamespace(self):
        '''Return true if namespaces shouuld be omitted in the output.'''
        if self.CMFBaseObj.omitNS:
            return True
        else:
            return False

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getNamespace(self):
        if self.parent is not None:
            return parent.getNamespace()
        return None
        
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getParents(self):
        '''return the list of ordered parent nodes until root.'''
        if self.parent is None:
            return []
        else:
            return [self.parent] + self.parent.getParents()

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getParentalFilterList(self):
        '''Returns a list of all global filter objects from this element up to the root element.'''
        # check if this element has global filters
        if self.filterObj is not None:
            filters = [self.filterObj]
        else:
            filters = []
        # check parental filters
        if self.parent is not None:
            filters += self.parent.getParentalFilterList()
        return filters

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getMandatoryDescendants(self):
        '''Return a list of mandatory children and their mandatory descendants.
        If there is a choice, return all mandatory choices.'''
        return []
        
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasW3CType(self):
        '''True if object has a w3c type set.'''
        if self.Type is not None:
            return True
        else:
            return False
    
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasFilter(self):
        '''True if object has a filter.'''
        if self.filterObj is not None:
            return True
        else:
            return False

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getFilter(self):
        '''Returns filter of object or None.'''
        return self.filterObj
    
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasMappings(self):
        '''True if object has at least 1 mapping.'''
        if len(self.mappings) > 0:
            return True
        else:
            return False
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getMappings(self):
        '''Returns list of all mappings.'''
        return self.mappings
    
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def setMapping(self, index, mapObj=None):
        '''Sets the mapping object at a special index of the list.'''
        if mapObj is None:
            mapObj = CMFMappingClass()
        self.mappings[index] = mapObj
    
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def addMapping(self, mapObj=None):
        '''Adds a new mapping object to the mapping list.'''
        if mapObj is None:
            mapObj = CMFMappingClass()
        self.mappings.append(mapObj)
    
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def delMapping(self, index):
        '''Removes a mapping object from the mapping list.'''
        del self.mappings[index]
    
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def concatenateValues(self, valueList, encoding):
        valueList = [self.transformDBVal2XML(val, encoding, stripWhitespace=False) for val in valueList]
        tmp = ""
        for val in [v for v in valueList if v is not None]:
            tmp += val
        # finally strip whitespace
        return tmp.strip()

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def createCMFET_2_1(self):
        '''Returns the ElementTree nodes for this CMF attribute and according to CMF version 2.1'''
        NS21
        datanode = element(NS21, 'attribute', path=self.xpath, name=self.Name, w3cType=self.Type, min=unicode(self.RepeatMin))
        # voc node
        if len(self.Voc) > 0:
            vocnode = subelement(datanode, NS21, 'vocabulary')
            i = 0
            for voc in self.Voc:
                termnode = subelement(vocnode, NS21, 'term', schemaTerm=voc)
                try:
                    if len(self.VocDB[i]) > 0:
                        for vocdb in self.VocDB[i]:
                            subelement(termnode, NS21, 'dbTerm', vocdb)
                except:
                    pass
                i += 1
        # global filter nodes
        if self.filterObj is not None:
            filternode = subelement(datanode, NS21, 'filterRecords')
            filterET = self.filterObj.createET()
            filternode.append(filterET)
        # mapping nodes
        for mapObj in self.mappings:
            mapNode = subelement(datanode, NS21, 'mapping')
            for item in mapObj.items:
                if isinstance(item, DBAttributeClass):
                    # this is a DBAttribute object
                    if item.type in ('string','unicode'):
                        item.type = 'text'
                    subelement(mapNode, NS21, 'dbAttribute', attribute=item.attr, tablealias=item.table, type=item.type, nosearch=str(item.nosearch))
                elif isinstance(item, TotalMatchedRecords):
                    # this is a literal
                    subelement(mapNode, NS21, 'TotalMatchedRecords')
                else:
                    # this is a literal
                    subelement(mapNode, NS21, 'literal', value=item)
            if mapObj.filterObj is not None:
                # create a mapping filter
                filternode = subelement(mapNode, NS21, 'if')
                filterET = mapObj.filterObj.createET()
                filternode.append(filterET)
        return datanode

# ********************
# ***  ELEMENT OBJECT
# ********************

# supports sequences for subelements
# uses choice objects for choice-subelements

# ------------------------------------------------------------------------------------
class CMFElementClass(CMFAttributeClass):
    '''This class represents one CMF element with its logic to produce proper XML output
    for the element and its nested children.
    When an instance is created, it uses the passed initDict
    to create all child objects as well.
    A reference to the parental object will be stored in self.parent for attributes and elements.
    Class represents all relevant data and basic behavior for XML elements. 
    '''
    CMFAttributeClass = CMFAttributeClass
    NodeType = 'Element'
    
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def __init__(self, CMFBaseObj, path2CMFElemObj={}, initDict={}, parentObject=None):
        # subelements will be created as elemClass. If no higher class is supplied, use this basic class.
        self.__class__.CMFAttributeClass.__init__(self, CMFBaseObj, path2CMFElemObj, initDict, parentObject)
        # the namespace of the element and all subelements unless specified again in a subelement.
        self.Namespace        = initDict.get('ns', None)
        # max number of this element. integer, -1 =unbound
        self.RepeatMax         = int(initDict.get('repmax', 1))
        #
        # flag to not allow repeating of this element.
        self.RepeatDont        = initDict.get('dontrepeat', None)
        if self.RepeatDont <> None:
            self.RepeatDont = isTrue(self.RepeatDont)
        # ChoiceNodeID, id of choice-Node this element belongs to. specific for one choice level.
        self.CNID            = initDict.get('cnid', None)
        # ChoiceElementID, id of choice-sequence this element belongs to (if a sequence of elements is one choice of a choice node). specific for one choice level + choiceNode.
        self.CEID             = initDict.get('ceid', None)
        # Boolean value. If true (=1) use this choice element ID when several choices contain information.
        self.Preferred         = initDict.get('preferredChoice', None)
        #
        # ATTRIBUTES CALCULATED FROM THE ABOVE AND A PSF OBJECT
        #
        # For RepNodes only: list of table aliases, 
        # containing a list of DBAttrObjects for their primary keys of locked tables
        self.RepNodePKDBAttrObjLoL = []
        
        # INTEGRITY CHECKS FOR CMF OBJECT:
        # check if repmax >= repmin
        if self.RepeatMax <> -1 and self.RepeatMax < self.RepeatMin:
            raise CMFwrongDefinition(self.xpath, '','The element has a bigger minimum than maximum repeatablility. This is impossible.')

        # SUBELEMENTS AND ATTRIBUTES FOR RAPID OBJECT INITIALIZATION
        #
        # array of attribute-objects representing the potential element-attributes
        self._Attributes     = []
        for attrInitDict in initDict.get('attributes', []):
            attrObj = self.__class__.CMFAttributeClass(CMFBaseObj, path2CMFElemObj, attrInitDict    , self)
            # allow multiple mappings for an attribute
            self._Attributes.append(attrObj)
        #
        # array of logic-objects representing the potential XML-subset
        self._ChildElements    = []
        for elemInitDict in initDict.get('subelements', []):
            elemObj = self.__class__(CMFBaseObj, path2CMFElemObj, elemInitDict    , self)
            self._ChildElements.append(elemObj)

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def __repr__(self):
        if self.RepeatDont:
            max = "!(Max=%i)"%(self.RepeatMax)
        else:
            max = "Max=%i"%(self.RepeatMax)
        attrString = self.__class__.CMFAttributeClass.__repr__(self)

        return "%s, NS=%s, %s, Num Childs=%i, Num Attributes=%i, locked tables=%s" %(attrString, self.Namespace, max, len(self._ChildElements), len(self._Attributes), str(self.getLockedTables()) )

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def fullRepr(self, ind=0):
        '''Give the full documentation of all child nodes also.'''
        subString = ''
        for child in self._ChildElements:
            subString += child.fullRepr(ind+1)

        attributes = ''
        for child in self._Attributes:
            attributes += "\n%s%s" %( "  "*(ind+1), child.fullRepr() )
        return "\n%s%s, Namespace=%s, %s, Num Childs=%i, Num Attributes=%i, locked tables=%s%s%s" %("  "*ind, self.__repr__(), self.Namespace, max, len(self._ChildElements), len(self._Attributes), str(self.getLockedTables()), attributes, subString )

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def isAttribute(self):
        '''Checks if this node whether this node is a attribute.'''
        return 0

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getNamespace(self):
        '''Returns the namespace for this element. Maybe defined in a parental element.'''
        if self.Namespace <> None:
            return self.Namespace
        elif self.parent <> None:
            return self.parent.Namespace
        else:
            return None

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def isMapped(self):
        '''Does this CMF element or its attributes contain mapping objects?.'''
        # check if at least one mapping for this element exists
        if len(self.mappings) > 0:
            return 1
        # check if at least one mapping for this elements CMF attributes exist
        if len(self._Attributes) > 0:
            for attr in self._Attributes:
                if len(attr.mappings) > 0:
                    return 1
        # no data existing
        return 0

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getMappingsOfElementAndAttributes(self):
        '''Return a list of mapping objects for this element and its attributes.'''
        # check this element
        mappings = []
        mappings += self.mappings
        # check this elements CMF attributes
        if len(self._Attributes) > 0:
            for attr in self._Attributes:
                mappings += attr.mappings
        return mappings
        
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasMandatoryAttributes(self):
        '''Check if there is at least one mandatory attribute for this element.'''
        if len(self._Attributes)>0 and max([attr.RepeatMin for attr in self._Attributes]) > 0:
            return 1
        else:
            return 0

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasMappedChildren(self):
        '''Check if direct subelements (children) are mapped at all.'''
        if len(self._ChildElements) > 0:
            # do direct childs have content ?
            for child in self._ChildElements:
                if child.isMapped():
                    return True
        return False
            
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def hasMappedDescendants(self):
        '''Check if any subelements (descendants) are mapped at all.'''
        if self.hasMappedChildren():
            return True
        else:
            for child in self._ChildElements:
                if child.hasMappedDescendants():
                    return True
        return False

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getMandatoryDescendants(self):
        '''Return a list of mandatory children and their mandatory descendants.
        If there is a choice, return all mandatory choices.'''
        mChilds=[]
        # get mandatory direct children
        for child in self._ChildElements:
            if child.RepeatMin > 0:
                mChilds.append(child)
        # get mandatory children of the mandatory childs...
        for child in mChilds:
            mChilds += child.getMandatoryDescendants()
        # get mandatory attributes
        for child in self._Attributes:
            if child.RepeatMin > 0:
                mChilds.append(child)
        return mChilds

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getMandatoryChildren(self):
        '''Return a list of child elements, that are mandatory - that is RepeatMin > 0.
        If a choice of several childs is mandatory it returns a list of child elements 
        instead of a single element resulting in a LOL.'''
        tmp = []
        mandChoices={}
        for child in self._ChildElements:
            if child.RepeatMin > 0:
                if child.CNID is None:
                    tmp.append((child.Name, child.RepeatMin))
                else:
                    if not mandChoices.has_key(child.CNID):
                        mandChoices[child.CNID] = []
                    mandChoices[child.CNID].append( (child.Name, child.RepeatMin) )
        # add the lists of madatory choice elements to the main list
        for choiceList in mandChoices.values():
            tmp.append(choiceList)
        return tmp

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def isRepNode(self):
        '''Return False if the node is not potentially repeatable (maxRep < 2).
        If it is potentially repeatable, return a value based on the state of the rep node:
            0 = inactive; the node is not being repeated for the current configuration (taking tablealiases into account)
            1 = active; the node is being repeated for the current configuration (taking tablealiases into account)
            2 = supressed; the dont repeat flag is set to force the node to be inactive
        '''
        if self.RepeatMax in (0, 1):
            # not even potentially repeatable
            return False
        else:
            if self.RepeatDont:
                # supressed rep node
                return 2
            elif len(self.RepNodePKDBAttrObjLoL) > 0:
                # active rep node
                return 1
            else:
                # inactive rep node
                return 0
        

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def isRepeatable(self):
        '''Return 0 for non repeatable CMF objects and the max number for repeatable objects.'''
        if self.RepeatMax == 1 or self.RepeatDont:
            return 0
        else:
            max = self.RepeatMax
            if max == -1:
                max = 9999999999
            return max
        
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def walk(self, boolFunc=lambda x:1, inclAttributes=1, onlyLeafs=0):
        '''Walk all children and list all element objects that are true for the passed fuction boolFunc.
        If inclAttributes is true: walk elements and attributes,
        if onlyLeafs is true: test only leaf elements'''
        objList = []
        testit = 1
        if onlyLeafs:
            if len(self._ChildElements)>0:
                testit = 0
        # walk attributes?
        if testit and inclAttributes:
            for attrObj in self._Attributes:
                if boolFunc(attrObj):
                    objList.append(attrObj)
        # test this element?
        if testit:
            if boolFunc(self):
                objList.append(self)
        # walk children
        if len(self._ChildElements)>0:
            for child in self._ChildElements:
                objList += child.walk(boolFunc, inclAttributes=inclAttributes, onlyLeafs=onlyLeafs)
        return objList

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def detachFromParent(self):
        '''Remove this node from parents child list.'''
        #log.debug("DETACH %s FROM PARENT NODE %s" % (self.xpath, str(self.parent)))
        if self.parent is None:
            return None
        self.parent._ChildElements.remove(self)

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def getLockedTables(self):
        '''Returns the list of locked table aliases or [] for this element.
        List only contains data for repeatable elements with locked tables.'''
        aliases = []
        for DBAttrlist in self.RepNodePKDBAttrObjLoL:
            aliases.append(DBAttrlist[0].table)
        return aliases

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def activateRepNodes(self, cmfBaseObj, psfObj, lastRepeatableElemObj=None):
        '''Takes a psf object and checks which repeatable elements are active rep nodes
        - that is really repeated depending upon the table aliases defined in the psf.
        Sets the attribute self.RepNodePKDBAttrObjLoL'''
        # get the list of already locked tables from the root until this element:
        alreadyLockedTableAliases = self.getParentalLockedTablealiases()
        log.debug("Already locked tablealiases at concept '%s': %s"%(self.xpath, str(alreadyLockedTableAliases)))
        # update last repeatable element object
        if self.isRepeatable():
            if log.isEnabledFor(logging.DEBUG):
                log.debug("RepNode %s [%s times] exists in CMF at '%s'" %(self.Name, str(self.isRepeatable()), self.xpath))
            lastRepeatableElemObj = self
        # There was a (parental or this) repeatable element 
        # and this element is mapped. Check locking...
        log.debug("Element %s has %i mappings and %i attribute mappings"%(self.Name, len(self.mappings), len(self.getMappingsOfElementAndAttributes())-len(self.mappings) ))
        if lastRepeatableElemObj <> None and self.isMapped():
            for mapObj in self.getMappingsOfElementAndAttributes():
                for dbao in mapObj.getDBAttributes():
                    # was this table locked already?
                    if not dbao.table in alreadyLockedTableAliases:
                        # debugging
                        if log.isEnabledFor(logging.DEBUG):
                            log.debug("Activate Table Lock for table '%s' in concept '%s'"%(dbao.table, lastRepeatableElemObj.xpath) )
                            log.debug("Already locked the tablealiases %s before"%(str(alreadyLockedTableAliases)))
                        # Add table lock
                        PKdbaoList = psfObj.getPKDBAttrObjListOfAlias(dbao.table)
                        lastRepeatableElemObj.RepNodePKDBAttrObjLoL.append(PKdbaoList)
                        alreadyLockedTableAliases.append(dbao.table)
                        for PKDBAttrObj in PKdbaoList:
                            # update cmf base object to include PK DBAtrrObjects in the * list
                            if PKDBAttrObj.rsindex == None:
                                # it is not yet included in the list
                                cmfBaseObj.allDBAttr.append(PKDBAttrObj)
                                PKDBAttrObj.rsindex = len(cmfBaseObj.allDBAttr)
        # process all children
        for child in self._ChildElements:
            child.activateRepNodes(cmfBaseObj, psfObj, lastRepeatableElemObj)

    def getParentalLockedTablealiases(self):
        '''Returns a list of all locked tablealiases from this element up to the root element.'''
        # check if this element has global filters
        localLockedTableList = []
        if len(self.RepNodePKDBAttrObjLoL) > 0:
            for pk in self.RepNodePKDBAttrObjLoL:
                localLockedTableList += [dbao.table for dbao in pk]
        # check parents
        if self.parent is not None:
            localLockedTableList += self.parent.getParentalLockedTablealiases()
        return localLockedTableList
            
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def createETListFromRecordSet(self, rsOrig):
        '''Returns a list of minidom objects of this cmf-element and all xml child elements. 
        Needs a recordset rs(list), that has the same sequence as the list of all mapping objects in "cmf_base.allDBAttr".
        Takes care of empty elements, XML types, choices and repeating elements.
        If its a repNode, it will iterate through all records and pass on only the relevant one (record).
        If the filter object is set, the record set is filtered accordingly.'''
        # the result ET object list tmp:
        tmp = []
        #
        # process the record filter object (CMF2 only)
        #
        rs = []
        if self.filterObj <> None:
            log.debug("Record Filter found at %s: %s"%(self.xpath, self.filterObj))
            transformer = XMLtransformerClass()
            transformer.Type = 'string'
            for r in rsOrig:
                # transform database values into strings:
                #log.debug("ROW=%s"%unicode(r))
                #log.debug("RSIndex=%i"%self.filterObj.arg1.rsindex)
                dbval = transformer.transformDBVal2XML( r[self.filterObj.arg1.rsindex], self.__class__.CMFAttributeClass.encoding)
                if matchFilter(self.filterObj, dbval):
                    rs.append(r)
                else:
                    if log.isEnabledFor(logging.DEBUG):
                        log.debug("Record Filtered at %s. False condition: dbval=%s, filter=%s"%(self.xpath, dbval, self.filterObj))
        else:
            rs = rsOrig
        #
        # The main job of this method is to loop over active rep nodes
        # and to reduce the recordset that is passed on to child cmf-elements!
        #
        if len(self.RepNodePKDBAttrObjLoL) > 0:
            #
            # this is an active repNode, loop through the recordset !
            # loop through every key of all tables
            # create a dict with key = value of PK and value=list of whole records with this PK
            #
            resultDict = {}
            PKdbaos = flatten(self.RepNodePKDBAttrObjLoL)
            for r in rs:
                # create new unique ID:
                PKrsindex = []
                for PKdbao in PKdbaos:
                    PKrsindex.append(unicode(r[PKdbao.rsindex]) )
                uniqueID = '*$*'.join(PKrsindex)
                resultDict.setdefault(uniqueID, [])
                resultDict[ uniqueID ].append(r)
            # loop through "records" of dict and produce list of ET objects
            sortedKeys = resultDict.keys()
            # debug
            if log.isEnabledFor(logging.DEBUG):
                log.debug("Loop of %i records at active rep node '%s'"%(len(sortedKeys), self.xpath))
            # sort and loop 
            sortedKeys.sort()
            _debugIdx = 0
            for PKkey in sortedKeys:
                if log.isEnabledFor(logging.DEBUG):
                    _debugIdx += 1
                    log.debug("Process record %i from loop at active rep node '%s'"%(_debugIdx, self.xpath))
                tmp += self._createETListFromRecord(resultDict[PKkey])
        else:
            if log.isEnabledFor(logging.DEBUG):
                log.debug("processing %i records at the non-active rep node '%s'"%(len(rs), self.xpath))
            tmp = self._createETListFromRecord(rs)
            # Remark: tmp mabe a list of None objects !
            if tmp == None:
                tmp = []
        if log.isEnabledFor(logging.DEBUG):
            log.debug("%i ET objects returned by concept '%s'"%(len(tmp), self.xpath))
        return tmp

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def _createETListFromRecord(self,rs):
        '''For a given recordset it takes the first record and returns a list of ET objects
        with data and child elements from it. Uses "_createETListFromSingleMapping" to create
        XML nodes for a single mapping object and record.

        This method also receives the list of ET node objects from the subelements.
        It evaluates their repeatability and decides whether to create repeating subelements,
        or to create multiple copies of this element (including all other subelements)
        that are then passed on to this elements parents.

        This is mainly done to allow multiple mappings to non-repeatable elements. 
        The multiple ET nodes created are then passed on to next higher repeating node.

        There are two steps involved to process    the list(every subelement)
        of lists (dom-node for the respective subelement)
        1) process the LoL according to the subelements repeatability
           -> see method "_addChildETobjectsToLoL"
        2) process the LoL to create a LoL with the sub-lists all of the same length.
        Multiply LoL like a matrix

        Example: 
        A,B,C represent CMF subelements of this element.
        For repeating subelements the list of returned nodes end with index x (C1,...,Cx)
        Non repeating subelements that return several nodes are shown with concrete indices (B1,B2)

        orig LoL
        [[A], [B1,B2], [C1,...,Cx]]
          ->  process subelements repeatability
        [[A], [B1,B2], [C1], [...], [Cx]]
          ->  multiply LoL
        [[A,B1,C1,...,Cx], [A,B2,C1,...,Cx]]
        '''
        # log for debugging
        if log.isEnabledFor(logging.DEBUG):
            log.debug("First record of %i in total processed at concept '%s' with %i element and %i attribute mappings" %(len(rs), self.xpath, len(self.mappings), len(self.getMappingsOfElementAndAttributes())-len(self.mappings)))
        #
        # a single element, but if several DB attributes are assigned 
        # to it or some non-repeatable children, then several Elements are created.
        #
        ETlist = []
        #
        # Are there any DB Attributes mapped to this element ?
        #
        if len(self.mappings) > 0:
            # loop over all mapping objects:
            i = 0 # remember index of mapping to sync it with attribute mappings
            for mapObj in self.mappings:
                ETlist += self._createETListFromSingleMapping(rs, mapObj, i)
                i += 1
        else:
            # no DB Attribute assigned to this element. check at least for children:
            ETlist += self._createETListFromSingleMapping(rs)
        #
        # remove None's in list (only ET objects allowed !)
        #
        return [elem for elem in ETlist if elem is not None]

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def _createETListFromSingleMapping(self,rs, MapObj=None, mapIndex=None):
        '''For a given recordset and mapping object it takes the first record 
        and returns a list of ET node objects with data and child elements from it.
        Extensively used by function _createETListFromRecord.'''

        nodeListOfThisElement = [] # node list to be returned
        # The template node represents only this CMF element without any xml attributes
        template_node = None
        # list of "template nodes" that are ET objects without children representing this CMF element with its attributes
        # Because of multiple mappings to a single attribute this is a list of "template nodes" and not a single one.
        template_nodes = []

        # debugging
        if log.isEnabledFor(logging.DEBUG):
            try:
                log.debug("The first record of %i in total at concept '%s' for a single mapping ET generation is: %s" %(len(rs), self.xpath, str(rs[0])))
            except:
                log.debug("Logging the first record of %i in total of concept '%s' for a single mapping ET generation created an error"%(len(rs), self.xpath))
        #
        # test if a recordset was passed to this method or if there are literals present
        if len(rs) < 1 and (MapObj is None or not MapObj.hasLiterals()):
            if log.isEnabledFor(logging.DEBUG):
                log.debug("No data recordset for concept %s" %(self.xpath))
            return []
        # process only the first row/record
        if len(rs) > 0:
            record = rs[0]
        else:
            record = []
        #
        # create a filled template node if content data exists.
        # does not contain any attributes yet.
        if MapObj is not None and MapObj.hasItems():
            template_node = self._getElemTemplateNode(MapObj, record)
            if template_node is not None and log.isEnabledFor(logging.DEBUG):
                log.debug("%s: template node created: %s" %(self.Name, ET.tostring(template_node)))
        #
        # check if attributes with data exist
        #
        if len(self._Attributes) > 0:
            template_nodes = self._getElemTemplateNodesWithAttributes(template_node, record, mapIndex)
            template_nodes = [n for n in template_nodes if n is not None]
            # check if a template node was required
            # return an empty list if there was none and stop this method
            if self.hasMandatoryAttributes() and len(template_nodes)<1:
                #
                # break out of function !!!
                log.debug("Concept '%s' dropped, because of missing mandatory attributes." %(self.Name) )
                return []
        if len(template_nodes)<1 and template_node is not None:
            template_nodes = [template_node]
        #
        # check if sub elements potentially contain data
        #
        if self.hasMappedDescendants():
            # potentially data is contained in subelements.
            # loop thru CMF object childs and take care of max repeat.
            #
            # check if the template nodes list is filled.
            # otherwise create a new empty template node for child data.
            if len(template_nodes)<1:
                if self.omitNamespace():
                    template_node = element(None, self.Name)
                else:
                    template_node = element(self.getNamespace(), self.Name)
                template_nodes = [ template_node ]
                if log.isEnabledFor(logging.DEBUG):
                    log.debug("%s: Create empty template node for mapped descendants data: %s" % (self.Name, ET.tostring(template_node)))
            # debug
            if log.isEnabledFor(logging.DEBUG):
                log.debug("%s: Template nodes used for concepts childs: %s" %(self.Name, unicode(template_nodes) ))
            # temporarily store all child ET lists in a list
            # to decide which list to use in the case that choices are present.
            #
            choiceLessChildListOfLists = self._getChoicelessChildNodesLOL(rs)
            #
            # process CHOICES if they are present:
            nodeListOfThisElement = self._processChoicelessLOL(choiceLessChildListOfLists, template_nodes)
        # was this element created with data only and had no children with mappings?
        else:
            if log.isEnabledFor(logging.DEBUG):
                log.debug("No mapped children for concept %s. Use template nodes directly: %s" %(self.Name, unicode(template_nodes)))
            for tNode in template_nodes:
                if tNode is not None:
                    nodeListOfThisElement.append(tNode)
        if log.isEnabledFor(logging.DEBUG):
            log.debug("Concept '%s' produces %i node(s)" %(self.Name, len(nodeListOfThisElement)))
        return nodeListOfThisElement

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def _getElemTemplateNode(self, MapObj, record):
        '''Returns a single template node of this element without attributes being considered.
        Also returns a template without data.'''
        
        template_node = None # node to be returned

        # if no mapping object was given create an empty one 
        # (to be able to process children)
        if MapObj == None:
            MapObj = CMFMappingClass()
        
        # DB attributes existing. check for optional filters
        createNodeFlag = 1
        if MapObj.filterObj <> None:
            transformer = XMLtransformerClass()
            transformer.Type = 'string'
            # transform database values into strings:
            dbval = transformer.transformDBVal2XML( record[MapObj.filterObj.arg1.rsindex], self.__class__.CMFAttributeClass.encoding)
            if not matchFilter(MapObj.filterObj, dbval):
                createNodeFlag = 0
                if log.isEnabledFor(logging.DEBUG):
                    log.debug("Mapping Filtered for %s. False condition: %s"%(self.xpath, str(MapObj.filterObj)))
        if createNodeFlag == 1:
            # it matches. use mapping. get value
            try:
                tmp = self.concatenateValues( MapObj.getValues(record), self.__class__.CMFAttributeClass.encoding )
            except:
                tmp = None
                log.error("DBVal2XML transformation error at %s"%(self.xpath))
                try:
                    log.info("DBValue causing error: %s"%(str(MapObj.getValues(record))))
                except:
                    log.debug("Couldnt log DBValue causing error")
            if tmp != None and len(tmp) > 0:
                if self.omitNamespace():
                    template_node = element(None, self.Name, tmp)
                else:
                    template_node = element(self.getNamespace(), self.Name, tmp)
                if log.isEnabledFor(logging.DEBUG):
                    try:
                        log.debug("ET node created at concept '%s' with value '%s'"%(self.xpath, tmp))
                    except:
                        log.debug("ET node created at concept '%s', but value could not be logged" %(self.xpath))
        return template_node

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def _getElemTemplateNodesWithAttributes(self, template_node, record, mapIndx=None):
        '''Returns a list of template nodes of this element including attributes.'''
        template_nodes = [] # nodes to be returned
        # check the number of maximum mappings for all attributes
        maxNumberOfMappings = max([len(attr.mappings) for attr in self._Attributes])
        #log.debug("%s: maxNumberOfMappings for all attributes: %i" % (self.Name, maxNumberOfMappings))
        if maxNumberOfMappings < 1:
            # no mappings exist. return only the template_node
            if template_node is not None:
                template_nodes = [template_node]
        else:
            # there is at least one attribute mapping!
            # create an empty template_node ET object to hold attributes if it does not already exist
            # remember if initial node was already None. If no valid attribute can be created, then remove the template node at the end if this flag is true
            if template_node is None:
                template_node_was_None = True
                if self.omitNamespace():
                    template_node = element(None, self.Name)
                else:
                    template_node = element(self.getNamespace(), self.Name)
            else:
                template_node_was_None = False
            # there are mapping(s) for at least one attribute
            # check if element mapping and attribute mappings should be synced.
            # The number of element mappings and max attribute mappings must be the same for the syncing to be activated
            if maxNumberOfMappings == len(self.mappings):
                log.debug("%s: There are %i sets of attribute mappings for the XML element %s. Try to correlate mappings by their configuration sequence!" %(self.Name, maxNumberOfMappings, self.xpath) )
                # sync attribute mappings with element mappings
                # If there is a different number of mappings for the different attributes, 
                # repeat the last mapping of the attributes with less mappings.
                # use only cloned nodes to build new node_templates
                for attr in self._Attributes:
                    if len(attr.mappings) > 0:
                        useMapIndx = mapIndx
                        if useMapIndx > len(attr.mappings)-1:
                            # use the last mapping if this attribute has less mappings than the maximum of mappings of all mappings
                            useMapIndx = len(attr.mappings)-1
                        MapObj = attr.mappings[useMapIndx]
                        log.debug("%s: Use map index %i of mapping object %s" %(self.Name, useMapIndx, str(MapObj)) )
                        if (MapObj.hasDBAttributes() and record is not None) or MapObj.hasLiterals():
                            tmp = attr.concatenateValues( MapObj.getValues(record), self.__class__.CMFAttributeClass.encoding )
                            if tmp != None and len(tmp) > 0:
                                attributeDict={str(attr.Name):tmp}
                                log.debug("attributeDict=%s"%str(attributeDict))
                                addOptionalAttributes(elem=template_node, ns=None, **attributeDict) #self.getNamespace()
                            elif attr.RepeatMin > 0:
                                # mandatory attribute not present. Remove node template
                                template_node = None
                    # check if this non-mapped attribute was mandatory
                    elif attr.RepeatMin > 0:
                        # mandatory attribute not present. Remove node template
                        template_node = None
                    #
                    # test for validness of this element
                    if template_node is None:
                        # not valid !
                        log.warn('The mandatory attribute %s could not be created. The element %s was also dropped.'%(attr.Name, self.xpath))
                        break
                # only add template node if it is not None and it existed before or did not exist but contains attributes now
                if template_node is not None and (not template_node_was_None or (template_node_was_None and len(template_node.keys())>0 )):
                    template_nodes.append(template_node)
            else:
                # Loop over mappings and create seperate template nodes for each attribute mapping set.
                # But sync several attributes of this element by the sequence of their mappings.
                # use only cloned nodes to build new node_templates
                for mapIndx in range(maxNumberOfMappings):
                    new_template_node = clone(template_node)
                    for attr in self._Attributes:
                        #log.debug("PROCESS ATTRIBUTE %s - len(attr.mappings)=%i" % (attr.Name, len(attr.mappings)))
                        if len(attr.mappings) > 0:
                            useMapIndx = mapIndx
                            if useMapIndx > len(attr.mappings)-1:
                                # use the last mapping if this attribute has less mappings than the maximum of mappings of all mappings
                                useMapIndx = len(attr.mappings)-1
                            MapObj = attr.mappings[useMapIndx]
                            #log.debug("MAP OBJECT = %s" % str(MapObj))
                            if (MapObj.hasDBAttributes() and record is not None) or MapObj.hasLiterals():
                                tmp = attr.concatenateValues( MapObj.getValues(record), self.__class__.CMFAttributeClass.encoding )
                                if tmp != None and len(tmp) > 0:
                                    attributeDict={str(attr.Name):unicode(tmp)}
                                    addOptionalAttributes(new_template_node, self.getNamespace(), **attributeDict)
                                elif attr.RepeatMin > 0:
                                    # mandatory attribute not present. Remove node template
                                    new_template_node = None
                        # check if this non-mapped attribute was mandatory
                        elif attr.RepeatMin > 0:
                            # mandatory attribute not present. Remove node template
                            new_template_node = None
                        # test for validness of this element
                        if new_template_node is None:
                            # not valid !
                            log.warn('The mandatory attribute %s could not be created. The element %s was also dropped.'%(attr.Name, self.xpath))
                            break
                    # only add template node if it is not None and it existed before or did not exist but contains attributes now
                    if new_template_node is not None and (not template_node_was_None or (template_node_was_None and len(new_template_node.keys())>0 )):
                        template_nodes.append(new_template_node)
        return template_nodes

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def _getChoicelessChildNodesLOL(self, rs):
        '''Returns a list of list of nodes of child nodes for this element not considering any choices.'''

        choiceLessChildListOfLists = [] # nodes LOL to be returned

        for child in self._ChildElements:
            # temporarily store all created ET objects for this child element in a list:
            tmpChildETObjs = []
            # loop through ET objects originated from one (CMF) child element:
            for ETobj in child.createETListFromRecordSet(rs):
                #
                # check min-repeatability of childs of this child-element ET object:
                #
                # generate list of XML child elements. 
                # list of tuples (elemName, occurence)
                ChildTupleList = []
                for childETObj in ETobj:
                    name = localName(childETObj)
                    #log.debug("localname=%s, childETObj.tag=%s"%(name,childETObj.tag))
                    if len(ChildTupleList) > 0 and name == ChildTupleList[-1][0]:
                        ChildTupleList[-1][1] += 1
                    else:
                        ChildTupleList.append([name, 1])
                # generate list of mandatory childs for this child element:
                CNIDcheck = {} 
                for mandPart in child.getMandatoryChildren():
                    ChildNameList = [x[0] for x in ChildTupleList]
                    if type(mandPart) == ListType:
                        # this is a list of mandatory choice elements. 
                        # There needs to be at least one of them present
                        oneChildExists = False
                        for (mChild, repmin) in mandPart:
                            if mChild in ChildNameList and repmin <= ChildTupleList[ChildNameList.index(mChild)][1]:
                                # delete already checked existing elements (there might be elements with the same name !)
                                # ???
                                ChildTupleList = ChildTupleList[ChildNameList.index(mChild):]
                                oneChildExists = True
                        # check if at least one child exists
                        if not oneChildExists:
                            # mandatory child not existing. break out of this loop
                            log.warn("None of the mandatory child elements %s could be created. The element %s was also dropped."%( str([x[0] for x in mandPart]), child.xpath))
                            ETobj = None
                            break
                    else:
                        # its a single mandatory element. check if its present
                        # check if element name is existing and if repmin is smaller
                        (mChild, repmin) = mandPart
                        if mChild in ChildNameList and repmin <= ChildTupleList[ChildNameList.index(mChild)][1]:
                            # delete already checked existing elements (there might be elements with the same name !)
                            ChildTupleList = ChildTupleList[ChildNameList.index(mChild):]
                        else:
                            # mandatory child not existing. break out of this loop
                            num = 0
                            if mChild in ChildNameList:
                                num = ChildTupleList[ChildNameList.index(mChild)][1]
                            log.warn("The mandatory child element %s could not be created [%s existing, but %s required]. The element %s was also dropped."%(mChild, num, repmin, child.xpath))
                            log.debug("Mandatory child %s (mChild type=%s) required %s times"%(str(mChild), str(type(mChild)), str(repmin)))
                            log.debug("ChildNameList: %s"%(str(ChildNameList)))
                            ETobj = None
                            break
                #
                # Only store objects with data:
                #
                if ETobj <> None:
                    tmpChildETObjs.append(ETobj)
            choiceLessChildListOfLists.append(tmpChildETObjs)
            # debugging
            if log.isEnabledFor(logging.DEBUG):
                log.debug("%i choiceless child ET nodes of child concept '%s'" %(len(tmpChildETObjs), child.xpath))
        # return data
        return choiceLessChildListOfLists 

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def _processChoicelessLOL(self, choiceLessChildListOfLists, template_nodes):
        '''Returns a list of nodes of child nodes that considered choices and removed non relevant nodes.'''

        nodeListOfThisElement = [] # nodes list to be returned 

        # debug
        if log.isEnabledFor(logging.DEBUG):
            log.debug("choiceLessChildListOfLists[%s] = %s;   template_nodes=%s" %(self.Name, unicode(choiceLessChildListOfLists), unicode(template_nodes))  )
        # store all relevant, valid ET objects in this list of lists:
        # main list is list of used childs,
        # the nested list is the list of ET objects from that child
        resultChildETObjLoL = []
        i = 0
        lastCNID = None
        lastCEID = None
        # temporarily store different choice possibilities for one choice node in this list:
        choices = []
        prefChoiceIdx = None
        for child in self._ChildElements:
            #debug: log.debug("child xpath="+child.xpath)
            #debug: log.debug("choices:" +str(choices))
            #
            # go through all children again and look at their choices and repeatability
            # Treat REPEATABILITY of childs by using the method "_addChildETobjectsToLoL"
            #
            if child.CNID == None or lastCNID <> child.CNID:
                # first check for potentially accumulated choices and decide which to add to result list:
                if len(choices) > 0:
                    resultChildETObjLoL += self._addChildETobjectsToLoL(child, self._getOneChoice(choices, prefChoiceIdx))
                    # clear stored choices for old CNID
                    prefChoiceIdx = None
                    choices=[]
                # Is this sub-element a new choice node ?
                if child.CNID == None:
                    # no choice node, add this childs objects to result list:
                    resultChildETObjLoL += self._addChildETobjectsToLoL(child, choiceLessChildListOfLists[i])
                else:
                    # new choice node !
                    #debug: log.debug("new CNID.")
                    lastCNID = child.CNID
                    lastCEID = child.CEID
                    if child.Preferred:
                        prefChoiceIdx = 0
                    choices = [choiceLessChildListOfLists[i]]
            else:
                # same coice node as before. check ChoiceElementID:
                #debug: log.debug("same cnid.")
                #debug: log.debug("choices:" +str(choices))
                if child.CEID == lastCEID:
                    # this sub-element belongs to the same Choice Element Group (ID)
                    choices[-1] += choiceLessChildListOfLists[i]
                else:
                    # this is another possibility for this choice-node:
                    lastCEID = child.CEID
                    choices.append(choiceLessChildListOfLists[i])
                # is this a preferred choice ?
                if child.Preferred:
                    prefChoiceIdx = len(choices)-1
                    #debug: log.debug("preferred. Idx="+str(prefChoiceIdx))
            i += 1
        # check if some unfinished choices are left:
        if len(choices) > 0:
            resultChildETObjLoL += self._addChildETobjectsToLoL(child, self._getOneChoice(choices, prefChoiceIdx))
        # debug
        if log.isEnabledFor(logging.DEBUG):
            log.debug(" *** resultChildETObjLoL[%s] = %s" %(self.Name, unicode(resultChildETObjLoL))  )
        #
        # PROCESS the List of Lists "resultChildETObjLoL" 
        # to prepare list of nodes for this element:
        #
        # 1) Make it a modified list of lists with each nested list of the same length
        #    Reason: If a nested list has more than 1 item,
        #    these items are not repeatable (see method _addChildETobjectsToLoL),
        #    so we have to generate copies of this element for them.
        # 2) Create a new ET object of this element for each nested list 
        #    with every item in the list being a subelement.
        #
        # Use raw template nodes in case there is no data in the result list
        # if the template nodes contain data = have attributes (this is no leaf node!)
        #
        if len(resultChildETObjLoL) > 0 and max([len(subList) for subList in resultChildETObjLoL]) > 0:
            #log.debug("CHILDREN OF %s WITH DATA: child list=%s" % (self.Name, str(resultChildETObjLoL)))
            #
            # multiply LoL to generate "even" list.
            resultChildETObjLoL = multiplyLoL(resultChildETObjLoL)
            if len(template_nodes)<1:
                # add an empty tNode if there is none existing yet
                if self.omitNamespace():
                    template_nodes.append(doc.createElement(self.Name))
                else:
                    template_nodes.append(doc.createElementNS(self.getNamespace(), self.Name))
            #
            # loop over all template nodes to add always the same child subelements
            #
            for tNode in template_nodes:
                #
                # loop through main list (will become new ET nodes for each template node)
                #
                if log.isEnabledFor(logging.DEBUG):
                    log.debug("Create nodes '%s' from this child LoL %s" %(self.Name, str(resultChildETObjLoL)))
                for childList in resultChildETObjLoL:
                    # use only cloned nodes to build new ET nodes
                    node = None
                    for obj in childList:
                        if node == None:
                            node = clone(tNode, deep=False)
                        node.append(clone(obj))
                    if node <> None:
                        nodeListOfThisElement.append(node)
        else:
            # no child data. use templates if they contain data or attributes
            #log.debug("CHILDREN OF %s WITHOUT DATA. Use non-empty template nodes directly" % (self.Name))
            for tNode in template_nodes:
                if tNode is not None and len(tNode)>0:
                    nodeListOfThisElement.append(tNode)
                else:
                    pass
                    #log.debug("TMPL NODE WITHOUT DATA. Removed %s" % ET.tostring(tNode))
        # debug
        if log.isEnabledFor(logging.DEBUG):
            log.debug(" ***** nodeListOfThisElement[%s] = %s" %(self.Name, unicode(nodeListOfThisElement)))
        # return data
        return nodeListOfThisElement

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def _getOneChoice(self, choices, prefChoiceIdx):
        '''returns a list of ET objects for one choice node. Try to use the preferred index first.'''
        tmp = []
        if prefChoiceIdx >= 0 and len(choices[prefChoiceIdx]) > 0:
            tmp = choices[prefChoiceIdx]
            log.debug("Used the preferred choice for element %s"%(self.xpath))
        else:
            for choice in choices:
                if len(choice) > 0:
                    # this is the first choice node containing data. Use it !
                    tmp = choice
                    # ignore other choices of this node:
                    if log.isEnabledFor(logging.DEBUG):
                        log.debug("Used the first choice filled with data for element %s"%(self.xpath))
                    break
        return tmp

    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def _addChildETobjectsToLoL(self, child, ETobjList):
        '''Returns a list of lists. Each list represents a subelement that will be present as a subnode of this node.
        If a child is repeatable, the nested list is therefore translated into a list of lists, each containing only one item.
        If a child is not repeatable, leave the nested list as it is to create nodes with different subnodes.'''
        if child.isRepeatable():
            # unbound or limited repeatable child. use limited ET objects in the list to create subelements.
            LoL = [[obj] for obj in ETobjList[:child.isRepeatable()]]
        else:
            # not repeatable, pass list of child objects to parent
            LoL = [ETobjList]
        return LoL
        
    # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    def createCMFET_2_1(self):
        '''Returns the ET node for this CMF element and its childElements according to CMF version 2.1.'''
        datanode = self.__class__.CMFAttributeClass.createCMFET_2_1(self)
        datanode.tag = '{%s}dataElement'%NS21
        preferred = self.Preferred
        if preferred == 0:
            preferred=None
        addOptionalAttributes(datanode, NS21, max=self.RepeatMax, dontRepeat=self.RepeatDont, preferredChoiceFlag=preferred, choiceElemId=self.CEID, choiceNodeId=self.CNID)
        if self.Namespace <> None:
            addOptionalAttributes(datanode, NS21, namespace=self.Namespace)
        if len(self._Attributes) > 0:
            attributesnode = subelement(datanode, NS21, 'attributes')
            for attr in self._Attributes:
                attrnode = attr.createCMFET_2_1()
                attributesnode.append(attrnode)
        if len(self._ChildElements) > 0:
            childrennode = subelement(datanode, NS21, 'childElements')
            for child in self._ChildElements:
                childrennode.append(child.createCMFET_2_1())
        return datanode
