# -*- coding: utf8 -*-
'''
CENTRAL PROTOCOL OPERATIONS
*********************************
***  CENTRAL PROTOCOL OPERATIONS
*********************************
$RCSfile: operations.py,v $
$Revision: 877 $
$Author: markus $
$Date: 2008-04-24 15:53:40 +0200 (Do, 24. Apr 2008) $

Module implementing the different operations as defined in the protocol <header><type>.
Also responsible for detecting the appropiate DBmodule.
'''


from biocase.wrapper.errorclasses    import *
from biocase.tools.xmlutils         import countElementsOfPath
from biocase.tools.various_functions  import unique
from biocase.wrapper.sql.base         import sqlBase
from biocase.wrapper.sql.operators  import comparisonOperator, logicalOperator, compressOperator
from biocase.wrapper.typeconverter    import XMLtransformerClass
from biocase.wrapper.cmf_base       import CMFClass
from biocase.wrapper.cmf_element    import TotalMatchedRecords

import biocase.configuration
cfg = biocase.configuration.Cfg()

import logging
log=logging.getLogger("pywrapper.operations")


# ----------------------------------------------------------------------------
class recordStatus:
    '''Class containing attribute information of element <content> like record state.'''
    def __init__(self):
        self.start = 0
        self.count = 0
        self.drop  = 0
        self.total = 0
        
    def __repr__(self):
        return "RecordStatus: start=%i, count=%i, drop=%i, total=%i"%(self.start, self.count, self.drop, self.total)

    def getNextPageIndex(self):
        '''Returns the index used for start of the next page or None if there is no more page'''
        if self.total > self.start+self.drop+self.count:
            return self.start+self.drop+self.count+1
        else:
            return None


# ------------------------------------------------------------------------------------
def getOperationsObjectForDBMod(psfObj, responseObj=None, removeCMFSubtreePaths=[], altRootTableAlias=None):
    # chose right DB-module as the parental class for the LogicManager Class
    try:
        modulename = 'biocase.wrapper.dbmod.dbmod_' + psfObj.dbms
        module = __import__(modulename, globals(), locals(), 'DBmodule')
        DBModuleClass = vars(module)['DBmodule']
        log.info("The %s database module has been used."%(modulename))
    except:
        log.error("ConnectionError. The %s database module has been detected."%(psfObj.dbms))
        raise ConnectionError(dbms=psfObj.dbms)
    # attention!!!
    # function continues at the bottom after inline class !!!    
    # ---------------------------------------------------------------------------------
    class operations(DBModuleClass):
        '''The central class for processing the request operations. holds the CMF-output object for full searches.'''
        # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
        def __init__(self, psfObj, removeCMFSubtreePaths=[], altRootTableAlias=None):
            #
            # init class instance
            # and store reference of log / psf object in inherited class
            #
            log.debug("DBModuleClass: %s"%str(DBModuleClass))
            DBModuleClass.__init__(self, psfObj)
            # the output CMF object. Instanciated only for CMF defined content.
            self.CMFObject = None
            # the request object reference
            self.requestObj = None
            # the psfObj
            self.psfObj = psfObj
            # parts of the CMF to be removed:
            self.removeCMFSubtreePaths = removeCMFSubtreePaths
            # alternative root table alias. If set this should be used instead of the normal default one!
            try:
                self.altRootTableAlias = int(altRootTableAlias)
            except:
                self.altRootTableAlias = None

        # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
        def __del__(self):
            # destruct class instance
            #DBModuleClass.__del__(self)
            pass

        # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
        def count(self, requestObj):
            '''The count operation as specified in the protocol. Returns the number of matching records to the search.'''
            self.requestObj = requestObj
            # create CMF-output object
            try:
                # build new CMF object
                self.CMFObject = CMFClass()
                self.CMFObject.loadCMFdata(self.psfObj.getCMFfilename(self.requestObj.reqSchema), psfObj=self.psfObj)
            except CouldNotPickleError:
                pass
            except:
                raise CMFError(cmf=self.requestObj.reqSchema)
            #
            # read the filter part of the request! This is a count.
            #
            requestObj.getFilterObjFromFilterString(self.CMFObject, self.requestObj.doc)
            return self._countTotalRecordsMatched(self.requestObj.filter)

        # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
        def scan(self, requestObj):
            '''The scan operation as specified in the protocol. 
            Scans the database for distinct values of one attribute.
            Returns a list of distinct values.'''
            #
            # prepare other objects
            #
            self.requestObj = requestObj
            # create CMF-output object
            self.CMFObject = self.getCMFobject(self.requestObj.reqSchema, altRootTableKey=self.altRootTableAlias)
            #
            # get the CMF element object for the concept/path to be scanned:
            #
            CMFobj = self.CMFObject.path2ElemObj.get(requestObj.scan, None)
            # Does it exist?
            if CMFobj != None:
                # check existing global filters of parental records
                FilterObj = compressOperator( CMFobj.getParentalFilterList() )
                if FilterObj is not None:
                    FilterObj = logicalOperator('and', FilterObj)
                    log.debug("Global CMF Filters integrated into the SQL: %s"%(unicode(FilterObj)))
                # Check if filter was secretly given in the protocol 
                # (official protocol does not support filters on scans)
                requestObj.getFilterObjFromFilterString(self.CMFObject, self.requestObj.doc)
                if requestObj.filter is not None:
                    log.debug("The scan request contained a hidden filter which was integrated: %s"%(unicode(requestObj.filter)))
                    if FilterObj is None:
                        FilterObj = requestObj.filter
                    else:
                        FilterObj.args += [requestObj.filter]
                FilterObj = compressOperator(FilterObj)
                if FilterObj is not None and FilterObj.isFalse():
                    # the overall filter of this request is False. So no db query needed: no results
                    log.info("The filter evaluated to False. No results without querying the db")
                    return []
                elif len(CMFobj.getSearchableDBAttrObjs()) > 0:
                    log.debug("Final fully compressed filter: %s"%unicode(FilterObj))
                    # create tranformer instance to convert values to XML strings
                    t = XMLtransformerClass()
                    t.Type = 'string'
                    # there may be several mappings per concept
                    # scan each of them and make the combined list unique again.
                    wasScanned = 0
                    rs = []
                    for mapping in CMFobj.mappings:
                        if mapping.isSearchable():
                            log.info("Scan mapping %s"%unicode(mapping))
                            # check if this mapping contains additional filters
                            if mapping.filterObj is not None:
                                FilterObj.args += [mapping.filterObj]
                                log.debug("Mapping contained additional filters. Total is now: %s"%unicode(FilterObj))
                            rs += self.execDistinct([mapping.getItems()[0]], FilterObj)
                            wasScanned = 1
                        else:
                            log.info("Mapping %s is not searchable and cant be scanned"%unicode(mapping))
                    if wasScanned == 1:
                        rs = [t.transformDBVal2XML(val, self.psfObj.encoding) for val in rs]
                        rs = unique(rs)
                        log.info("Scan operation succeeded.")
                        return rs
                    else:
                        # it is not allowed to search on any of the mappings!
                        raise RequestNotSearchableConceptError(CMFobj.xpath    )
                elif CMFobj.hasMappings():
                    raise RequestNotSearchableConceptError(CMFobj.xpath    )
                else:
                    raise RequestUnknownConceptError(requestObj.scan    )
            else:
                log.warn('No valid xpath was given for the scan search. The concept "%s" does not exist in the schema "%s"' % (requestObj.scan, self.requestObj.reqSchema))
                return []

        # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
        def search(self, requestObj):
            '''The full search operation as specified in the protocol.
            Reads the CMF file of the requested response schema and builds output object.
            Searches the DB and returns the full result set as a xml-dom object.'''
            log.debug("start of search operation")
            self.requestObj = requestObj
            # create CMF object of the response-CMF for output of XML
            # build new CMF object
            self.CMFObject = self.getCMFobject(self.requestObj.resSchema, self.removeCMFSubtreePaths, altRootTableKey=self.altRootTableAlias)
            #
            # build CMF object of the request-CMF for building the Where Clause objects later
            if requestObj.resSchema == requestObj.reqSchema:
                reqCMFobj = self.CMFObject
            else:
                reqCMFobj = self.getCMFobject(self.requestObj.reqSchema, altRootTableKey=self.altRootTableAlias)
            #
            # read the filter part of the request! This is a search.
            # Use the requested CMF and set the self.requestObj.filter property
            # check the global CMF filter objects for each COP when building the filter.
            requestObj.getFilterObjFromFilterString(reqCMFobj, self.requestObj.doc)
            # check if response CMF needs to count all matched records
            if self.CMFObject.needsTotalMatchedRecords():
                c = self._countTotalRecordsMatched(self.requestObj.filter)
                TotalMatchedRecords.count = unicode(c)
            #
            # query DB for result set in list form and use this recordset to
            # generate xml with the help of the CMFObject. 
            # Uses filterObject, start and limit from request object
            #
            (recordset, recStatus) = self._getSearchResultList()
            xmldom = self.CMFObject.createXMLResultET(recordset)
            recNumber = 0
            if xmldom <> None:
                # count RecID records ! Might be different from DB Hits:
                for path in self.CMFObject.recIdents:
                    node = xmldom
                    recNumber += countElementsOfPath(node, path)
            # update number of hits to real XML output:
            recStatus.drop  = recStatus.count - recNumber
            recStatus.count = recNumber
            # return results
            log.debug("end of search operation")
            return (xmldom, recStatus)

        # --------------------------------
        # -     private
        # --------------------------------
        
        # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
        def getCMFobject(self, namespace, removeCMFSubtreePaths=[], altRootTableKey=None):
            # create CMF object of the response-CMF for output of XML
            CMFObject = None
            try:
                # build new CMF object
                CMFObject = CMFClass()
                CMFObject.loadCMFdata(self.psfObj.getCMFfilename(namespace), psfObj=self.psfObj)
            except CouldNotPickleError:
                pass
            except:
                raise CMFError(cmf=namespace)
            if len(removeCMFSubtreePaths) > 0:
                # cut off the unused parts:
                for path in removeCMFSubtreePaths:
                    CMFObject.removeSubtree(path, self.psfObj)
            if altRootTableKey is not None:
                # string and integers are treated differently. Use only strings
                altRootTableKey = unicode(altRootTableKey)
                if CMFObject.altRootTableAliasSPICE.has_key(altRootTableKey) and CMFObject.altRootTableAliasSPICE[altRootTableKey] is not None:
                    log.info("Set new alternative Root Table Alias: %s" % CMFObject.altRootTableAliasSPICE[altRootTableKey])
                    CMFObject.setRootTableAlias( CMFObject.altRootTableAliasSPICE[altRootTableKey] )
                    CMFObject.initForDataProcessing( self.psfObj, CMFObject.altRootTableAliasSPICE[altRootTableKey] )
                else:
                    log.warn("Alternative Root Table Alias %s not found in CMF dict: %s" % (altRootTableKey, unicode(CMFObject.altRootTableAliasSPICE)))
            return CMFObject
            
        # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
        def _countTotalRecordsMatched(self, filterObj):
            log.debug("Counting total matched records with filter=%s"%unicode(filterObj))
            if filterObj is not None and filterObj.isFalse():
                return 0
            else:
                return self.execCount(self.CMFObject.tableTreeObj.rootTableAliasPK, filterObj)

        # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
        def _getSearchResultList(self):
            '''query DB for the full recordset used for XML output.
            Include the request filter, start and limit of records.
            '''
            log.debug("start of recordset retrieval")
            # the root table primary key as a DBAttributeObject
            recID = self.CMFObject.tableTreeObj.rootTableAliasPK
            log.debug("Record Identifier used: "+unicode(recID))
            # start with record number ...
            start = self.requestObj.start
            # limit amount of retrieved records to ...
            if self.requestObj.limit == -1973:
                limit = 0
            elif self.requestObj.limit <= 0:
                limit = self.psfObj.recLimit
            elif self.psfObj.recLimit <= 0:
                limit = self.requestObj.limit
            else:
                limit = min(self.requestObj.limit, self.psfObj.recLimit)
            # first check if a valid where clause exists.
            if isinstance(self.requestObj.filter, comparisonOperator) and self.requestObj.filter.arg1 == 'tmp':
                log.debug("magic pia")
                # magic pia
                recStatus = recordStatus()
                recStatus.start = 1
                recStatus.count = 1
                recStatus.drop  = 0
                recStatus.total = 1
                #
                # add XMLtype list to mapping objects in self.CMFObject.allDBAttr
                self.CMFObject.setXMLTypeListInMappingObjs()
                # produce result-set:
                rs = []
                for attrObj in [recID] + self.CMFObject.allDBAttr:
                    try:
                        if len( intersects(attrObj.XMLtypes, ('date','gyearmonth','gyear','gmonthday','gday','gmonth'))) > 0:
                            # date
                            x = self.requestObj.filter.arg2[2]
                        elif len( intersects(attrObj.XMLtypes, ('int','integer','long','short','unsignedint','unsignedlong','unsignedshort','nonpositiveinteger','nonnegativeinteger','positiveinteger','negativeinteger','float','double','decimal','boolean'))) > 0:
                            # int
                            x = self.requestObj.filter.arg2[1]
                        else:
                            # string
                            x = self.requestObj.filter.arg2[0]
                    except:
                        # no key. maybe its a primary key of a table not mapped in CMF. ignore
                        x = self.requestObj.filter.arg2[0]
                    rs.append(x)
                self.requestObj.filter = None
                self.requestObj.doc = ''
                return ([tuple(rs)],recStatus)
            else:
                # ask for recordIDs that match the query
                # using the select-limit approach only selecting the limited records
                if self.requestObj.filter is not None and self.requestObj.filter.isFalse():
                    # the filter is false already. No records match the query!
                    AllIDs = []
                elif sqlBase.templates['select-limit'] == None:
                    # dbmod does not support sql limit/top.
                    AllIDs = [ item[0] for item in self.execSelect([recID], self.requestObj.filter, orderByList=[recID])]
                else:
                    log.debug("Use SQL LIMIT to limit the first result set of IDs only to the maximum of %i records." %(start+limit))
                    AllIDs = [ item[0] for item in self.execSelect([recID], self.requestObj.filter, orderByList=[recID], limit=start+limit)]
                # remove duplicates from result set (due to joins possible !)
                log.debug("All RecIDs retrieved from DB: %s"%unicode(AllIDs) )
                AllIDs = unique(AllIDs)
                # sort IDs so they will always be in the same order
                AllIDs.sort()
                # create new list of wanted IDs for full output
                if limit > 0 and len(AllIDs) >= (start+limit):
                    IDs = AllIDs[start:(start+limit)]
                    if limit == self.psfObj.recLimit:
                        log.info("The Provider limited the resulting maximum record number to " + unicode(limit))
                else:
                    IDs = AllIDs[start:]
                # query DB and retrieve full recordset with all requested data if there are hits
                recStatus = recordStatus()
                recStatus.start = start
                recStatus.count = len(IDs)
                recStatus.drop  = 0
                recStatus.total = len(AllIDs)
                if len(IDs) > 0:
                    rs = self.execSelect( [recID] + self.CMFObject.allDBAttr, comparisonOperator('in', recID, IDs), orderByList=[recID])
                    # sort result recordset after recIDs (so that output order is always the same):
                    rs = list(rs)
                    rs.sort()
                else:
                    rs = []
            log.debug("end of recordset retrieval")
            return (rs, recStatus)

    # -----continue function -------------------------------------------------------
    operationsObj = operations(psfObj, removeCMFSubtreePaths, altRootTableAlias)
    # tell response object which DBmod was used:
    if responseObj is not None:
        responseObj._usedDBmod = unicode(operationsObj)
    # return object
    return operationsObj
    
