import logging
import appconfig as config
from google.appengine.ext import db
from google.appengine.api import memcache
from orangescape.model.engine.exception.Error import InstanceNotFoundError

from Data import PrimaryData, SecondaryData
from orangescape.model.server.wrapper.Model import DynamicModel
from orangescape.model.server.gae.GFSModels import Attachment, AttachmentVersion
from AbstractSession import AbstractSession
from orangescape.model.server.Constants import GFS_MAX_COMMIT_SIZE
from orangescape.model.server.util.gfsutil import __is_valid_keyname__, put,put_async_write
from orangescape.model.server.PerfMonitor import stopPerfMonitor, startSessionFormulaMonitor
import datetime
class GFSSession(AbstractSession):
    readCache = None

    def __init__(self, application):
        AbstractSession.__init__(self, application)
        self.readCache = dict()

    def createModel(self, metadata,keyName=None):
        if keyName:
            if not __is_valid_keyname__(keyName):
                raise Exception('Cannot create the sheet %s with the given keyName %s'%(metadata.getSheetName(), keyName))
            id = keyName
        else:
            id = self.__generate_pk__(metadata)
        model = AbstractSession.createModel(self, metadata, id)
        return model

    def readFromDatastore(self, sheetId, metadata):
        if not __is_valid_keyname__(sheetId):
            """Since key name will not begin with number so no need to to even disturb the big table"""
            return None
        sheet = self.memcacheGet(sheetId)
        if sheet is None or sheet.getSheetMetadataId() != metadata.getSheetId():
            sheet = self.getClass().get_by_key_name(sheetId)
            if sheet is not None and sheet.getSheetMetadataId() == metadata.getSheetId():
                self.memcacheSet(sheetId, sheet)
            else:
                return None
        return sheet

    def getListFromDatastore(self, model, referenceCell):
        startTime = datetime.datetime.now()
        key = self.getQueryCacheKey(model.getSheetId(), referenceCell)
        idList = self.memcacheGet(key)

        dbResults = None
        if idList is None:
            idList, dbResults = self.setQueryCache(model, referenceCell)
        cacheData = self.checkWithMemcache(idList, dbResults)
        return cacheData

    def setQueryCache(self, model, referenceCell):
        limit = 2000
        startTime = datetime.datetime.now()
        q = self.getQuery()
        q.filter(referenceCell.getSheetId(), model.getSheetId())
        q.filter('SheetMetadataId', referenceCell.getRowMetadata().getSheetMetadata().getSheetId())
        dbResults = q.fetch(limit)
        noRes = len(dbResults)
        if noRes>0:
            idList = list()
            for result in dbResults:
                idList.append(result.getSheetId())
            self.memcacheSet(self.getQueryCacheKey(model.getSheetId(), referenceCell), idList)
            return idList, dbResults
        else:
            self.memcacheSet(self.getQueryCacheKey(model.getSheetId(), referenceCell), list())
            return list(), list()

    def checkWithMemcache(self, idList, dbResults = None):
        if idList is None or len(idList)==0:
            return dict()
        dictRef = self.memcacheGetMulti(idList)
        #logging.debug('%s - %s - %s'%(idList, dbResults, dbResults is None))
        if len(dictRef) > 0:
            if len(dictRef) < len(idList):
                diffIdList = [sheetId for sheetId in idList if not dictRef.__contains__(sheetId)]
                if len(diffIdList)>0:
                    dbResults = self.getClass().get_by_key_name(diffIdList)
        elif not dbResults or len(dbResults)==0:
            dbResults = self.getClass().get_by_key_name(idList)
        if dbResults:
            for result in dbResults:
                if result is not None:
                    dictRef[result.getSheetId()] = result
        return dictRef

    def queryFromDataStore(self, metadata, baseQuery, limit=2000, offset=0, orderBy=None, isDescending=False):
        pm = startSessionFormulaMonitor({'name':'QueryDS','sheetName':metadata.getSheetName(),'type':'session','inputValues':baseQuery})
        limit = limit or 2000
        q = self.getQuery()
        q.filter('SheetMetadataId', metadata.getSheetId())
        self.__apply_filter__(q, baseQuery, orderBy, isDescending)
        stopPerfMonitor(pm)
        return q.fetch(limit, offset)

    def adv_query_from_ds(self, metadata, baseQuery, limit=None, offset=0, orderBy=None, isDescending=False):
        limit = limit or 2000
        q = self.getQuery()
        q.filter('SheetMetadataId', metadata.getSheetId())
        self.__apply_adv_filter__(metadata, q, baseQuery, orderBy, isDescending)
        return q.fetch(limit, offset)

    def __apply_adv_filter__(self, metadata, q, baseQuery, orderBy=None, isDescending=False, cursor = None):
        if baseQuery:
            self.constructFilter(metadata, q, baseQuery)
        if orderBy:
            if isDescending:
                q.order('-'+orderBy)
            else:
                q.order(orderBy)
        if cursor:
            q.with_cursor(cursor)

    def constructFilter(self, metadata, q, baseQuery):
        bool_operator, filter_params = baseQuery
        if bool_operator == 'or':
            raise Exception('Bad Filter Error OR condition not supported in appengine BigTable, used in the model %s'%(metadata.getSheetName()))
        for tup in filter_params:
            if len(tup)==2:
                self.constructFilter(metadata, q, tup)
            else:
                lhs, operator, rhs = tup
                if operator.lower()=='like':
                    rhs=rhs if not lhs=='__key__' else db.Key(rhs)
                    if rhs[0]=='%':
                         rhs=rhs[1:]
                    if rhs[-1]=='%':
                        rhs=rhs[:-1]
                    q.filter(lhs+' <', rhs+ u"\ufffd")
                    operator=' >='
                else:
                    operator = ' '+operator
                q.filter(lhs+operator, db.Key(rhs) if lhs=='__key__' else None if rhs==0.0 or rhs=='' else rhs)

    def updateQueryCache(self, model, updatedIdCache):
        referenceCells = model.getSheetMetadata().getManyToOneList()
        for refCell in referenceCells:
            if self.isReferenceChanged(model, refCell):
                #logging.info('Reference is changed for the '+model.getSheetId()+' - '+refCell.getCellName())
                if model.get(refCell) is not None and len(model.get(refCell))>0:
                    "Need to add this id to new parent id cache"
                    if __is_valid_keyname__(model.get(refCell)):
                        newKey = self.getQueryCacheKey(model.get(refCell), refCell)
                        newParentIdList = updatedIdCache.get(newKey)
                        if newParentIdList is not None:
                            newParentIdList.append(model.getSheetId())
                    else:
                        logging.error('GFSSession updateQueryCache: Invalid SheetId :%s stored in the reference cell %s %s in the model %s for the instance %s'%(model.get(refCell), refCell.getCellDisplayName(), refCell.getCellName() or '', model.getSheetMetadata().getSheetName(), model.getSheetId()))
                        raise Exception('GFSSession updateQueryCache: Invalid SheetId :%s stored in the reference cell %s %s in the model %s for the instance %s'%(model.get(refCell), refCell.getCellDisplayName(), refCell.getCellName() or '', model.getSheetMetadata().getSheetName(), model.getSheetId()))
                if model.getOriginalData().get(refCell) is not None and len(model.getOriginalData().get(refCell))>0:
                    "Needs to remove this id from the old parent id cache "
                    if __is_valid_keyname__(model.getOriginalData().get(refCell)):
                        oldKey =  self.getQueryCacheKey(model.getOriginalData().get(refCell), refCell)
                        self.removeIdFromQueryCache(oldKey, model, updatedIdCache)
                    else:
                        logging.error('GFSSession updateQueryCache: Invalid SheetId (old value) :%s stored in the reference cell %s %s in the model %s for the instance %s'%(model.getOriginalData().get(refCell), refCell.getCellDisplayName(), refCell.getCellName() or '', model.getSheetMetadata().getSheetName(), model.getSheetId()))

    def removeIdFromQueryCache(self, key, model, updatedIdCache):
        list = updatedIdCache.get(key)
        if list is not None:
#            logging.info('Removing the id '+model.getSheetId()+' from the old parent query cache '+key)
            list.remove(model.getSheetId())

    def commitInGFS(self,asyncWrite=False):
        "Commit the updated sheets in GFS"
        dataDict = dict()
        updatedIdCache = dict()
        idList = list()
        for modelDict in self.updatedSheets.values():
            for model in modelDict.values():
                for refCell in model.getSheetMetadata().getManyToOneList():
                    if self.isReferenceChanged(model, refCell):
                        if model.get(refCell) is not None:
                            if __is_valid_keyname__(model.get(refCell)):
                                qryCacheKey = self.getQueryCacheKey(model.get(refCell), refCell)
                                if not idList.__contains__(qryCacheKey):
                                    idList.append(qryCacheKey)
                            else:
                                logging.error('GFSSession commitInGFS: Invalid SheetId :%s stored in the reference cell %s %s in the model %s for the instance %s'%(model.get(refCell), refCell.getCellDisplayName(), refCell.getCellName() or '', model.getSheetMetadata().getSheetName(), model.getSheetId()))
                                raise Exception('GFSSession commitInGFS: Invalid SheetId :%s stored in the reference cell %s %s in the model %s for the instance %s'%(model.get(refCell), refCell.getCellDisplayName(), refCell.getCellName() or '', model.getSheetMetadata().getSheetName(), model.getSheetId()))
                        if model.getOriginalData().get(refCell) is not None:
                            if __is_valid_keyname__(model.getOriginalData().get(refCell)):
                                qryCacheKey = self.getQueryCacheKey(model.getOriginalData().get(refCell), refCell)
                                if not idList.__contains__(qryCacheKey):
                                    idList.append(qryCacheKey)
                            else:
                                logging.error('GFSSession commitInGFS: Invalid SheetId (old value) :%s stored in the reference cell %s %s in the model %s for the instance %s'%(model.getOriginalData().get(refCell), refCell.getCellDisplayName(), refCell.getCellName() or '', model.getSheetMetadata().getSheetName(), model.getSheetId()))


        for modelDict in self.deletedSheets.values():
            for model in modelDict.values():
                for refCell in model.getSheetMetadata().getManyToOneList():
                    if model.getOriginalData().get(refCell) is not None:
                        if __is_valid_keyname__(model.getOriginalData().get(refCell)):
                            qryCacheKey = self.getQueryCacheKey(model.getOriginalData().get(refCell), refCell)
                            if not idList.__contains__(qryCacheKey):
                                idList.append(qryCacheKey)
                        else:
                            logging.error('GFSSession commitInGFS-delete: Invalid SheetId (old value) :%s stored in the reference cell %s %s in the model %s for the instance %s'%(model.getOriginalData().get(refCell), refCell.getCellDisplayName(), refCell.getCellName() or '', model.getSheetMetadata().getSheetName(), model.getSheetId()))


        updatedIdCache = self.memcacheGetMulti(idList)
        for modelDict in self.updatedSheets.values():
            for id, model in modelDict.iteritems():
                dataDict[model.getSheetId()] = model.getDataObject()
                self.updateQueryCache(model, updatedIdCache)
                model.updateOriginalData()

        dataObjList = dataDict.values()
        size = len(dataObjList)
        if size>0:
            self.put_into_datastore(dataObjList,asyncWrite)
            self.memcacheSetMulti(dataDict)

        dataDict.clear()
        for modelDict in self.deletedSheets.values():
            for id, model in modelDict.iteritems():
                dataDict[model.getSheetId()] = model.getDataObject()
                referenceCells = model.getSheetMetadata().getManyToOneList()
                for refCell in referenceCells:
                    if model.getOriginalData().get(refCell) is not None:
                        key = self.getQueryCacheKey(model.getOriginalData().get(refCell), refCell)
                        self.removeIdFromQueryCache(key, model, updatedIdCache)


        if len(dataDict)>0:
            if config.env != config.SL and logging.getLogger().level == logging.DEBUG:
                logging.debug('Deleting the records from GFS %s'%(dataDict.keys()))
            db.delete(dataDict.values())
            self.memcacheDeleteMulti(dataDict)

        if len(updatedIdCache)>0:
            self.memcacheSetMulti(updatedIdCache)

        self.updatedSheets.clear()
        self.deletedSheets.clear()

        self.commitAttachments()

    def put_into_datastore(self, entities, asyncWrite=False):
        if asyncWrite == True:
            put_async_write(entities)
        else:
            put(entities)

    def isReferenceChanged(self, model, refCell):
        oldData = model.getOriginalData()
        if oldData.get(refCell) is None and model.get(refCell) is None:
            return False
        if oldData.get(refCell) is None and model.get(refCell) is not None:
            return True
        if model.get(refCell) is None and oldData.get(refCell) is not None:
            return True
        return model.get(refCell) != oldData.get(refCell)

    def getQueryCacheKey(self, foreignKeyValue, referenceCell):
        return ''.join([foreignKeyValue, '.', referenceCell.getSheetId(), '.', referenceCell.getRowMetadata().getSheetMetadata().getSheetId()])

    def memcacheGet(self, key):
        data = self.readCache.get(key)
        if data is None:
            data = memcache.get(key)
            if data is not None:
                self.readCache[key] = data
        return data

    def memcacheSet(self, key, value):
        self.readCache[key] = value
        return memcache.set(key, value, time=config.CACHE_EXPIRATION)

    def memcacheGetMulti(self, keyList):
        dataDict = self.getDictMulti(keyList)
        if len(dataDict) > 0:
            if len(dataDict) == len(keyList):
                return dataDict
            else:
                tempList = [sheetId for sheetId in keyList if not dataDict.__contains__(sheetId)]
                cacheData = memcache.get_multi(tempList)
                for k, v in cacheData.iteritems():
                    self.readCache[k] = v
                    dataDict[k] = v
        else:
            dataDict = memcache.get_multi(keyList)
            self.setDictMulti(dataDict)
        return dataDict

    def memcacheSetMulti(self, mapping):
        self.setDictMulti(mapping)
        failedKeys = memcache.set_multi(mapping, time=config.CACHE_EXPIRATION)
        if failedKeys:
            logging.error('Failed to update in memcache keys %s out of %s. %s'%(len(mapping), len(failedKeys), failedKeys))
            faildData = {}
            for key in failedKeys:
                faildData[key] = mapping.get(key)
            while not memcache.delete_multi(faildData):
                logging.warn('memcache Delete multi failed')
            for key, entity in faildData.iteritems():
                if not memcache.set(key, entity, time=config.CACHE_EXPIRATION):
                    logging.error('Re-try failed for updating memcache for the id %s'%(key))

    def memcacheDeleteMulti(self, mapping):
        self.deleteDictMulti(mapping.keys())
        return memcache.delete_multi(mapping)

    def getDictMulti(self, keyList):
        localDict = dict()
        for key in keyList:
            obj = self.readCache.get(key)
            if obj is not None:
                localDict[key]=obj
        return localDict

    def setDictMulti(self, mapping):
        for key,val in mapping.iteritems():
            self.readCache[key]=val

    def deleteDictMulti(self, keyList):
        for key in keyList:
            if self.readCache.__contains__(key):
                del self.readCache[key]

    def getUser(self):
        return self.user

    def commitAttachments(self):
        updatedAttachments = {}
        versionedAttachments = {}
        for id, content in self.attachmentCache['created'].iteritems():
            attachment = self.createAttachment(id)
            attachment.content = content
            updatedAttachments[id] = attachment

        for id, content in self.attachmentCache['updated'].iteritems():
            attachment = Attachment(key_name = id)
            attachment.content = content
            updatedAttachments[id] = attachment

        for id, versionDetails in self.attachmentVersionCache.iteritems():
            attachmentVersion = AttachmentVersion()
            attachmentVersion.attachment = updatedAttachments[id]
            attachmentVersion.versionedAt = datetime.datetime.now()
            attachmentVersion.versionedBy = self.getUser().getSheetName()
            attachmentVersion.content = versionDetails['content']
            attachmentVersion.comments = str(versionDetails['comments'])
            versionedAttachments[id] = attachmentVersion

#        if len(updatedAttachments) > 0:
#            logging.info('Commiting %s attachments'%(len(updatedAttachments)))
        for key, updatedAttachment in updatedAttachments.iteritems():
            updatedAttachment.put()

        if len(versionedAttachments) > 0:
            logging.debug('No of attachments: %s'%(len(versionedAttachments)))

        for key, versionedAttachment in versionedAttachments.iteritems():
            versionedAttachment.put()

        deletedAttachments = Attachment.get_by_key_name(self.attachmentCache['deleted'].keys())
        toBeDeleted = []
        toBeDeletedVersions = []
        for deletedAttachment in deletedAttachments:
            if deletedAttachment:
                toBeDeleted.append(deletedAttachment)

                childList = deletedAttachment.attachmentVersionList
                toBeDeletedVersions.extend(childList)

        while toBeDeleted:
            toBeDeleted.pop().delete()
        while toBeDeletedVersions:
            toBeDeletedVersions.pop().delete()

        self.clearAttachmentCache()

    def getAttachmentFromDS(self, id):
        attachment = self.getAttachmentRecord(id)
        if attachment:
            return attachment.content
        return None

    def getAttachmentRecord(self, id):
        return Attachment.get_by_key_name(id)

    def getAttachmentVersionList(self, id, baseQuery, limit=50, offset=0, orderBy=None, isDescending=False, cursor = None):
        attachment = self.getAttachmentRecord(id)
        q = db.Query(AttachmentVersion)
        q.filter('attachment', attachment)
        self.__apply_filter__(q, baseQuery, orderBy, isDescending)
        return q.fetch(limit, offset)

    def getVersionFromDS(self, key):
        return AttachmentVersion.get(key)

    def replaceAttachment(self, id, versionKey):
        attachment = self.getAttachmentRecord(id)
        version = self.getVersionFromDS(versionKey)
        if version:
            versionContent = version.content
            if versionContent:
                attachment.content = versionContent
                logging.debug('Replacing the attachment %s with the version %s'%(id, versionKey))
                attachment.put()
            else:
                logging.error('Replace attachment %s failed. The Version content is empty'%(id))
        else:
            logging.error('Replace attachment %s failed. The Version with key %s doesn\'t exist'%(id, versionKey))

    def getMultiFromDatastore(self, metadataIdDict):
        return self.checkWithMemcache(metadataIdDict.keys())

    def __apply_filter__(self, q, filterDict, orderBy=None, isDescending=False, cursor = None):
        if filterDict is not None:
            for key, value in filterDict.iteritems():
                col = key[0]
                operator = ''
                if len(key)>1:
                    if key[1].lower()=='like':
                        value=value if not col=='__key__' else db.Key(value)
                        if value[0]=='%':
                             value=value[1:]
                        if value[-1]=='%':
                            value=value[:-1]
                        q.filter(col+' <', value+ u"\ufffd")
                        operator=' >='
                    else:
                        operator = ' '+key[1]
                q.filter(col+operator, db.Key(value) if col=='__key__' else None if value==0.0 or value=='' else value)

        if orderBy:
            if isDescending:
                q.order('-'+orderBy)
            else:
                q.order(orderBy)
        if cursor:
            q.with_cursor(cursor)

    def count(self, metadata, baseQuery):
        q  = self.getQuery()
        q._keys_only = True
        q.filter('SheetMetadataId', metadata.getSheetId())
        self.__apply_adv_filter__(metadata, q, baseQuery)
#        self.__apply_filter__(q, baseQuery)
        limit = 100000#1000000000
        return q.count(limit)

    def createAttachment(self, keyName):
        return Attachment(key_name = keyName)

    def keys(self, metadata, baseQuery, orderBy=None, isDescending=False):
        q  = self.getQuery()
        q._keys_only = True
        q.filter('SheetMetadataId', metadata.getSheetId())
        self.__apply_adv_filter__(metadata, q, baseQuery, orderBy, isDescending)
#        self.__apply_filter__(q, baseQuery, orderBy, isDescending)
        limit = 100000000
        keys = [str(k) for k in q.fetch(limit)]
        return keys

    def get_by_keys_from_ds(self, metadata, keys):
        return db.get(keys)

    def aggregate_query(self, metadata, limit, baseQuery, measures, dimensions, operations, orderBy = None, isDescending = False, offset=0):
        logging.info('test in sql session before perfM')
        pm = startSessionFormulaMonitor({'name':'QueryDS_aggregate','sheetName':metadata.getSheetName(),'type':'session','inputValues':baseQuery})
        from orangescape.model.server.gae.GFSReportQuery import AggregateQuery
#        entities = self.query(metadata, limit, baseQuery, None, orderBy, isDescending, offset)
        entities = self.advQuery(metadata, limit, baseQuery, None, orderBy, isDescending, offset)
        rowList =  AggregateQuery(self, metadata, entities, measures, dimensions, operations).execute()
        stopPerfMonitor(pm,len(rowList))
        return rowList

    def report_query(self, metadata, limit, baseQuery, properties, orderBy = None, isDescending = False, offset=0, inMemorySorting=False):
        entities = list()
        if inMemorySorting and orderBy:
            entities = self.adv_query_from_ds(metadata, baseQuery, limit, offset)
            orderByCell = metadata.getCellMetadataById(orderBy)
            entities.sort(lambda x, y:cmp(x.get(orderByCell), y.get(orderByCell)))
            if isDescending:
                entities.reverse()
        else:
            entities = self.adv_query_from_ds(metadata, baseQuery, limit, offset, orderBy, isDescending)
        searchResult = [[entity.get(metadata.getPublicCellMetadata(cellName)) for cellName in properties] for entity in entities]
        return searchResult

class PrimaryGFSSession(GFSSession):
    def __init__(self, application):
        GFSSession.__init__(self, application)

    def createDataObject(self, metadata, keyname):
#        logging.info('Creating entity %s (%s)from PrimaryGFSSession...'%(metadata.getSheetName(), keyname))
        data = PrimaryData(key_name=keyname)
        data.SheetId = keyname
        return data

    def getQuery(self):
        return db.Query(PrimaryData)

    def getClass(self):
        return PrimaryData

    def commit(self):
        self.commitInGFS()

class SecondaryGFSSession(GFSSession):

    def __init__(self, application):
        GFSSession.__init__(self, application)

    def createDataObject(self, metadata, keyname):
        data = SecondaryData(key_name=keyname)
        data.SheetId = keyname
        return data

    def getQuery(self):
        return db.Query(SecondaryData)

    def getClass(self):
        return SecondaryData

    def commit(self):
        self.commitInGFS()

class TransientSession(GFSSession):
    def __init__(self, application):
        GFSSession.__init__(self, application)

    def createDataObject(self, metadata, keyname):
        data = PrimaryData(key_name=keyname)
        data.SheetId = keyname
        return data

    def put(self, model):
        metadata = model.getSheetMetadata()
        metadataId = metadata.getSheetId()
        sheets = self.updatedSheets.get(metadataId)
        if sheets is None:
            sheets = dict()
            self.updatedSheets[metadataId] = sheets
        sheets[model.getSheetId()] = model

        self.memcacheSet(model.getSheetId(), model.getDataObject())

    def get(self, sheetId, metadata):
        metadataId = metadata.getSheetId()

        sheets = self.updatedSheets.get(metadataId)
        if sheets is None:
            sheets = dict()
            self.updatedSheets[metadataId] = sheets
        sheet = sheets.get(sheetId)
        if sheet is not None:
            return sheet
        if self.deletedSheets.__contains__(metadataId) and self.deletedSheets[metadataId].__contains__(sheetId):
            "Object is deleted from the session but not yet committed"
            return None
        sheet = self.memcacheGet(sheetId)
        if sheet is not None:
            model = DynamicModel(sheet, metadata, self)
            sheets[model.getSheetId()] = model
            return model
        return None

    def getQuery(self):
        return db.Query(PrimaryData)
    def commit(self):
        pass
