#!/usr/bin/python
#
#

import getopt
import sys
import os
import logging
import logging.handlers
import ConfigParser
import imaplib
import re
import json
import tempfile
import base64
import fcntl
import getpass
from Crypto.Cipher import PKCS1_v1_5
from Crypto.PublicKey import RSA

versionstring='benno-imapsync version 2.6.2'

class accountConfig:
    """Configuration reader and account initialization"""
    def __init__(self,configfile):
        self.defaults = {
            'dbtype':'sqlite3',
            'userdb':'/var/lib/benno-web/bennoweb.sqlite',
            'dbhost':'localhost',
            'dbuser':'benno',
            'dbpass':'secret',
            'dbname':'benno',
            'keyfile':'',
            'keypass':'',
            'logfile':'',
            'envelope':False,
            'disabled':False,
            'inboxdir':'/srv/benno/inbox',
            'skipfolders':[],
            'ssl':True
        }
        self.config=ConfigParser.SafeConfigParser(self.defaults)
        try:
            self.config.read(configfile)
        except ConfigParser.MissingSectionHeaderError:
            sys.exit("Config file format incorrect")
        # kein Logging bevor Initialisierung!
        #logging.debug("Load configuration from \"%s\"", configfile)


    def get(self,section,option):
        """Fetch configuration value"""
        if self.config.has_section(section):
            return self.config.get(section,option)
        else:
            return self.config.get('DEFAULT',option)


    def getboolean(self,section,option):
        """Fetch boolean configuration value"""
        # Python 2.7 has not implemented automagic access to DEFAULT for
        # getboolean
        value=False
        try:
            if self.config.has_section(section):
                value=self.config.getboolean(section,option)
            else:
                value=self.config.getboolean('DEFAULT',option)
        except Exception:
            value=self.defaults[option]
        return value


    def getAccountList(self):
        accountlist=[]
        conn=DBaccess(self).conn
        c=conn.cursor()
        try:
            sql="SELECT id,imapuser,imaphost,password,imapstatus,status FROM imapuser"
            c.execute(sql)
            for row in c:
                try:
                    account=self.initAccount(row[0],row[1],row[2],row[3],row[4],row[5])
                except Exception, e:
                    logging.warn('Initialization of %s@%s failed: %s' % (row[0],row[1],e))
                    continue
                if not account:
                    continue
                accountlist.append(account)
        except Exception, e:
            logging.debug("Database error while executing \"%s\"" % sql)
            logging.error("Cannot read data from imapuser table: %s" % e)
            sys.exit(5)
        c.close()
        conn.close()
        return accountlist


    def initAccount(self,id,imapuser,imaphost,pwstring,imapstatus,accountstatus):
        logging.debug('Initialize account %s' % imapuser)
        account=imapAccount(imapuser,imaphost,imapstatus,accountstatus)
        # strip leading '$rsa$_' marker and decode base64 encstring
        try:
            account.setPassword(base64.b64decode(pwstring.split('_',1)[1]))
        except IndexError, e:
            # missing '$rsa$_' prefix
            account.setPassword(base64.b64decode(pwstring))
        except Exception,e:
            account.setAccountstatus(2)
            logging.warn("Cannot set password string: %s" % e)
        account.setEnvelopeTo(self.getboolean(imapuser,'envelope'))
        return account



class imapAccount:
    """IMAP account data"""
    def __init__(self,imapuser,imaphost,jsonstatus,accountstatus):
        self.imapuser=imapuser
        self.setImaphost(imaphost)
        self.status=Status(jsonstatus)
        self.accountstatus=accountstatus
        self.id=imapuser+'@'+imaphost
        self.password=''

    def setImaphost(self,hostname):
        try:
            host,port=hostname.split(':',1)
            self.hostname=host
            self.port=port
        except ValueError:
            self.hostname=hostname
            self.port=0

    def setPassword(self,password):
        self.password=password

    def setAccountstatus(self,status):
        self.accountstatus=status

    def setEnvelopeTo(self,envelope_flag):
        self.envelopeTo=envelope_flag

    def getImapuser(self):
        return self.imapuser

    def getImaphost(self):
        return self.hostname

    def getImapport(self):
        return self.port

    def getPassword(self):
        return self.password

    def getAccountstatus(self):
        return self.accountstatus

    def getStatus(self,mailbox):
        """Returns [UIDVALIDITY,UIDNEXT] for mailbox of account"""
        return self.status.getStatus(mailbox)

    def isEnvelopeTo(self):
        return self.envelopeTo

    def save(self,config):
        """Save user data or update user password"""
        conn=DBaccess(config).conn
        conn.text_factory = str
        c=conn.cursor()
        try:
            jsonstatus=self.status.getJSON()
            sql="UPDATE imapuser SET imapstatus='%s' WHERE imapuser='%s' AND imaphost='%s'" % (jsonstatus,self.imapuser,self.hostname)
            c.execute(sql)
            logging.debug("Update IMAP status for %s", self.id)
        except Exception, e:
            logging.error("Cannot update IMAP status for %s: %s", self.id,e)
        c.close()
        conn.commit()
        conn.close


class IMAPConn:
    """IMAP connection to mailbox"""
    def __init__(self,imaphost,imapport,imapuser,ssl=True):
        self.imaphost=imaphost
        self.imapport=imapport
        self.imapuser=imapuser
        self.ssl=ssl
        self.id=imapuser+'@'+imaphost


    def login(self,password):
        if self.ssl == True:
            if self.imapport==0:
                imapport=993
            logging.debug('Open encrypted connection.');
            connection=imaplib.IMAP4_SSL(self.imaphost,imapport)
        else:
            if self.imapport==0:
                imapport=143
            logging.debug('Open plaintext connection.');
            connection=imaplib.IMAP4(self.imaphost,imapport)
        try:
            ret,status=connection.login(self.imapuser,password)
        except Exception:
            raise Exception('Login error.')
        self.connection=connection


    def getStatus(self,mailbox):
        """Returns a list [UIDVALIDITY, UIDNEXT of this mailbox"""
        connection=self.connection
        connection.select(mailbox,True)
        mbox_status=[]
        ret,status=connection.status(mailbox,"(UIDVALIDITY)")
        if ret=='OK':
            #re.search('\((\S+)\s+(\S+)\)',status[0]).groups()
            param,value=re.search('\((\S+)\s+(\S+)\)',status[0]).groups()
            mbox_status.append(value)
        else:
            #raise IMAPError(status)
            logging.error('Error detecting UIDVALIDTY of %s: %s[%s]',mailbox,ret,status)
            mbox_status.append(-1)
        ret,status=connection.status(mailbox,"(UIDNEXT)")
        if ret=='OK':
            param,value=re.search('\((\S+)\s+(\S+)\)',status[0]).groups()
            mbox_status.append(value)
        else:
            #raise IMAPError(status)
            logging.error('Error detecting UIDNEXT of %s: %s[%s]',mailbox,ret,status)
            mbox_status.append(0)
        return mbox_status


    def listFolder(self):
        connection=self.connection
        mailboxes=[]
        logging.debug('Fetch folder list.')
        ret,retlines=connection.list()
        list_pattern=re.compile(r'\((?P<flags>.*?)\) "(?P<delimiter>.*)" (?P<name>.*)')
        if ret=='OK':
            for line in retlines:
                logging.debug('  Server return: %s',line)
                flags,delimiter,mailbox=list_pattern.match(line).groups()
                mailbox=mailbox.strip('"')
                mailboxes.append(mailbox)
        return mailboxes


    def fetchMails(self,inboxdir,folder,uidnext,envelope_to=''):
        connection=self.connection
        ret,status=connection.select(folder,True)
        logging.debug("Status of folder \"%s\": %s" % (folder,status))
        if ret!='OK':
            raise IMAPError(status)
        searchstring='(UID %s:*)' % uidnext
        logging.debug("Search messages: %s",searchstring)
        result,data = connection.uid('SEARCH', None, searchstring)
        logging.debug("Search result: %s/%s" % (result,data))
        uidlist=data[0].split()
        logging.info("Load %s message(s) from %s/%s to %s" % (len(uidlist),self.imapuser,folder,inboxdir))
        for uid in uidlist:
            # fetch UIDs
            # PEEK does not set the \Seen flag implicitly
            typ,head=connection.UID('FETCH', uid, '(BODY.PEEK[HEADER])')
            typ,text=connection.UID('FETCH', uid, '(BODY.PEEK[TEXT])')
            tmpfh,tmppath=tempfile.mkstemp('.tmp',uid+'_',inboxdir)
            if envelope_to:
                os.write(tmpfh,'X-REAL-RCPTTO: %s\r\n' % envelope_to)
            try:
                os.write(tmpfh,head[0][1])
                os.write(tmpfh,text[0][1])
            except Exception, e:
                logging.warn('Fail to fetch message "%s": %s"' % (uid,repr(e)))
                os.close(tmpfh)
                os.remove(tmppath)
                continue
            os.close(tmpfh)
            emlpath=re.sub('\.tmp$','.eml',tmppath)
            os.link(tmppath,emlpath)
            os.remove(tmppath)
            logging.debug('Message %s from %s of %s stored in %s' % (uid,folder,self.imapuser,emlpath))


    def logout(self):
        self.connection.close()
        self.connection.logout()



class IMAPError(Exception):
    def __init__(self,value):
        self.value=value

    def __str__(self):
        return repr(self.value)



class Status:
    """Stores UIDVALIDITY and UIDNEXT of the account"""
    def __init__(self,statusdata):
        if statusdata != '':
            self.status=json.loads(statusdata)
        else:
            self.status={}

    def checkValidity(self,mailbox,uidvalidity):
        if not mailbox in self.status:
            self.status[mailbox]=[-1,1]
        if self.status[mailbox][0] == uidvalidity:
            return True
        else:
            return False

    def newMails(self,mailbox,uidnext):
        if self.status[mailbox][1] == uidnext:
            return False
        else:
            return True


    def getNext(self,mailbox):
        return self.status[mailbox][1]

    def getJSON(self):
        return json.dumps(self.status)


    def getStatus(self,mailbox):
        """Returns list [UIDVALIDTY,UIDNEXT] of the mailfolder"""
        return self.status[mailbox]

    def setNext(self,mailbox,uidnext):
        self.status[mailbox][1]=uidnext


    def setValidity(self,mailbox,uidvalidity):
        self.status[mailbox][0]=uidvalidity


class EMail:
    def __init__(self,maildata):
        self.maildata=maildata


class DBaccess:
    def __init__(self,config):
        self.conn=''
        dbtype=config.get('DEFAULT','dbtype')
        if dbtype=='sqlite3':
            import sqlite3
            dbfile=config.get('DEFAULT','userdb')
            try:
                self.conn=sqlite3.connect(dbfile)
            except Exception, e:
                sys.exit('Cannot connect database "%s": %s' % (dbfile,e))
        elif dbtype=='mysql':
            import MySQLdb
            dbhost=config.get('DEFAULT','dbhost')
            dbuser=config.get('DEFAULT','dbuser')
            dbpass=config.get('DEFAULT','dbpass')
            dbname=config.get('DEFAULT','dbname')
            try:
                self.conn=MySQLdb.connect(host=dbhost,user=dbuser,passwd=dbpass,db=dbname)
            except Exception, e:
                sys.exit('Cannot connect database "%s": %s' % (dbname,e[1]))


class ImportLock:
    def __init__(self,inboxdir,imapuser):
        self.initialized = False
        self.lockfile = inboxdir+'/.'+imapuser+'.fetching'
        self.fp = open(self.lockfile, 'w')
        self.fp.write(str(os.getpid()))
        self.fp.write("\n")
        self.fp.flush()
        fcntl.lockf(self.fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
        self.initialized = True
        

    def unlock(self):
        if not self.initialized:
            return
        fcntl.lockf(self.fp, fcntl.LOCK_UN)
        # os.close(self.fp)
        if os.path.isfile(self.lockfile):
            os.unlink(self.lockfile)


class NoUserError(Exception):
    def __init__(self,id):
        self.id=id


################################################################################

########################################
# read RSA key to decrypt user passwords
def readKey(config):
    # decrypt key
    keyfile=config.get('DEFAULT','keyfile')
    if keyfile=='':
        logging.debug('Read key from stdin')
        keystring=sys.stdin.readlines()
        if not keystring:
            sys.exit('No key from STDIN')
        key=RSA.importKey(keystring)
    else:
        try:
            key=RSA.importKey(open(keyfile).read())
        except ValueError, e:
            logging.debug("Need passphrase to decrypt key.")
            keypass=config.get('DEFAULT','keypass')
            if keypass=='':
                # read password from stdin if keyfile configured
                if sys.stdin.isatty():
                    keypass=getpass.getpass('Key password: ')
                else:
                    keypass=sys.stdin.readline().rstrip()
            try:
                key=RSA.importKey(open(keyfile).read(),keypass)
            except ValueError, e:
                sys.exit('Cannot decrypt %s: %s' % (keyfile,e))
            except Exception, e:
                sys.exit('Cannot read %s: %s' % (keyfile,e))
    cipher=PKCS1_v1_5.new(key)
    return cipher


### MAIN ###
def print_usage(errormsg=''):
    if errormsg != '':
        print errormsg
    print "Usage: "+sys.argv[0]+" -c <config> [-f <logfile>] [-L <WARNING|INFO|DEBUG>]"
    print "  -c <configfile>            path to config file"
    print "  -f <logfile>               path to logfile"
    print "  -L <WARNING|INFO|DEBUG>    loglevel (default ERROR)"

if __name__ == "__main__":
    logarg='ERROR'
    logfile=''
    lockfile=''
    opts, args = getopt.getopt(sys.argv[1:], "hvc:f:L:", ["help", "version", "config=", "logfile=", "loglevel="])
    for o, v in opts:
        if o in ('-c', '--config'):
            configfile=v
        if o in ('-f', '--logfile'):
            logfile=v
        if o in ('-L', '--loglevel'):
            logarg=v
        if o in ('-h', '--help'):
            print_usage()
            sys.exit(0)
        if o in ('-v', '--version'):
            print "%s" % versionstring
            sys.exit(0)
    try:
        if not os.path.isfile(configfile):
            print_usage("Configfile "+configfile+" does not exist.")
            sys.exit(1)
    except NameError, e:
        print_usage("Configfile not given.")
        sys.exit(1)
    loglevel=getattr(logging, logarg.upper(), None)
    if not isinstance(loglevel, int):
        sys.exit('Invalid log level: %s' % loglevel)

    accountConfig=accountConfig(configfile)
    if logfile:
        logging.basicConfig(format='%(asctime)s %(message)s', filename=logfile, level=loglevel)
    else:
        logging.basicConfig(format='%(asctime)s %(message)s', level=loglevel)

    ssl=accountConfig.getboolean('DEFAULT','ssl')
    skipfolders=accountConfig.get('DEFAULT','skipfolders')
    skiplist=re.split(';\s+|;\s?',skipfolders)
    logging.info("Start email import.")

    cipher=readKey(accountConfig)

    for account in accountConfig.getAccountList():
        if (account.accountstatus & 1) == 0:
            logging.info("User %s disabled." % account.id)
            continue
        imapuser=account.getImapuser()
        imaphost=account.getImaphost()
        imapport=account.getImapport()
        cryptpw=account.getPassword()
        ## sentinel = Invalid message if decryption fails
        sentinel=False
        try:
            imappass=cipher.decrypt(cryptpw,sentinel)
        except ValueError as e:
            logging.warning('Skip import for %s. Password decryption failed' % imapuser)
            logging.debug('  %s' % e)
            continue

        inboxdir=accountConfig.get(imapuser,'inboxdir')

        try:
            lock=ImportLock(inboxdir,imapuser)
        except IOError as e:
            logging.warning('Skip import for %s: %s' % (imapuser,e))
            continue

        logging.info("Import %s from %s", imapuser,imaphost)
        # set X-REAL-RCPTTO header
        if account.isEnvelopeTo():
            envelopeTo=account.getImapuser()
        else:
            envelopeTo=False

        try:
            imapconn=IMAPConn(imaphost,imapport,imapuser,ssl)
            imapconn.login(imappass)
        except Exception, e:
            logging.error('"%s" cannot login: %s',imapuser,e)
            continue

        for folder in imapconn.listFolder():
            if folder in skiplist:
                logging.debug('Skip mailbox %s/%s' % (imapuser,folder))
                continue
            logging.debug('Load mailbox %s/%s' % (imapuser,folder))
            try:
                # status of current connection
                uidvalidity,uidnext=imapconn.getStatus(folder)
            except Exception, e:
                logging.error('Status error on: %s/%s:%s',imapuser,folder,e)
                try:
                    account.login()
                except Exception, e:
                    logging.error('"%s" cannot re-login: %s',imapuser,e)
                    continue
            try:
                fetchfrom=account.status.getNext(folder)
            except KeyError:
                fetchfrom=1
            if not account.status.checkValidity(folder,uidvalidity):
                account.status.setValidity(folder,uidvalidity)
                fetchfrom=1
            if account.status.newMails(folder,uidnext):
                # new mails in folder
                logging.debug('Load %s to %s from folder %s/%s',fetchfrom,uidnext,imapuser,folder)
                try:
                    imapconn.fetchMails(inboxdir,folder,fetchfrom,envelopeTo)
                    account.status.setNext(folder,uidnext)
                except Exception, e:
                    logging.error('Fail to fetch messages from %s/%s: %s',imapuser,folder,e)
                except IMAPError, e:
                    logging.error('Cannot access folder %s/%s: %s',imapuser,folder,e)
            else:
                logging.info('No new emails in folder %s/%s',imapuser,folder)
        account.save(accountConfig)
        lock.unlock()
    logging.info("Email import done.")

