# coding: utf8
# Debian Contributors data mining on debbugs spool directories
#
# Copyright (C) 2015  Enrico Zini <enrico@debian.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from ...submission import Identifier, Contribution
import os
import os.path
import datetime
import logging
import email.utils

log = logging.getLogger(__name__)

__all__ = ["DebbugsScanner"]


states = {
    1: 'autocheck',
    2: 'recips',
    3: 'kill-end',
    5: 'go',
    6: 'html',
    7: 'incoming-recv',
}


def read_incoming_recv(fh):
    """
    Parse a log file and generate only incoming-recv lines
    (all lines of incoming emails).

    It generates an array of lines for each email. Lines are byte() strings.
    """
    cur_lines = None
    for line in fh:
        if len(line) == 2:
            if line[0] == 7:
                cur_lines = []
            elif line[0] in states:
                if cur_lines is not None:
                    yield cur_lines
                    cur_lines = None
            else:
                if cur_lines is not None:
                    cur_lines.append(line)
        elif cur_lines is not None:
            cur_lines.append(line)

def parse_bts_received(line):
    """
    Parse the leading BTS log synthetic Received: line and return the date
    header for it, if any. If anything goes wrong, it returns None.
    """
    pos = line.rfind(b";")
    if pos == -1: return None
    line = line[pos + 2:]
    try:
        parsed = email.utils.parsedate(line.decode("utf-8", errors="replace"))
    except IndexError:
        return None
    if parsed is None: return None
    try:
        date = datetime.date(parsed[0], parsed[1], parsed[2])
    except ValueError:
        return None

    # Sanity cutoff date
    if date.year < 1995: return None

    return date

def parse_from_header(line):
    """
    Get the email address out of the From: header line
    """
    # From: field
    realname, addr = email.utils.parseaddr(line[6:].decode("utf-8", errors="replace"))
    return addr

def scan_file(fh):
    """
    Scan a .log file and generate a (datetime.date, email_addr) couple for each
    email found.

    emails that do not parse correctly are ignored.

    The email address it generates is a str() object.
    """
    for lines in read_incoming_recv(fh):
        # Parse the date from the leading synthetic Received field

        # Must start with /^Received: \(at \S+\) by \S+;/
        if not lines[0].startswith(b"Received:"):
            continue
        date = parse_bts_received(lines[0])
        if date is None: continue

        # Parse the email address from the From: field

        addr = None
        for l in lines[1:]:
            if l.startswith(b"From: "):
                # From: field
                addr = parse_from_header(l)
                break
            elif l == b"\n":
                # Stop at end of headers
                break
        if addr is None: continue

        yield date, addr

class DebbugsScanner:
    def __init__(self):
        self.data = {}

    def scan_dir(self, pathname):
        """
        Collect data from all {pathname}/??/*.log files
        """
        rootfd = os.open(pathname, os.O_RDONLY)
        for idx, dn in enumerate(sorted(os.listdir(rootfd))):
            processed = 0
            if len(dn) != 2: continue
            nnfd = os.open(dn, os.O_RDONLY, dir_fd=rootfd)
            for fn in os.listdir(nnfd):
                if not fn.endswith(".log"): continue
                fd = os.open(fn, os.O_RDONLY, dir_fd=nnfd)
                fh = os.fdopen(fd, "rb")
                for date, addr in scan_file(fh):
                    c = self.data.get(addr, None)
                    if c is None:
                        self.data[addr] = [1, Contribution("correspondant", date, date)]
                    else:
                        c[0] += 1
                        c[1].extend_by_date(date)
                    processed += 1
            log.info("%s: %d emails processed", os.path.join(pathname, dn), processed)

