#!/usr/bin/env python3
#
# Copyright (C) 2022 Wildfire Games.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
# following conditions are met:
#
#   Redistributions of source code must retain the above copyright notice, this list of conditions and the following
#   disclaimer.
#   Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following
#   disclaimer in the documentation and/or other materials provided with the distribution.
#   The name of the author may not be used to endorse or promote products derived from this software without specific
#   prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import codecs, re, os, sys
import json as jsonParser

from tokenize import generate_tokens, COMMENT, NAME, OP, STRING
from textwrap import dedent

def pathmatch(mask, path):
    """ Matches paths to a mask, where the mask supports * and **.

    Paths use / as the separator
    * matches a sequence of characters without /.
    ** matches a sequence of characters without / followed by a / and
    sequence of characters without /
    :return: true iff path matches the mask, false otherwise
    """
    s = re.split(r"([*][*]?)", mask)
    p = ""
    for i in range(len(s)):
        if i % 2 != 0:
            p = p + "[^/]+"
            if len(s[i]) == 2:
                p = p + "(/[^/]+)*"
        else:
            p = p + re.escape(s[i])
    p = p + "$"
    return re.match(p, path) != None


class Extractor(object):

    def __init__(self, directoryPath, filemasks, options):

        self.directoryPath = directoryPath
        self.options = options

        if isinstance(filemasks, dict):
            self.includeMasks = filemasks["includeMasks"]
            self.excludeMasks = filemasks["excludeMasks"]
        else:
            self.includeMasks = filemasks
            self.excludeMasks = []


    def run(self):
        """ Extracts messages.

        :return:    An iterator over ``(message, plural, context, (location, pos), comment)`` tuples.
        :rtype:     ``iterator``
        """
        directoryAbsolutePath = os.path.abspath(self.directoryPath)
        for root, folders, filenames in os.walk(directoryAbsolutePath):
            for subdir in folders:
                if subdir.startswith('.') or subdir.startswith('_'):
                    folders.remove(subdir)
            folders.sort()
            filenames.sort()
            for filename in filenames:
                filename = os.path.relpath(os.path.join(root, filename), self.directoryPath).replace(os.sep, '/')
                for filemask in self.excludeMasks:
                    if pathmatch(filemask, filename):
                        break
                else:
                    for filemask in self.includeMasks:
                        if pathmatch(filemask, filename):
                            filepath = os.path.join(directoryAbsolutePath, filename)
                            for message, plural, context, breadcrumb, position, comments in self.extractFromFile(filepath):
                                # Replace spaces in filenames by non-breaking spaces so that word
                                # wrapping in po files does not split up our paths
                                yield message, plural, context, (filename.replace(' ', u"\xa0") + (":"+breadcrumb if breadcrumb else ""), position), comments


    def extractFromFile(self, filepath):
        """ Extracts messages from a specific file.

        :return:    An iterator over ``(message, plural, context, breadcrumb, position, comments)`` tuples.
        :rtype:     ``iterator``
        """
        pass



class javascript(Extractor):
    """ Extract messages from JavaScript source code.
    """

    empty_msgid_warning = ( '%s: warning: Empty msgid.  It is reserved by GNU gettext: gettext("") '
                            'returns the header entry with meta information, not the empty string.' )

    def extractJavascriptFromFile(self, fileObject):

        from extractors.jslexer import tokenize, unquote_string
        funcname = message_lineno = None
        messages = []
        last_argument = None
        translator_comments = []
        concatenate_next = False
        last_token = None
        call_stack = -1
        comment_tags = self.options.get('commentTags', [])
        keywords = self.options.get('keywords', {}).keys()

        for token in tokenize(fileObject.read()):
            if token.type == 'operator' and \
                (token.value == '(' or (call_stack != -1 and \
                (token.value == '[' or token.value == '{'))):
                if funcname:
                    message_lineno = token.lineno
                    call_stack += 1

            elif call_stack == -1 and token.type == 'linecomment':
                value = token.value[2:].strip()
                if translator_comments and \
                translator_comments[-1][0] == token.lineno - 1:
                    translator_comments.append((token.lineno, value))
                    continue

                for comment_tag in comment_tags:
                    if value.startswith(comment_tag):
                        translator_comments.append((token.lineno, value.strip()))
                        break

            elif token.type == 'multilinecomment':
                # only one multi-line comment may preceed a translation
                translator_comments = []
                value = token.value[2:-2].strip()
                for comment_tag in comment_tags:
                    if value.startswith(comment_tag):
                        lines = value.splitlines()
                        if lines:
                            lines[0] = lines[0].strip()
                            lines[1:] = dedent('\n'.join(lines[1:])).splitlines()
                            for offset, line in enumerate(lines):
                                translator_comments.append((token.lineno + offset,
                                                            line))
                        break

            elif funcname and call_stack == 0:
                if token.type == 'operator' and token.value == ')':
                    if last_argument is not None:
                        messages.append(last_argument)
                    if len(messages) > 1:
                        messages = tuple(messages)
                    elif messages:
                        messages = messages[0]
                    else:
                        messages = None

                    # Comments don't apply unless they immediately precede the
                    # message
                    if translator_comments and \
                    translator_comments[-1][0] < message_lineno - 1:
                        translator_comments = []

                    if messages is not None:
                        yield (message_lineno, funcname, messages,
                            [comment[1] for comment in translator_comments])

                    funcname = message_lineno = last_argument = None
                    concatenate_next = False
                    translator_comments = []
                    messages = []
                    call_stack = -1

                elif token.type == 'string':
                    new_value = unquote_string(token.value)
                    if concatenate_next:
                        last_argument = (last_argument or '') + new_value
                        concatenate_next = False
                    else:
                        last_argument = new_value

                elif token.type == 'operator':
                    if token.value == ',':
                        if last_argument is not None:
                            messages.append(last_argument)
                            last_argument = None
                        else:
                            messages.append(None)
                        concatenate_next = False
                    elif token.value == '+':
                        concatenate_next = True

            elif call_stack > 0 and token.type == 'operator' and \
                (token.value == ')' or token.value == ']' or token.value == '}'):
                call_stack -= 1

            elif funcname and call_stack == -1:
                funcname = None

            elif call_stack == -1 and token.type == 'name' and \
                token.value in keywords and \
                (last_token is None or last_token.type != 'name' or
                last_token.value != 'function'):
                funcname = token.value

            last_token = token


    def extractFromFile(self, filepath):

        with codecs.open(filepath, 'r', encoding='utf-8-sig') as fileObject:
            for lineno, funcname, messages, comments in self.extractJavascriptFromFile(fileObject):
                if funcname:
                    spec = self.options.get('keywords', {})[funcname] or (1,)
                else:
                    spec = (1,)
                if not isinstance(messages, (list, tuple)):
                    messages = [messages]
                if not messages:
                    continue

                # Validate the messages against the keyword's specification
                context = None
                msgs = []
                invalid = False
                # last_index is 1 based like the keyword spec
                last_index = len(messages)
                for index in spec:
                    if isinstance(index, (list, tuple)):
                        context = messages[index[0] - 1]
                        continue
                    if last_index < index:
                        # Not enough arguments
                        invalid = True
                        break
                    message = messages[index - 1]
                    if message is None:
                        invalid = True
                        break
                    msgs.append(message)
                if invalid:
                    continue

                # keyword spec indexes are 1 based, therefore '-1'
                if isinstance(spec[0], (tuple, list)):
                    # context-aware *gettext method
                    first_msg_index = spec[1] - 1
                else:
                    first_msg_index = spec[0] - 1
                if not messages[first_msg_index]:
                    # An empty string msgid isn't valid, emit a warning
                    where = '%s:%i' % (hasattr(fileObject, 'name') and \
                                        fileObject.name or '(unknown)', lineno)
                    print(self.empty_msgid_warning % where, file=sys.stderr)
                    continue

                messages = tuple(msgs)
                message = messages[0]
                plural = None
                if len(messages) == 2:
                    plural = messages[1]

                yield message, plural, context, None, lineno, comments



class cpp(javascript):
    """ Extract messages from C++ source code.
    """
    pass



class txt(Extractor):
    """ Extract messages from plain text files.
    """

    def extractFromFile(self, filepath):
        with codecs.open(filepath, "r", encoding='utf-8-sig') as fileObject:
            lineCount = 0
            for line in [line.strip("\n\r") for line in fileObject.readlines()]:
                lineCount += 1
                if line:
                    yield line, None, None, None, lineCount, []



class json(Extractor):
    """ Extract messages from JSON files.
    """

    def __init__(self, directoryPath=None, filemasks=[], options={}):
        super(json, self).__init__(directoryPath, filemasks, options)
        self.breadcrumbs = []
        self.keywords = self.options.get("keywords", {})
        self.context = self.options.get("context", None)
        self.comments = self.options.get("comments", [])

    def setOptions(self, options):
        self.options = options
        self.keywords = self.options.get("keywords", {})
        self.context = self.options.get("context", None)
        self.comments = self.options.get("comments", [])

    @staticmethod
    def formatBreadcrumbs(breadcrumbs):
        firstPiece = breadcrumbs[0]
        if isinstance(firstPiece, int): outputString = "[" + str(firstPiece) + "]"
        else: outputString = firstPiece
        for piece in breadcrumbs[1:]:
            if isinstance(piece, int): outputString += "[" + str(piece) + "]"
            else: outputString += "." + piece
        return outputString

    def extractFromFile(self, filepath):
        with codecs.open(filepath, "r", 'utf-8') as fileObject:
            for message, context, breadcrumbs in self.extractFromString(fileObject.read()):
                yield message, None, context, self.formatBreadcrumbs(breadcrumbs), None, self.comments

    def extractFromString(self, string):
        self.breadcrumbs = []
        jsonDocument = jsonParser.loads(string)
        if isinstance(jsonDocument, list):
            for message, context, breadcrumbs in self.parseList(jsonDocument):
                if message: # Skip empty strings.
                    yield message, context, breadcrumbs
        elif isinstance(jsonDocument, dict):
            for message, context, breadcrumbs in self.parseDictionary(jsonDocument):
                if message: # Skip empty strings.
                    yield message, context, breadcrumbs
        else:
            raise Exception("Unexpected JSON document parent structure (not a list or a dictionary). You must extend the JSON extractor to support it.")

    def parseList(self, itemsList):
        index = 0
        for listItem in itemsList:
            self.breadcrumbs.append(index)
            if isinstance(listItem, list):
                for message, context, breadcrumbs in self.parseList(listItem):
                    yield message, context, breadcrumbs
            elif isinstance(listItem, dict):
                for message, context, breadcrumbs in self.parseDictionary(listItem):
                    yield message, context, breadcrumbs
            del self.breadcrumbs[-1]
            index += 1

    def parseDictionary(self, dictionary):
        for keyword in dictionary:
            self.breadcrumbs.append(keyword)
            if keyword in self.keywords:
                if isinstance(dictionary[keyword], str):
                    yield self.extractString(dictionary[keyword], keyword)
                elif isinstance(dictionary[keyword], list):
                    for message, context, breadcrumbs in self.extractList(dictionary[keyword], keyword):
                        yield message, context, breadcrumbs
                elif isinstance(dictionary[keyword], dict):
                    extract = None
                    if "extractFromInnerKeys" in self.keywords[keyword] and self.keywords[keyword]["extractFromInnerKeys"]:
                        for message, context, breadcrumbs in self.extractDictionaryInnerKeys(dictionary[keyword], keyword):
                            yield message, context, breadcrumbs
                    else:
                        extract = self.extractDictionary(dictionary[keyword], keyword)
                        if extract:
                            yield extract
            elif isinstance(dictionary[keyword], list):
                for message, context, breadcrumbs in self.parseList(dictionary[keyword]):
                    yield message, context, breadcrumbs
            elif isinstance(dictionary[keyword], dict):
                for message, context, breadcrumbs in self.parseDictionary(dictionary[keyword]):
                    yield message, context, breadcrumbs
            del self.breadcrumbs[-1]

    def extractString(self, string, keyword):
        context = None
        if "tagAsContext" in self.keywords[keyword]:
            context = keyword
        elif "customContext" in self.keywords[keyword]:
            context = self.keywords[keyword]["customContext"]
        else:
            context = self.context
        return string, context, self.breadcrumbs

    def extractList(self, itemsList, keyword):
        index = 0
        for listItem in itemsList:
            self.breadcrumbs.append(index)
            if isinstance(listItem, str):
                yield self.extractString(listItem, keyword)
            elif isinstance(listItem, dict):
                extract = self.extractDictionary(dictionary[keyword], keyword)
                if extract:
                    yield extract
            del self.breadcrumbs[-1]
            index += 1

    def extractDictionary(self, dictionary, keyword):
        message = dictionary.get("_string", None)
        self.breadcrumbs.append("_string")
        if message and isinstance(message, str):
            context = None
            if "context" in dictionary:
                context = str(dictionary["context"])
            elif "tagAsContext" in self.keywords[keyword]:
                context = keyword
            elif "customContext" in self.keywords[keyword]:
                context = self.keywords[keyword]["customContext"]
            else:
                context = self.context
            return message, context, self.breadcrumbs
        del self.breadcrumbs[-1]
        return None

    def extractDictionaryInnerKeys(self, dictionary, keyword):
        for innerKeyword in dictionary:
            self.breadcrumbs.append(innerKeyword)
            if isinstance(dictionary[innerKeyword], str):
                yield self.extractString(dictionary[innerKeyword], keyword)
            elif isinstance(dictionary[innerKeyword], list):
                for message, context, breadcrumbs in self.extractList(dictionary[innerKeyword], keyword):
                    yield message, context, breadcrumbs
            elif isinstance(dictionary[innerKeyword], dict):
                extract = self.extractDictionary(dictionary[innerKeyword], keyword)
                if extract:
                    yield extract
            del self.breadcrumbs[-1]



class xml(Extractor):
    """ Extract messages from XML files.
    """

    def __init__(self, directoryPath, filemasks, options):
        super(xml, self).__init__(directoryPath, filemasks, options)
        self.keywords = self.options.get("keywords", {})
        self.jsonExtractor = None

    def getJsonExtractor(self):
        if not self.jsonExtractor:
            self.jsonExtractor = json()
        return self.jsonExtractor

    def extractFromFile(self, filepath):
        from lxml import etree
        with codecs.open(filepath, "r", encoding='utf-8-sig') as fileObject:
            xmlDocument = etree.parse(fileObject)
            for keyword in self.keywords:
                for element in xmlDocument.iter(keyword):
                    position = element.sourceline
                    if element.text is not None:
                        comments = []
                        if "extractJson" in self.keywords[keyword]:
                            jsonExtractor = self.getJsonExtractor()
                            jsonExtractor.setOptions(self.keywords[keyword]["extractJson"])
                            for message, context, breadcrumbs in jsonExtractor.extractFromString(element.text):
                                yield message, None, context, json.formatBreadcrumbs(breadcrumbs), position, comments
                        else:
                            context = None
                            breadcrumb = None
                            if "locationAttributes" in self.keywords[keyword]:
                                attributes = [element.get(attribute) for attribute in self.keywords[keyword]["locationAttributes"] if attribute in element.attrib]
                                breadcrumb = "({attributes})".format(attributes=", ".join(attributes))
                            if "context" in element.attrib:
                                context = str(element.get("context"))
                            elif "tagAsContext" in self.keywords[keyword]:
                                context = keyword
                            elif "customContext" in self.keywords[keyword]:
                                context = self.keywords[keyword]["customContext"]
                            if "comment" in element.attrib:
                                comment = element.get("comment")
                                comment = u" ".join(comment.split()) # Remove tabs, line breaks and unecessary spaces.
                                comments.append(comment)
                            if "splitOnWhitespace" in self.keywords[keyword]:
                                for splitText in element.text.split():
                                    # split on whitespace is used for token lists, there, a leading '-' means the token has to be removed, so it's not to be processed here either
                                    if splitText[0] != "-":
                                        yield str(splitText), None, context, breadcrumb, position, comments
                            else:
                                yield str(element.text), None, context, breadcrumb, position, comments


# Hack from http://stackoverflow.com/a/2819788
class FakeSectionHeader(object):

    def __init__(self, fp):
        self.fp = fp
        self.sechead = '[root]\n'

    def readline(self):
        if self.sechead:
            try: return self.sechead
            finally: self.sechead = None
        else: return self.fp.readline()


class ini(Extractor):
    """ Extract messages from INI files.
    """

    def __init__(self, directoryPath, filemasks, options):
        super(ini, self).__init__(directoryPath, filemasks, options)
        self.keywords = self.options.get("keywords", [])

    def extractFromFile(self, filepath):
        import ConfigParser
        config = ConfigParser.RawConfigParser()
        config.readfp(FakeSectionHeader(open(filepath)))
        for keyword in self.keywords:
            message = config.get("root", keyword).strip('"').strip("'")
            context = None
            position = " ({})".format(keyword)
            comments = []
            yield message, None, context, None, position, comments
