###############################################################################
# Local Security Check Automation Framework
#
# Authors:
# Veerendra GG <veerendragg@secpod.com>
#
# Revision 1.0
# Date: 2009/03/23
#
# Copyright:
# Copyright (c) 2009 SecPod , http://www.secpod.org
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2
# (or any later version), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
###############################################################################

import re
import os
import sys

from common import utils


## Supported Ubuntu OSes for parsing. The value is as used in
## gather-package-list.nasl to set "ssh/login/release"
os_map = {

    'Ubuntu 4.1' : 'UBUNTU4.1',
    'Ubuntu 5.04' : 'UBUNTU5.04',
    'Ubuntu 5.10' : 'UBUNTU5.10',
    'Ubuntu 6.06 LTS' : 'UBUNTU6.06 LTS',
    'Ubuntu 6.06' : 'UBUNTU6.06 LTS',
    'Ubuntu 6.10' : 'UBUNTU6.10',
    'Ubuntu 7.04' : 'UBUNTU7.04',
    'Ubuntu 7.10' : 'UBUNTU7.10',
    'Ubuntu 8.04' : 'UBUNTU8.04 LTS',
    'Ubuntu 8.04 LTS' : 'UBUNTU8.04 LTS',
    'Ubuntu 8.10' : 'UBUNTU8.10',
    'Ubuntu 9.04' : 'UBUNTU9.04',
    'Ubuntu 9.10' : 'UBUNTU9.10', 
}

## Strips these from strip list
strip_val = ['_i386.deb', '_all.deb']

append_url = 'https://lists.ubuntu.com/archives/ubuntu-security-announce/'

## These are not advisories
skip_list = ['']


class Parser:
    """
    Ubuntu security advisory parser, parse and populate the global variables
    """

    ## Global parse structure, initializing
    AdvID = ''
    Description = ''
    Packages = {}
    CVEs = ''
    Name = ''
    Summary = ''
    Platforms = ''
    Product = []
    Html_content = ''
    XREF = []
    FileName = ''


    def _getYearLinks(self, link, year, debug=0):
        """
        Gets the advisory links for the given year
        """
        year_links = []

        data = utils.getHTMLCon(link)
        links = re.findall('href="(' + str(year) + '.*thread.html)', data)
        if links:
            for i in links:
                year_links.append(append_url + i)

            if debug:
                if year_links:
                    print "\nAdvisory Links for (%s) year" %(year)
                    for i in year_links:
                        print i

            return year_links

        return ''


    def _getEachAdvLink(self, link, debug=0):
        """
        Get security advisory links.
        """
        month_links = []

        data = utils.getHTMLCon(link)
        links = re.findall('HREF="(.*.html)', data)
        year_month = os.path.basename(os.path.split(link)[0])

        if links:
            for i in links:
                month_links.append(append_url + year_month + '/' + i)

            if debug:
                print "\nUbuntu Advisories for (%s) year" %(year_month)
                print "Total (%s) Ubuntu Advisories : " %(len(month_links))

            return month_links

        return []


    def fetchHTML(self, year, debug=0):
        """
        Retrive Ubuntu Advisories locally
        """

        try:
            all_adv_links = []

            year_links = self._getYearLinks(self.main_url, year, debug)

            if not year_links:
                print "ERROR: Din't find mentioned (%s) year in Ubuntu "+ \
                                             "Advisories..." %(year)
                print "Exiting ..."
                sys.exit(0)

            for link in year_links:
                month_links = self._getEachAdvLink(link, debug)
                if not month_links:
                    if debug:
                        print "ERROR: No Fedora Security Advisories for:\n", \
                                                                          link
                    continue

                all_adv_links.extend(month_links)

            if not all_adv_links:
                print "ERROR: Din't find any Ubuntu Security Advisories...", \
                                                                          year
                print "Exiting ..."
                sys.exit(0)

            all_adv_links = utils.removeDups(all_adv_links)

            for adv_url in all_adv_links:

                base_name = adv_url.split('/')[-1]
                month_year = adv_url.split('/')[-2]
                file_name = self.html_cache + month_year + '_' + base_name

                if not os.path.isfile(file_name):
                    if debug:
                        print "\nFetching Ubuntu Security Advisory..." + \
                                                 os.path.basename(adv_url)
                    try:
                        utils.fetchFiles(adv_url, file_name, debug)
                    except Exception, msg:
                        print 'ERROR: Error fething the url %s' % msg

        except Exception, msg:
            print "Exception in : ubuntu -> Parser(Class) -> fetchHTML method()"
            sys.exit(msg)


    def _findAll(self, regex):
        """
        Returns Matched data
        """
        return regex.findall(self.Html_content)


    def getCVE(self, debug=0):
        """
        Returns CVE list
        """
        if debug:
            print "\nGetting CVE List..."

        cve_regex = re.compile('CVE-[0-9]+-[0-9]+')
        can_regex = re.compile('CAN-[0-9]+-[0-9]+')

        cve_list = self._findAll(cve_regex)
        cve_list.extend(self._findAll(can_regex))

        cve_list = utils.removeDups(cve_list)

        if cve_list:
            cve_list = '", "'.join(cve_list)
        else:
            cve_list = ''

        if debug:
            print "CVE List : ", cve_list

        return cve_list


    def getAdvID(self, debug=0):
        """
        Returns Ubuntu Security Advisory ID
        """

        if debug:
            print "\nGetting Advisory ID..."

        adv_id_regex =  re.compile('Ubuntu Security Notice\s?(USN.*\d) .*')
        adv_id = self._findAll(adv_id_regex)

        if not adv_id:
            adv_id =  re.findall("<h1>.*\[(USN.*)\].*", self.Html_content, \
                                                              re.IGNORECASE)
        if not adv_id:
            return ''

        if debug:
            print "Advisory ID : ", adv_id

        adv_id = adv_id[0].strip()
        adv_id = adv_id.replace(' ','-')

        return adv_id


    def getAffectedPackage(self, debug=0):
        """
        Returns Affected Packages/RPM's
        """

        if debug:
            print "\nGetting Affected Packages/RPM List..."

        pkg_regex =  re.compile("Ubuntu Security Notice USN.*\\n(.*)", \
                                                          re.IGNORECASE)
        pkg = self._findAll(pkg_regex)

        if pkg:
            pkg = pkg[0].strip()
        else:
            pkg = ''

        ## Some time name is coming like postgresql-8.1, postgresql-8.3,
        ## postgresql-8.4 vulnerabilities. Below code is to avoid such a 
        ## situaction. EX : USN-876-1 or 2010-January/001018.html

        if len(pkg) > 40 and ', ' in pkg:
            pkg_regex =  re.compile("<H1>\[USN-\d\d\d?\d?\d?-\d\d?\](.*)</H1>",\
                                                          re.IGNORECASE)
        tmp_pkg = self._findAll(pkg_regex)

        if tmp_pkg:
            pkg = tmp_pkg[0].strip()

        if debug:
            print "Affected Packages/RPMS : ", pkg

        return pkg


    def getDescription(self, debug=0):
        """
        Returns Vulnerability Description
        """
        desc_1 = ''
        description = ''

        if debug:
            print "\nGetting Vulnerability Description..."

        desc_regex =  re.compile("(?s)Details follow:"+ \
                      "(.*Updated packages for Ubuntu)", re.IGNORECASE)
        desc = self._findAll(desc_regex)

        if not desc:
            desc_regex =  re.compile("(?s)Updating your system:"+\
                          "(.*Updated packages for Ubuntu)", re.IGNORECASE)
            desc = self._findAll(desc_regex)

        if not desc:
            return ''

        if desc:
            for i in desc[0].split('\n'):
                if "Updated packages for Ubuntu" in i:
                    break
                else:
                    desc_1 += '\n' + i

            ## Formatting the description
            for i in desc_1.split('\n'):
                if not i == '\n':
                    i = i.strip()
                description += '  ' + i + '\n'
                    
            description = description.replace('"',"'")
            description = description.strip()
        else:
            description = ''

        return description


    def getAffectedProduct(self, debug=0):
        """
        Returns Affected Product/Platform
        """
        prd_list = []

        ## Get Affected Product/Platform
        products = re.findall('(?s)security issue affects.*Ubuntu '+ \
                          'releases:(.*)This advisory also applies', \
                                                    self.Html_content)
        if not products:
            products = re.findall('(?s)security issue affects.*Ubuntu' + \
                      ' releases:(.*)The following packages are affected',\
                                                      self.Html_content)
        if not products:
            products = re.findall('(?s)The following Ubuntu releases are' + \
                                ' affected:(.*)This advisory also applies', \
                                                           self.Html_content)
        if products:
            products = products[0].strip().split('\n')

            if products:
                for i in products:
                    prd_list.append(i.split('(')[0].strip())

        if debug:
            print "\nAffected Product is/are : (%s)" %(products)

        ## Don't include Product/Platform, If not in "os_map" Dict
        ref_list = []
        self.total_prod_list = prd_list
        for prod in prd_list:
            if os_map.has_key(prod):
                ref_list.append(prod)
            elif debug and prod:
                  print "\nUPDATE: Not Generating Code for (%s) OS" %(prod)
                  print "If Needed to generate code, then "+ \
                        "add into dict variable os_map in parser"

        if ref_list and debug:
            print "\nGenerating Code for (%s) Products " %(ref_list)

        return ref_list


    def getRPM(self, prod_list,  debug=0):
        """
        Returns OS Package Dictionary
        """

        if debug:
            print "\nGetting RPM List..."

        all_debs = []
        ubuntu_list = []
        os_pkg_dict = {}


        for prod in prod_list:
            ubuntu_list = []
            prod = prod.split('(')
            prod = prod[0].strip()

            start_index = self.Html_content.find("Updated packages for " +prod)
            start_index += 10
            data = self.Html_content[start_index:]
            end_index = data.find("Updated packages for ")
            pkg_data = data[: end_index]

            for pattern in strip_val:
                ubuntu = re.findall('>http.*\/(.*'+ pattern + ')<', pkg_data)
                if ubuntu:
                    ubuntu_list.extend(ubuntu)

            if not ubuntu_list:
                if debug:
                    ubuntu = re.findall(">http.*\/(.*i386\.deb)<", pkg_data)
                    if ubuntu:
                          print "ERROR: Found Ubuntu pkg other then, %s :: %s"\
                                    % (', '.join(strip_val), ubuntu)
                    else:
                        print "ERROR: Ubuntu package not found for product : ",\
                                                                       prod_list

                return os_pkg_dict

            all_debs = utils.stripIt(ubuntu_list, strip_val)
            all_debs = utils.removeDups(all_debs)

            if os_map.has_key(prod):
                os_pkg_dict[os_map[prod]] = all_debs

        if debug:
            print "OS PKG Dict : ", os_pkg_dict

        return os_pkg_dict


    def formatReference(self, main_url, file_name):
       """
       Constructs a reference for advisory
       """
       if not main_url.endswith('/'):
           main_url = main_url + '/'

       reference = main_url + '/'.join(file_name.split('_'))

       return reference


    def parser(self, html_content, debug=0):
        """
        Main parser function, builds the parser object
        by invoking parse functions
        """

        try:
            if debug:
                print "Ubuntu Parser Initiated..."

            self.Html_content = html_content.replace('\r\n', '\n')

            self.CVEs = self.getCVE(debug)

            self.Platforms = self.getAffectedProduct(debug)
            if not self.Platforms or self.Platforms == []:
                if debug:
                    print "ERROR: Required Products not found..."
                return False

            self.Packages = self.getRPM(self.Platforms, debug)
            if not self.Packages or self.Packages == '':
                if debug:
                    print "ERROR: Required Packages not found..."
                return False

            self.Description = self.getDescription(debug)
            if not self.Description or self.Description == '':
                if debug:
                    print "ERROR: Description not found..."
                return False

            self.AdvID = self.getAdvID(debug)
            if not self.AdvID or self.AdvID == '':
                if debug:
                    print "ERROR: Advisory ID not found..."
                return False

            self.Product = self.getAffectedPackage(debug)
            if not self.Product or self.Product == '':
                if debug:
                    print "ERROR: Required Products not found..."
                return False

            self.Platforms = " ,\n  ".join(self.Platforms)           
 
            self.Summary = self.Product

            self.Name = self.Product + " " + self.AdvID

            self.Impact = '  '

            ## Construct File Name
            self.FileName = "_".join(['ubuntu', self.AdvID.replace('-','_')])

            ## Set XREF
            adv_id = self.AdvID.split('-')
            if len(adv_id) == 3:
                (name, value1, value2) = adv_id
                self.XREF = [name, '-'.join([value1, value2])]
            elif len(adv_id) == 2:
                (name, value1) = adv_id
                self.XREF = [name, value1]
            else:
                if debug:
                    print "ERROR : Check the Advisory ID : ", self.AdvID
                return False

            if debug:
                print "\nAll mandatory attributes are parsed: ", self.AdvID

            return True

        except Exception, msg:
            print 'Exception in Parser ubuntu -> Parser -> parser() Method '
            sys.exit(msg)
