###############################################################################
# Local Security Check Automation Framework
#
# Authors:
# Veerendra GG <veerendragg@secpod.com>
#
# Revision 1.0
# Date: 2009/04/30
#
# Copyright:
# Copyright (c) 2009 SecPod , http://www.secpod.org
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2
# (or any later version), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
###############################################################################

import re
import os
import sys
import string
import urllib
import httplib2 

from common import utils


## Supported HP-UX OSes for parsing. The value is as used in
## gather-package-list.nasl to set "ssh/login/release"
os_map = {

    'HP-UX B.10.01' : 'HPUX10.01',
    'HP-UX B.10.10' : 'HPUX10.10',
    'HP-UX B.10.20' : 'HPUX10.20',
    'HP-UX B.10.24' : 'HPUX10.24',
    'HP-UX B.10.26' : 'HPUX10.26',

    'HP-UX B.11.00' : 'HPUX11.00',
    'HP-UX B.11.04' : 'HPUX11.04',
    'HP-UX B.11.11' : 'HPUX11.11',
    'HP-UX B.11.20' : 'HPUX11.20',
    'HP-UX B.11.22' : 'HPUX11.22',
    'HP-UX B.11.23' : 'HPUX11.23',
    'HP-UX B.11.31' : 'HPUX11.31',

    'HP-UX 10.01' : 'HPUX10.01',
    'HP-UX 10.10' : 'HPUX10.10',
    'HP-UX 10.20' : 'HPUX11.20',

    'HP-UX 11.00' : 'HPUX11.00',
    'HP-UX 11.04' : 'HPUX11.04',
    'HP-UX 11.11' : 'HPUX11.11',
}

append_url = 'https://www.redhat.com/archives/hp-ux-package-announce/'

## These are not advisories
skip_list = ['']


class Parser:
    """
    HP-UX security advisory parser, parse and populate the global variables
    """

    ## Global parse structure, initializing
    AdvID = ''
    Description = ''
    Packages = {}
    CVEs = ''
    Name = ''
    Summary = ''
    Platforms = ''
    Product = []
    Html_content = ''
    XREF = []
    FileName = ''


    def _getHTMLPage(self, url, headers, body, count, max_redir_limit, \
                                                      request, debug=0):

        try:
            count = count + 1
            http = httplib2.Http()
            if request.upper() == "POST":
                response, content = http.request(url, 'POST', body, headers)
            else:
                response, content = http.request(url, 'GET', body, headers)
            return (response, content)

        except httplib2.RedirectLimit, msg:
            if count > max_redir_limit:
                if debug:
                   print "Exceeded the max redirection limit..."
                return (msg, False)
            if debug:
                print "Redirect Exception Msg : ", msg
            res, con = self._getHTMLPage(url, headers, body, count, \
                                         max_redir_limit, request, debug)
            return res, con

        except Exception, msg:
            print 'Exception Msg : ', msg
            sys.exit(0)


    def _loginHPAdvPage(self, debug=0):
        count = 0
        max_redir_limit = 2

        if debug:
            print "HP Security Advisory Login..."

        url = 'http://www11.itrc.hp.com/service/ciss/doLogin.do'
        headers = {'Host':'www11.itrc.hp.com', 'Cookie':'HP-BCO-VISITORID=1239628916337509; s_vi=[CS]v1|49E33C7000000F17-A0208B0000000E2[CE]; HP-ESC-USERID=WW130371; HP-ESC-EMAIL=tech.veerendra@gmail.com; HP_EBUS_ITRC=true; HP-ESC-S-WPA-IDX=7TZDJknf4CZrwMJ7kKc5yFzMkzh7NDRftsBykTkFJHJbDSpJNN03!-189348884!-2058958028; HP-ESC-S-EVENTS=13|P|14994349|10:J:li:WW130371&veerendragg&tech.veerendra%40gmail.com&secpod&india|11:J:po:|12:J:lo:|13:J:pu:http%3A%2F%2Fwww13.itrc.hp.com%2Fservice%2Fcki%2FsecBullArchive.do'}

        body = 'f_UserID=tech.veerendra%40gmail.com&f_Password=testlab12&loginstandardsession.x=98&loginstandardsession.y=11&f_RememberMe=false'

        response, content = self._getHTMLPage(url, headers, body, count, \
                                              max_redir_limit, "POST", debug)

        return (response, content)


    def _getMainHPAdvPage(self, debug=0):
        count = 0
        max_redir_limit = 5

        if debug:
            print "Getting HP-UX Security Advisory Main Page..."

        url = 'http://www11.itrc.hp.com/service/cki/secBullArchive.do'
        headers = {'Host':'www11.itrc.hp.com', "User-Agent":"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko/2009032608 Firefox/3.0.8", "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language":"en-us,en;q=0.5", "Accept-Encoding":"gzip,deflate", "Accept-Charset":"ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Keep-Alive":"300", "Connection":"keep-alive", "Cookie":"HP-ESC-S-WPA-IDX=7TZDJknf4CZrwMJ7kKc5yFzMkzh7NDRftsBykTkFJHJbDSpJNN03!-189348884!-2058958028; HP-ESC-S-EVENTS=3|P|26761440|2:J:li:WW130371&veerendragg&tech.veerendra@gmail.com&secpod&india|3:J:po:|1:J:pu:http://www11.itrc.hp.com/service/cki/secBullArchive.do?admit=109447626+1239604635428+28353475"}

        body = urllib.urlencode({'  ': '  '})

        response, content = self._getHTMLPage(url, headers, body, count, \
                                              max_redir_limit, "GET", debug)

        return (response, content)


    def _getYearLinks(self, content, year, debug=0):
        adv_year_list = []
        url = 'http://www11.itrc.hp.com/service/cki/docDisplay.do?docId='

        if debug:
            print "Getting Hp-UX Security Advisory Links for year ", year

        tmp = content.split('<tr>')
        print ""
        for part in tmp:
            if str(year) in part and "HPSBUX" in part:
               avd_id = re.findall('<a href=".*\?docId=(.*)">', part)
               if avd_id:
                   adv_url = url + avd_id[0]
                   adv_year_list.append(adv_url)
               elif debug:
                   print "Din't find HP-UX Security Advisory ID : ", part

        if debug and adv_year_list:
            print "Total (%s) HP-UX Advisories for (%s) year : " \
                                                   %(len(adv_year_list), year)
            print "\nHP-UX Advisory Links for (%s) year" %(year)
            for i in adv_year_list:
                print i

        return adv_year_list


    def _getEachHPAdv(self, url, file_name, debug=0):
        count = 0
        max_redir_limit = 5

        if debug:
            print "Retriving HP-UX Security Advisory...", url

        headers = {'Host':'www11.itrc.hp.com', "User-Agent":"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko/2009032608 Firefox/3.0.8", "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language":"en-us,en;q=0.5", "Accept-Encoding":"gzip,deflate", "Accept-Charset":"ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Keep-Alive":"300", "Connection":"keep-alive", "Cookie":"HP-ESC-S-WPA-IDX=7TZDJknf4CZrwMJ7kKc5yFzMkzh7NDRftsBykTkFJHJbDSpJNN03!-189348884!-2058958028; HP-ESC-S-EVENTS=3|P|26761440|2:J:li:WW130371&veerendragg&tech.veerendra@gmail.com&secpod&india|3:J:po:|1:J:pu:http://www11.itrc.hp.com/service/cki/secBullArchive.do?admit=109447626+1239604635428+28353475"}

        body = urllib.urlencode({'  ': '  '})

        response, content = self._getHTMLPage(url, headers, body, count, \
                                              max_redir_limit, "GET",debug)

        if content == False:
            print "Not able to get (%s) page" %(file_name)
            print "ERROR: Exception:", response
            return False
        elif "SECURITY BULLETIN" in content and "Document ID" in content:
            utils.writeFile(file_name, content)
            return True
        elif debug:
            print "It's not a HP-UX Security Advisory : ", url
            return False


    def fetchHTML(self, year, debug=0):
        """
        Retrive HP-UX Advisories locally
        """

        try:
            response, content = self._loginHPAdvPage(debug)
            if "Moved Temporarily" in content:
                if debug:
                    print "Successfully logged in.."
            else:
                print "ERROR: Login Failed..."
                print "Exiting ..."
                return "Exit: No Advisory Found"
#                sys.exit(0)

            response, content = self._getMainHPAdvPage(debug)
            if content and "Logout" in content:
                adv_year_links = self._getYearLinks(content, year, debug)
                adv_year_links = utils.removeDups(adv_year_links)

                if adv_year_links:
                    for link in adv_year_links:
                        tmp_name = 'hp_ux_' + link.split('=')[-1] + '.html'
                        tmp_name = tmp_name.replace('-','_')
                        file_name = self.html_cache + tmp_name

                        if debug:
                            print "\nFile Path : ", file_name

                        if os.path.isfile(file_name):
                            if debug:
                                file = os.path.basename(file_name)
                                print "(%s) is already Fetched from HP-UX " \
                                                 "Security Advisory..." %(file)
                            continue
                        else:
                            if debug:
                                print "Fetching HP-UX Security Advisory...", \
                                                    os.path.basename(tmp_name)
                            try:
                                self._getEachHPAdv(link, file_name, debug)
                            except Exception, msg:
                                print 'ERROR: Error retriving the url %s' % msg
                else:
                    print "ERROR: Din't find mentioned (%s) year in HP-UX " \
                                                 "Advisories..." %(year)
#                    print "Exiting ..."
#                    sys.exit(0)
            elif debug:
                print "Getting Redirect Error : ", response
                return "Exit: No Advisory Found"
#                sys.exit(response)

        except Exception, msg:
            print "Exception in : hp-ux -> Parser(Class) -> fetchHTML "+ \
                                                               "method()", msg
            sys.exit(msg)


    def _findAll(self, regex):
        """
        Returns Matched data
        """
        return regex.findall(self.Html_content)


    def getCVE(self, debug=0):
        """
        Returns CVE list
        """
        if debug:
            print "\nGetting CVE List..."

        cve_regex = re.compile('CVE-[0-9]+-[0-9]+')
        can_regex = re.compile('CAN-[0-9]+-[0-9]+')

        cve_list = self._findAll(cve_regex)
        cve_list.extend(self._findAll(can_regex))

        cve_list = utils.removeDups(cve_list)

        if cve_list:
            cve_list = '", "'.join(cve_list)
        else:
            cve_list = ''

        if debug and cve_list:
            print "CVE List : ", cve_list

        return cve_list


    def getAdvID(self, debug=0):
        """
        Returns HP-UX Security Advisory ID
        """

        if debug:
            print "\nGetting Advisory ID..."

        adv_id_regex =  re.compile('<title>(HPSBUX\d+)')
        adv_id = self._findAll(adv_id_regex)

        if not adv_id:
            return ''

        if debug:
            print "Advisory ID : ", adv_id

        return adv_id[0].strip()


    def getAffectedPackage(self, debug=0):
        """
        Returns Affected Packages/RPM's
        """

        if debug:
            print "\nGetting Affected Packages/RPM List..."

        pkg_regex =  re.compile("<title>HPSBUX\d+.*HP-?UX Running (.*),", \
                                                            re.IGNORECASE)
        pkg = self._findAll(pkg_regex)

        if not pkg:
            pkg = re.findall('<title>HPSBUX\d+.*HP-?UX Using (.*),', \
                                    self.Html_content, re.IGNORECASE)
        if not pkg:
            pkg = re.findall('<title>HPSBUX\d+.*HP-?UX (.*),', \
                                    self.Html_content, re.IGNORECASE)
        if not pkg:
            pkg = re.findall('<title>HPSBUX\d+.*HP-?UX(.*)', \
                                    self.Html_content, re.IGNORECASE)
            if pkg:
                pkg[0] = 'HP-UX Pkg'
        if pkg:
            pkg = pkg[0].split(',')[0]
            pkg = pkg.strip()
        else:
            pkg = ''

        if debug:
            print "Affected Packages/RPMS : ", pkg

        return pkg


    def getDescription(self, debug=0):
        """
        Returns Vulnerability Description
        """
        desc = ''

        if debug:
            print "\nGetting Vulnerability Description..."

        desc_regex =  re.compile("(?s)VULNERABILITY SUMMARY(.*)References:", \
                                                               re.IGNORECASE)
        description = self._findAll(desc_regex)

        if not description:
            return desc

        description = description[0]
        for i in description.split('\n'):
           if i.startswith('<') or ("<" in i and "style=" in i):
               continue
           i = i.strip('</p>').strip()
           desc += i
        desc = desc.replace('"'," &qt ").strip()

        if desc:
            desc = utils.formatMultiLines(desc)

        return desc


    def getImpact(self, debug=0):
        """
        Returns HP-UX Security Advisory Impact
        """
        impact = ''

        if debug:
            print "\nGetting Advisory Imapct..."

        adv_impact_regex =  re.compile('Security Impact: (.*)', re.IGNORECASE)
        adv_impact = self._findAll(adv_impact_regex)

        if adv_impact:
            adv_impact = adv_impact[0].strip().strip('</p>').strip('</b>').strip()

        if not adv_impact:
            adv_impact_regex =  re.compile('Security Impact: .*\n(.*)', \
                                                                re.IGNORECASE)
            adv_impact = self._findAll(adv_impact_regex)

        if not adv_impact:
            return ' '

        if type(adv_impact) == list:
            adv_impact = adv_impact[0]

        adv_impact = adv_impact.strip().strip('</p>').strip('</b>').strip()
        for i in adv_impact.split(','):
            i = i.strip()
            if i:
                impact = impact + "  " + i + '\n'

        if debug:
            print "Advisory Impact : \n", impact

        return impact.strip()


    def getAffectedProduct(self, debug=0):
        """
        Returns Affected Product/Platform
        """
        prod = []
        ## Get Affected Product/Platform
        prod_data = re.findall("(?s)SUPPORTED SOFTWARE VERSI(.*)\</div\>"+ \
                                          ".*BACKGROUND", self.Html_content)
        if prod_data:
            prod_data = prod_data[0]
            for line in prod_data.split('\n'):
                line = line.strip()
                if "HP-UX" in line or "B.1" in line:
                    line = line.strip('</p>')
                    line = line.strip()
                    if 'style' in line and ">" in line:
                        line = line.split('>')[-1]

                    ## Don't include Product/Platform, If not in "os_map" Dict
                    prod.append(line)

            if not prod and debug:
                print "Products Not Found for HP-UX"
                return prod

        elif debug:
            print "ERROR: SUPPORTED SOFTWARE VERSION Not Found ..."

        if debug:
            print "\nGenerating Code for (%s) Products " %(prod)

        return prod


    def getRPM(self, debug=0):
        """
        Returns OS Package Dictionary
        """

        if debug:
            print "\nGetting Package List..."

        os_pkg_dict = {}
        prod = []
        pkg_list = []
        rev = []
        patch = []

        ## Get Affected Veresion part from the Advisory
        pkg_data = re.findall('(?s)AFFECTED VERSIONS(.*)END AFFECTED '+ \
                                           'VERSIONS', self.Html_content)
        if not pkg_data:
            pkg_data = re.findall('(?s)AFFECTED VERSIONS(.*)END.*AFFECTED '+\
                                                'VERSIONS', self.Html_content)
        if not pkg_data:
            if debug:
                print "ERROR: AFFECTED VERSIONS Section Not Found..."
            return {}

        ## Remove unwanted starting line
        pkg_data = pkg_data[0]
        pkg_data = pkg_data[pkg_data.find('HP-UX'):]
        if 'swlist -a' in pkg_data:
            pkg_data = pkg_data[5:]
            pkg_data = pkg_data[pkg_data.find('HP-UX'):]

        prd_list = []
        tmp_prd_list = []

        ## Parse line by line
        for i in pkg_data.split('\n'):
            i = i.strip()
            ## Ignore if it has follwing strings
            if not i or "<br>" in i or "<" in i or "===" in i or \
                        "URL" in i or "For" in i or 'criteria:' in i:
                continue

            ## Add into the prod list if line has HP-UX
            if 'HP-UX' in i:
                prod = re.findall('HP-UX\s\s?\s?\s?\s?\s?\s?\s?[a-zA-Z0-9.]+',i)
                if prod:
                    prod = re.sub(' +',' ', prod[0])
                    prd_list.append(prod)

                    ## Store the prd_list data for future use
                    tmp_prd_list = prd_list
                else:
                    ## Error if above pattern doesn't match
                    print "ERROR: Product Not found : ", i
                continue

            if not re.findall('action|fix',i , re.IGNORECASE):
                if len(i.split()) > 1 and re.findall('v[0-9.]+', i):
                    continue

            flag = 0
            ## Get Revision or Patch, if line has action or fix
            if re.findall('action|fix',i , re.IGNORECASE):
                flag = 1
                if not re.findall('(PH\w\w_\d+)',i):
                    ## Get Revision from the line
                    rev = re.findall('revision (.*) or subsequent', i)
                    if not rev:
                        rev = re.findall('upgrade (.*) or subsequent', i)
                    if not rev:
                        rev = re.findall('install (.*) or subsequent', i)
                        if rev:
                            rev = re.findall('[A-Z]?\.?[0-9.]+', rev[0])
                    if not rev and "upgrade" in i or "update" in i:
                        rev = re.findall('[A-Z]?\.?[0-9.]+', i)
                    if not rev:
                        rev = re.findall('[A-Z]?\.?[0-9.]+', i)
                else:
                    ## Get Patch from the line
                    patch = re.findall('(PH.*) or subsequent', i,re.IGNORECASE)
                    if not patch:
                        patch = re.findall('(PH\w\w_\d+)',i)

                ## Ignore the line, If revision matches only '.'
                if rev and rev[0] == '.':
                    rev = []
                    pkg_list = []
                    prd_list = []

                ## Continue, If revision or patch not found
                if not (patch or rev):
                    print "ERROR: Proper Action not found : ", i
                    continue

            ## If line doesn't have action, add the line to pkg_list.
            ## flag will be 1, if it has action or fix in the line
            if not flag:
                ## format the line
                if 'revision=' in i or 'fr=' in i:
                    i = i.split(',')[0]
                i = i.replace('--&gt;','')
                i = i.replace('-&gt;','')
                ## Ignore if it matches following strings
                if 'install' in i or re.findall('PH\w\w_\d+|[A-Z]\.[0-9.]+',i)\
                    or 'subsequent' in i or 'modify' in i or 'restart' in i \
                    or 'REVISED' in i:
                    continue
                pkg_list.append(i)

            ## If action or fix found in the line, then flag will be set to 1
            ## add revision or patch into the apply dict and
            ## clear rev, patch variables
            if flag:
                if rev:
                    apply = {'revision' : rev[0].strip()}
                    rev = []
                elif patch:
                    apply = {'patch' : patch[0]}
                    patch = []

                ## Remove Duplicate packages
                pkg_list = utils.removeDups(pkg_list)

                ## If packages are present but no prd_list is empty.
                ## get the previous product list.
                if not prd_list:
                    prd_list = tmp_prd_list

                ## Iterate over product list one by one.
                for prod in prd_list:
                    ## check os is in os_map dict
                    ## if not, then ignore the os
                    if os_map.has_key(prod):
                        act_prod = os_map[prod]
                    else:
                        print "UPDATE: OS %s not present in the os_map dict " %(prod)
                        continue

                    ## Iterate over packages and add into the proper os
                    for i in pkg_list:
                        if apply.has_key('revision'):
                            i = i + "~" + apply['revision']
                        elif apply.has_key('patch'):
                            i = i + "~" + apply['patch']

                        ## Append is os is already present in the os_pkg_dict,
                        ## else add new os entry.
                        if os_pkg_dict.has_key(act_prod):
                            os_pkg_dict[act_prod].append(i)
                        else:
                            os_pkg_dict[act_prod] = [i]

                pkg_list = []
                prd_list = []

        if debug:
            print "OS PKG Dict : ", os_pkg_dict

        return os_pkg_dict


    def formatReference(self, main_url, file_name):
       """
       Constructs a reference for advisory
       """
       main_url = 'http://www11.itrc.hp.com/service/cki/docDisplay.do'

       tmp = file_name.split('_')

       reference = main_url + '?docId=emr_' + tmp[-2] + '-' + \
                                            tmp[-1].strip('.html')

       return reference


    def parser(self, html_content, debug=0):
        """
        Main parser function, builds the parser object
        by invoking parse functions
        """

        try:
            if debug:
                print "HP-UX Parser Initiated..."

            self.Html_content = html_content.replace('\r\n', '\n')

            self.CVEs = self.getCVE(debug)

            self.Platforms = self.getAffectedProduct(debug)
            if not self.Platforms or self.Platforms == []:
                if debug:
                    print "ERROR: Required Products not found..."
                return False

            self.Packages = self.getRPM(debug)
            if not self.Packages or self.Packages == '':
                if debug:
                    print "ERROR: Required RPMS or Packages not found..."
                return False

            self.Impact = self.getImpact(debug)

            self.Description = self.getDescription(debug)
            if not self.Description or self.Description == '':
                if debug:
                    print "ERROR: Description not found..."
                return False

            self.AdvID = self.getAdvID(debug)
            if not self.AdvID or self.AdvID == '':
                if debug:
                    print "ERROR: Advisory ID not found..."
                return False

            self.Product = self.getAffectedPackage(debug)
            if not self.Product or self.Product == '':
                if debug:
                    print "ERROR: Required Product or Packages not found..."
                return False

            self.Platforms = " ".join(self.Platforms)
            self.Platforms = utils.formatMultiLines(self.Platforms)

            self.Summary = self.Product

            self.Name = self.Product + " " + self.AdvID

            self.FileName = "_".join(['hp_ux', self.AdvID])

            tmp = re.findall('[0-9.]+',self.AdvID)
            if tmp:
                self.XREF = [self.AdvID.replace(tmp[0],''), tmp[0]]
            else:
                self.XREF = ["HPSBUX", self.AdvID]
                if debug:
                    print "ERROR: Advisory ID is in Different Format : ", \
                                                                 self.AdvID
            if debug:
                print "\nAll mandatory attributes are parsed: ", self.AdvID

            return True


        except Exception, msg:
            print 'Exception in Parser hp-ux -> Parser -> parser() Method ',msg
            sys.exit(msg)
