lxml.html.html5parser

Package lxml :: Package html :: Module html5parser |
[hide private]
[frames] | no frames]

# Source Code for Module lxml.html.html5parser

      1  """
      2  An interface to html5lib that mimics the lxml.html interface.
      3  """
      4  import sys
      5  import string
      6
      7  from html5lib import HTMLParser as _HTMLParser
      8  from html5lib.treebuilders.etree_lxml import TreeBuilder
      9  from lxml import etree
     10  from lxml.html import Element, XHTML_NAMESPACE, _contains_block_level_tag
     11
     12  # python3 compatibility
     13  try:
     14      _strings = basestring
     15  except NameError:
     16      _strings = (bytes, str)
     17  try:
     18      from urllib2 import urlopen
     19  except ImportError:
     20      from urllib.request import urlopen
     21  try:
     22      from urlparse import urlparse
     23  except ImportError:
     24      from urllib.parse import urlparse
     25
     26
     27 -class HTMLParser(_HTMLParser):
     28      """An html5lib HTML parser with lxml as tree."""
     29
     30 -    def __init__(self, strict=False, **kwargs):
     31          _HTMLParser.__init__(self, strict=strict, tree=TreeBuilder, **kwargs)
     32
     33
     34  try:
     35      from html5lib import XHTMLParser as _XHTMLParser
     36  except ImportError:
     37      pass
     38  else:
     39 -    class XHTMLParser(_XHTMLParser):
     40          """An html5lib XHTML Parser with lxml as tree."""
     41
     42 -        def __init__(self, strict=False, **kwargs):
     43              _XHTMLParser.__init__(self, strict=strict, tree=TreeBuilder, **kwargs)
     44
     45      xhtml_parser = XHTMLParser()
     46
     47
     48 -def _find_tag(tree, tag):
     49      elem = tree.find(tag)
     50      if elem is not None:
     51          return elem
     52      return tree.find('{%s}%s' % (XHTML_NAMESPACE, tag))
     53
     54
     55 -def document_fromstring(html, guess_charset=None, parser=None):
     56      """
     57      Parse a whole document into a string.
     58
     59      If `guess_charset` is true, or if the input is not Unicode but a
     60      byte string, the `chardet` library will perform charset guessing
     61      on the string.
     62      """
     63      if not isinstance(html, _strings):
     64          raise TypeError('string required')
     65
     66      if parser is None:
     67          parser = html_parser
     68
     69      options = {}
     70      if guess_charset is None and isinstance(html, bytes):
     71          # html5lib does not accept useChardet as an argument, if it
     72          # detected the html argument would produce unicode objects.
     73          guess_charset = True
     74      if guess_charset is not None:
     75          options['useChardet'] = guess_charset
     76      return parser.parse(html, **options).getroot()
     77
     78
     79 -def fragments_fromstring(html, no_leading_text=False,
     80                           guess_charset=None, parser=None):
     81      """Parses several HTML elements, returning a list of elements.
     82
     83      The first item in the list may be a string.  If no_leading_text is true,
     84      then it will be an error if there is leading text, and it will always be
     85      a list of only elements.
     86
     87      If `guess_charset` is true, the `chardet` library will perform charset
     88      guessing on the string.
     89      """
     90      if not isinstance(html, _strings):
     91          raise TypeError('string required')
     92
     93      if parser is None:
     94          parser = html_parser
     95
     96      options = {}
     97      if guess_charset is None and isinstance(html, bytes):
     98          # html5lib does not accept useChardet as an argument, if it
     99          # detected the html argument would produce unicode objects.
    100          guess_charset = False
    101      if guess_charset is not None:
    102          options['useChardet'] = guess_charset
    103      children = parser.parseFragment(html, 'div', **options)
    104      if children and isinstance(children[0], _strings):
    105          if no_leading_text:
    106              if children[0].strip():
    107                  raise etree.ParserError('There is leading text: %r' %
    108                                          children[0])
    109              del children[0]
    110      return children
    111
    112
    113 -def fragment_fromstring(html, create_parent=False,
    114                          guess_charset=None, parser=None):
    115      """Parses a single HTML element; it is an error if there is more than
    116      one element, or if anything but whitespace precedes or follows the
    117      element.
    118
    119      If 'create_parent' is true (or is a tag name) then a parent node
    120      will be created to encapsulate the HTML in a single element.  In
    121      this case, leading or trailing text is allowed.
    122
    123      If `guess_charset` is true, the `chardet` library will perform charset
    124      guessing on the string.
    125      """
    126      if not isinstance(html, _strings):
    127          raise TypeError('string required')
    128
    129      accept_leading_text = bool(create_parent)
    130
    131      elements = fragments_fromstring(
    132          html, guess_charset=guess_charset, parser=parser,
    133          no_leading_text=not accept_leading_text)
    134
    135      if create_parent:
    136          if not isinstance(create_parent, _strings):
    137              create_parent = 'div'
    138          new_root = Element(create_parent)
    139          if elements:
    140              if isinstance(elements[0], _strings):
    141                  new_root.text = elements[0]
    142                  del elements[0]
    143              new_root.extend(elements)
    144          return new_root
    145
    146      if not elements:
    147          raise etree.ParserError('No elements found')
    148      if len(elements) > 1:
    149          raise etree.ParserError('Multiple elements found')
    150      result = elements[0]
    151      if result.tail and result.tail.strip():
    152          raise etree.ParserError('Element followed by text: %r' % result.tail)
    153      result.tail = None
    154      return result
    155
    156
    157 -def fromstring(html, guess_charset=None, parser=None):
    158      """Parse the html, returning a single element/document.
    159
    160      This tries to minimally parse the chunk of text, without knowing if it
    161      is a fragment or a document.
    162
    163      'base_url' will set the document's base_url attribute (and the tree's
    164      docinfo.URL)
    165
    166      If `guess_charset` is true, or if the input is not Unicode but a
    167      byte string, the `chardet` library will perform charset guessing
    168      on the string.
    169      """
    170      if not isinstance(html, _strings):
    171          raise TypeError('string required')
    172      doc = document_fromstring(html, parser=parser,
    173                                guess_charset=guess_charset)
    174
    175      # document starts with doctype or <html>, full document!
    176      start = html[:50]
    177      if isinstance(start, bytes):
    178          # Allow text comparison in python3.
    179          # Decode as ascii, that also covers latin-1 and utf-8 for the
    180          # characters we need.
    181          start = start.decode('ascii', 'replace')
    182
    183      start = start.lstrip().lower()
    184      if start.startswith('<html') or start.startswith('<!doctype'):
    185          return doc
    186
    187      head = _find_tag(doc, 'head')
    188
    189      # if the head is not empty we have a full document
    190      if len(head):
    191          return doc
    192
    193      body = _find_tag(doc, 'body')
    194
    195      # The body has just one element, so it was probably a single
    196      # element passed in
    197      if (len(body) == 1 and (not body.text or not body.text.strip())
    198          and (not body[-1].tail or not body[-1].tail.strip())):
    199          return body[0]
    200
    201      # Now we have a body which represents a bunch of tags which have the
    202      # content that was passed in.  We will create a fake container, which
    203      # is the body tag, except <body> implies too much structure.
    204      if _contains_block_level_tag(body):
    205          body.tag = 'div'
    206      else:
    207          body.tag = 'span'
    208      return body
    209
    210
    211 -def parse(filename_url_or_file, guess_charset=None, parser=None):
    212      """Parse a filename, URL, or file-like object into an HTML document
    213      tree.  Note: this returns a tree, not an element.  Use
    214      ``parse(...).getroot()`` to get the document root.
    215
    216      If ``guess_charset`` is true, the ``useChardet`` option is passed into
    217      html5lib to enable character detection.  This option is on by default
    218      when parsing from URLs, off by default when parsing from file(-like)
    219      objects (which tend to return Unicode more often than not), and on by
    220      default when parsing from a file path (which is read in binary mode).
    221      """
    222      if parser is None:
    223          parser = html_parser
    224      if not isinstance(filename_url_or_file, _strings):
    225          fp = filename_url_or_file
    226          if guess_charset is None:
    227              # assume that file-like objects return Unicode more often than bytes
    228              guess_charset = False
    229      elif _looks_like_url(filename_url_or_file):
    230          fp = urlopen(filename_url_or_file)
    231          if guess_charset is None:
    232              # assume that URLs return bytes
    233              guess_charset = True
    234      else:
    235          fp = open(filename_url_or_file, 'rb')
    236          if guess_charset is None:
    237              guess_charset = True
    238
    239      options = {}
    240      # html5lib does not accept useChardet as an argument, if it
    241      # detected the html argument would produce unicode objects.
    242      if guess_charset:
    243          options['useChardet'] = guess_charset
    244      return parser.parse(fp, **options)
    245
    246
    247 -def _looks_like_url(str):
    248      scheme = urlparse(str)[0]
    249      if not scheme:
    250          return False
    251      elif (sys.platform == 'win32' and
    252              scheme in string.ascii_letters
    253              and len(scheme) == 1):
    254          # looks like a 'normal' absolute path
    255          return False
    256      else:
    257          return True
    258
    259
    260  html_parser = HTMLParser()
    261