(* $Id$
 * ----------------------------------------------------------------------
 *
 *)

(* This file is divided up into sections, marked by (* [SECTION] *).
 * Sections are processed by lexpp.
 *)

(* ---------------------------------------------------------------------- *)
(* [LET] *)

(* These let-definitions work for all lexers. Note that we assume
 * that the character encoding is ASCII-compatible; WE DO THAT IN
 * THE WHOLE LEXER DEFINITION! This means we can simply write 'x'
 * when we mean the letter 'x'.
 * Non-ASCII characters must not be referred to directly, but only
 * by a character class. The character classes are defined in the
 * files char_classes_*.def:
 *
 * - character: The class of all valid characters that may occur in
 *   XML files, including CR, LF, TAB, but excluding other control
 *   characters
 * - baseChar: This class is defined by the XML standard
 * - ideographic: This class is defined by the XML standard
 * - letter: baseChar | ideographic
 * - combiningChar: This class is defined by the XML standard
 * - digit: This class is defined by the XML standard
 * - ascii_digit: The digits that are ASCII characters, i.e. '0'-'9'
 * - ascii_hexdigit: ascii_digit | 'a'-'f' | 'A'-'F'
 * - extender: This class is defined by the XML standard
 *
 * Furthermore, the following "ad hoc" definitions of character classes
 * can be used:
 * - char_but_qmark:             All characters but '?'
 * - char_but_rangle:            All characters but '>'
 * - char_but_minus:             All characters but '-'
 * - char_but_quot:              All characters but '"'
 * - char_but_apos:              All characters but "'"
 * - char_but_rbracket:          All characters but ']'
 * - char_but_rbracket_rangle:   All characters but ']' and '>'
 * - char_ignore:                (Currently:) All characters but '<', ']',
 *                               '"', "'"
 * - pchar_but_amp_lt:           All non-ctrl chars but '&', and '<'
 * - pchar_but_amp_percent:      All non-ctrl chars but '&', and '%'
 * - pchar_text:                 (Currently:) All non-ctrl chars but '&',
 *                               '<', ']', '{', '}'
 * - pchar_ebatt:                (Currently:) All non-ctrl chars but '&',
 *                               '<', '"', "'", '{', '}'
 *
 * The definitions of char_ignore and pchar_text may change in the future
 * if needed, but the other definition will remain intact.
 *)

let ws = [ ' ' '\t' '\r' '\n' ]

let namechar = letter | digit | '.' | ':' | '-' | '_' | combiningChar | extender

let name = ( letter | '_' | ':' ) namechar*

let nmtoken = namechar+

let pi_string = char_but_qmark*
                ( '?' char_but_rangle char_but_qmark* )*
                '?'?
(* A pi_string is the string inside a processing instruction <?...?> *)


let comment_string = char_but_minus*
                     ('-' char_but_minus+ )*

(* A comment_string is the string inside comment delimiters <!-- ... --> *)

let cdata_string =
  char_but_rbracket*
  ( "]" char_but_rbracket+ |
    "]]" ']'* char_but_rbracket_rangle char_but_rbracket*
  )*
  ']'*

(* A cdata_string is the string inside CDATA delimiters <![CDATA[...]]> *)


(* ---------------------------------------------------------------------- *)
(* [HEADER] *)

(* The common header for all output formats *)

  open Pxp_types
  open Pxp_lexer_types
  open Pxp_lex_aux
    (* Hint: All the tok_* values are declared here *)


(* ---------------------------------------------------------------------- *)
(* [HEADER_OCAMLLEX] *)

(* The specific header for the "ocamllex" output format *)

let lexeme_len lo = Pxp_lexing.lexeme_len
  (* Length in _characters_ *)

let sub_lexeme lo = Pxp_lexing.sub_lexeme
  (* Index args given as character counts; returns properly encoded
   * string
   *)

let lexeme lo = Lexing.lexeme
  (* Returns lexeme as properly encoded string *)

let lexeme_char lo lb k =
  Char.code(Lexing.lexeme_char lb k)
  (* Returns the code point of the character k of the lexeme.
   * The code point must be correct for ASCII characters, and
   * must return a non-ASCII number for non-ASCII characters.
   *)


(* ---------------------------------------------------------------------- *)
(* [HEADER_WLEX] *)

(* The specific header for the "wlex" output format *)

(* The same as for OCAMLLEX: *)
let lexeme_len lo = Pxp_lexing.lexeme_len
let sub_lexeme lo = Pxp_lexing.sub_lexeme
let lexeme lo = Lexing.lexeme
let lexeme_char lo lb k = Char.code(Lexing.lexeme_char lb k)

(* ---------------------------------------------------------------------- *)
(* [HEADER_ULEX] *)

(* The specific header for the "ulex" output format *)

module Ulexing = Netulex.Ulexing

let lexeme_len lo = Ulexing.lexeme_length
let sub_lexeme lo lb p n = lo # sub_lexeme p n
let lexeme lo lb = lo # lexeme
let lexeme_char lo lb p = lo # lexeme_char p

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This rule detects an XML declaration *)

detect_xml_pi lexobj = parse
    "<?xml" ws+
      { true }
  | ""
      { false }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This rule returns the contents of a processing instruction *)

scan_pi_string lexobj = parse
    pi_string "?>"
      { let len =
	  lexeme_len lexobj lexbuf in
	Some (sub_lexeme lexobj lexbuf 0 (len-2))
      }
  | ""
      { None }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This rule parses the contents of <?xml ...?> *)

scan_xml_pi lexobj = parse
    name ws*
      { let s = lexeme lexobj lexbuf in
	let j = get_name_end s 0 in   (* Assumes ASCII-compatbility *)
	Pro_name (String.sub s 0 j)
      }
  | "=" ws*
      { Pro_eq }
  | "'" char_but_apos* "'" ws+
      { let s = lexeme lexobj lexbuf in
	let j = String.index_from s 1 '\'' in  (* Assumes ASCII-compatbility *)
	Pro_string (String.sub s 1 (j-1))
      }
  | "'"
      { raise (WF_error ("Cannot find the second quotation mark"))
      }
  | '"' char_but_quot* '"' ws+
      { let s = lexeme lexobj lexbuf in
	let j = String.index_from s 1 '"' in   (* Assumes ASCII-compatbility *)
	Pro_string (String.sub s 1 (j-1))
      }
  | '"'
      { raise (WF_error ("Cannot find the second quotation mark"))
      }
  | eof
      { Pro_eof }
  | ""
      { (* Nothing matches: If at least a valid character follows, raise
         * the exception that this character is not allowed here. Otherwise,
         * raise Malformed_code.
         *)
        lexobj # scan_character();
	raise (WF_error("Illegal token or character"))
      }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* scan_document: Lexer for the outermost structures *)

scan_document lexobj = parse
    "<?"
      { (* Scan the rest of the processing instruction: *)
	match lexobj # scan_pi_string () with
  	  None ->
	    raise (WF_error ("Illegal processing instruction"))
	| Some pi ->
	    scan_pi pi lexobj#factory, Document
            (* scan_pi assumes ASCII-compatbility *)
      }
  | "<!DOCTYPE"
      { tok_Doctype__Document_type }
  | "<!--"
      { Comment_begin dummy_entity, (Comment Document) }
  | "<!"
      { raise (WF_error "Declaration either malformed or not allowed in this context")
      }
  | "<" name
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 1 (l-1) in
	( Tag_beg (s, dummy_entity),
	  Within_tag_entry
        )
      }
  | '<'
      { raise (WF_error ("Illegal token")) }
  | ws+
      { tok_Ignore__Document }
  | eof
      { tok_Eof__Document }
  | ""
      { (* Nothing matches: If at least a valid character follows, raise
         * the exception that this character is not allowed here. Otherwise,
         * raise Malformed_code.
         *)
        lexobj # scan_character ();
 	raise (WF_error ("Content not allowed here"))
      }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* scan_document_type: after "<!DOCTYPE" until matching ">" *)

scan_document_type lexobj = parse
    name
      { let s = lexeme lexobj lexbuf in
	Name s, Document_type }
  | ws+
      { tok_Ignore__Document_type }
  | '"' char_but_quot* '"'
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 1 (l-2) in
	(Unparsed_string s), Document_type }
  | '"'
      { raise (WF_error ("Cannot find the second quotation mark"))
      }
  | "'" char_but_apos* "'"
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 1 (l-2) in
	(Unparsed_string s), Document_type }
  | "'"
      { raise (WF_error ("Cannot find the second quotation mark"))
      }
  | '['
      { tok_Dtd_begin__Declaration }
  | '>'
      { tok_Doctype_rangle__Document }
  | eof
      { tok_Eof__Document_type }
  | '&'
      { raise (WF_error("References to general entities not allowed here")) }
  | '%'
      { raise (WF_error("References to parameter entities not allowed here")) }
  | ""
      { (* Nothing matches: If at least a valid character follows, raise
         * the exception that this character is not allowed here. Otherwise,
         * raise Malformed_code.
         *)
        lexobj # scan_character();
 	raise (WF_error ("Content not allowed here"))
      }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* Scans comments *)

scan_comment lexobj = parse
    "-->"
      { fun lexid ->
          Comment_end dummy_entity, lexid }
  | "--"
      { fun lexid ->
          raise (WF_error "Double hyphens are illegal inside comments") }
  | "-"
      { fun lexid ->
          Comment_material "-", (Comment lexid) }
  | char_but_minus+
      { fun lexid ->
	  Comment_material(lexeme lexobj lexbuf), (Comment lexid) }
  | eof
      { fun lexid -> Eof, (Comment lexid) }
  | _
      { fun lexid -> raise Netconversion.Malformed_code }


(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* scan_declaration: after "[" in DTD until matching "]" *)

scan_declaration lexobj = parse
    ws+
      { tok_Ignore__Declaration }
  | '%' name ';'
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 1 (l-2) in
	(PERef s), Declaration }
  | '%'
      { tok_Percent__Declaration }
  | '&'
      { raise(WF_error("References to general entities not allowed in DTDs")) }
  | name
      { Name (lexeme lexobj lexbuf), Declaration }
  | nmtoken
      { Nametoken (lexeme lexobj lexbuf), Declaration }
  | '+'
      { tok_Plus__Declaration }
  | '*'
      { tok_Star__Declaration }
  | '|'
      { tok_Bar__Declaration }
  | ','
      { tok_Comma__Declaration }
  | '?'
      { tok_Qmark__Declaration }
  | '('
      { tok_Lparen__Declaration }
  | ")+"
      { tok_RparenPlus__Declaration }
  | ")*"
      { tok_RparenStar__Declaration }
  | ")?"
      { tok_RparenQmark__Declaration }
  | ')'
      { tok_Rparen__Declaration }
  | "#REQUIRED"
      { tok_Required__Declaration }
  | "#IMPLIED"
      { tok_Implied__Declaration }
  | "#FIXED"
      { tok_Fixed__Declaration }
  | "#PCDATA"
      { tok_Pcdata__Declaration }
  | "<!ELEMENT"
      { tok_Decl_element__Declaration }
  | "<!ATTLIST"
      { tok_Decl_attlist__Declaration }
  | "<!ENTITY"
      { tok_Decl_entity__Declaration }
  | "<!NOTATION"
      { tok_Decl_notation__Declaration }
  | "<!--"
      { Comment_begin dummy_entity, (Comment Declaration) }
  | "<!["
      { tok_Conditional_begin__Declaration }
  | "]]>"
      { tok_Conditional_end__Declaration }
  | "["
      { tok_Conditional_body__Declaration }
  | "<?"
      { (* Scan the rest of the processing instruction: *)
	match lexobj # scan_pi_string () with
  	  None ->
	    raise (WF_error ("Illegal processing instruction"))
	| Some pi ->
	    scan_pi pi lexobj#factory, Declaration
	    (* scan_pi assume ASCII-compatibility *)
      }
  | '"' [^ '"']* '"'
      { let s = lexeme lexobj lexbuf in
        (* Check that characters are well-formed: *)
	let lexobj' = lexobj # factory # open_string s in
	ignore(lexobj' # scan_characters ());
	let l = lexeme_len lexobj lexbuf in
        let u = sub_lexeme lexobj lexbuf 1 (l-2) in
	(Unparsed_string u), Declaration }
  | '"'
      { raise (WF_error ("Cannot find the second quotation mark"))
      }
  | "'" [^ '\'']* "'"
      { let s = lexeme lexobj lexbuf in
        (* Check that characters are well-formed: *)
	let lexobj' = lexobj # factory # open_string s in
	ignore(lexobj' # scan_characters ());
	let l = lexeme_len lexobj lexbuf in
        let u = sub_lexeme lexobj lexbuf 1 (l-2) in
	(Unparsed_string u), Declaration }
  | "'"
      { raise (WF_error ("Cannot find the second quotation mark"))
      }
  | '>'
      { tok_Decl_rangle__Declaration }
  | ']'
      { tok_Dtd_end__Document_type }
  | eof
      { tok_Eof__Declaration }
  | "<!"
      { raise (WF_error "Declaration either malformed or not allowed in this context")
      }
  | ""
      { (* Nothing matches: If at least a valid character follows, raise
         * the exception that this character is not allowed here. Otherwise,
         * raise Malformed_code.
         *)
        lexobj # scan_character ();
 	raise (WF_error("Illegal token or character"))
      }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This rule checks whether a string consists only of valid characters.
 * Especially for the UTF8 encoding this is a non-trivial rule!
 *)

scan_characters lexobj = parse
  character*
    { () }
| eof
    { () }
| _
    { raise Netconversion.Malformed_code }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This rule checks whether the next character is a valid character.
 * Especially for the UTF8 encoding this is a non-trivial rule!
 *)

scan_character lexobj = parse
  character
    { () }
| eof
    { () }
| _
    { raise Netconversion.Malformed_code }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* The following scanner is used to determine the replacement text of
 * internal entities:
 *)

scan_dtd_string lexobj = parse
    '%' name ';'
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 1 (l-2) in
	PERef s }
  | '%'
      { raise(WF_error("The character '%' must be written as '&#37;'")) }
  | '&' name ';'
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 1 (l-2) in
	ERef s }
  | "&#" ascii_digit+ ";"
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 2 (l-3) in
	CRef (int_of_string s) }
  | "&#x" ascii_hexdigit+ ";"
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 3 (l-4) in
	CRef (int_of_string ("0x" ^ s)) }
  | '&'
      { raise(WF_error("The character '&' must be written as '&amp;'")) }
  | '\013' '\010'
      { CRef(-1) }
  | '\013'
      { CRef(-2) }
  | '\010'
      { CRef(-3) }
  | '\009'
      { tok_CharDataTAB }
  | pchar_but_amp_percent+
      { CharData (lexeme lexobj lexbuf) }
  | eof
      { Eof }
  | _
      { raise Netconversion.Malformed_code }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* Scan in content context *)

scan_content lexobj = parse
    "<?"
      { (* Scan the rest of the processing instruction: *)
	match lexobj # scan_pi_string () with
  	  None ->
	    raise (WF_error ("Illegal processing instruction"))
	| Some pi ->
	    scan_pi pi lexobj#factory, Content
            (* scan_pi assumes ASCII-compatibility *)
      }
  | "<!--"
      { Comment_begin dummy_entity, (Comment Content) }
  | '<' '/'? name
      (* One rule for Tag_beg and Tag_end saves transitions. *)
      { let l = lexeme_len lexobj lexbuf in
	if lexeme_char lexobj lexbuf 1 = 47 then
	  Tag_end (sub_lexeme lexobj lexbuf 2 (l-2), dummy_entity),
	  Within_tag_entry
	else
	  Tag_beg (sub_lexeme lexobj lexbuf 1 (l-1), dummy_entity),
	  Within_tag_entry
      }
  | "<![CDATA[" cdata_string "]]>"
      { let l = lexeme_len lexobj lexbuf in
	Cdata (sub_lexeme lexobj lexbuf 9 (l-12)), Content }
  | "<!"
      { raise (WF_error "Declaration either malformed or not allowed in this context")
      }
  | "<"
      { raise (WF_error ("The left angle bracket '<' must be written as '&lt;'"))
      }
  | "&#" ascii_digit+ ";"
      { let l = lexeme_len lexobj lexbuf in
	CRef (int_of_string (sub_lexeme lexobj lexbuf 2 (l-3))), Content }
  | "&#x" ascii_hexdigit+ ";"
      { let l = lexeme_len lexobj lexbuf in
	CRef (int_of_string ("0x" ^ sub_lexeme lexobj lexbuf 3 (l-4))), Content }
  | "&" name ";"
      { let l = lexeme_len lexobj lexbuf in
	ERef (sub_lexeme lexobj lexbuf 1 (l-2)), Content }
  | "&"
      { raise (WF_error ("The ampersand '&' must be written as '&amp;'"))
      }
  | "{{"
      { tok_LLcurly__Content }
  | "}}"
      { tok_RRcurly__Content }
  | "{"
      { tok_Lcurly__Content }
  | "}"
      { tok_Rcurly__Content }

  (* LineEnd: Depending on whether we are reading from a primary source
   * (file) or from the replacement text of an internal entity, line endings
   * must be normalized (converted to \n) or not.
   * The entity classes do that. The yacc parser will never see LineEnd;
   * this token is always converted to the appropriate CharData token.
   *)

  | '\013' '\010'
      { tok_LineEndCRLF__Content }
  | '\013'
      { tok_LineEndCR__Content }
  | '\010'
      { tok_LineEndLF__Content }
  | '\009'
      { tok_CharDataTAB__Content }
  | eof
      { tok_Eof__Content }
  | "]]>"
      { raise (WF_error ("The sequence ']]>' must be written as ']]&gt;'"))
      }
  | "]"
      { tok_CharDataRBRACKET__Content }
  | pchar_text+
      { let s = lexeme lexobj lexbuf in
	CharData s, Content
      }
  | _
      { raise Netconversion.Malformed_code }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This rule is used inside start tags of elements *)

scan_within_tag lexobj = parse
    '\013' '\010'
      { tok_IgnoreLineEnd__Within_tag }
  | '\013'
      { tok_IgnoreLineEnd__Within_tag }
  | '\010'
      { tok_IgnoreLineEnd__Within_tag }
  | [' ' '\t']+
      { tok_Ignore__Within_tag }
  | name
      { Name (lexeme lexobj lexbuf ), Within_tag }
  | '='
      { tok_Eq__Within_tag }
  | '"' char_but_quot* '"'
      { let l = lexeme_len lexobj lexbuf in
	let v = sub_lexeme lexobj lexbuf 1 (l-2) in
	Attval v, Within_tag }
  | '"'
      { raise (WF_error ("Cannot find the second quotation mark"))
      }
  | "'" char_but_apos* "'"
      { let l = lexeme_len lexobj lexbuf in
	let v = sub_lexeme lexobj lexbuf 1 (l-2) in
	Attval v, Within_tag }
  | "'"
      { raise (WF_error ("Cannot find the second quotation mark"))
      }
  | '>'
      { tok_Rangle__Content }
  | "/>"
      { tok_Rangle_empty__Content }
  | eof
      { tok_Eof__Within_tag }
  | ""
      { (* Nothing matches: If at least a valid character follows, raise
         * the exception that this character is not allowed here. Otherwise,
         * raise Malformed_code.
         *)
        lexobj # scan_character();
 	raise (WF_error ("Illegal inside tags"))
      }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* The following rule is used instead of scan_within_tag if event-based
 * attribute parsing is enabled. The difference is that more tokens are
 * generated for attribute values.
 *)

scan_tag_eb lexobj = parse
    '\013' '\010'
      { tok_IgnoreLineEnd__Tag_eb }
  | '\013'
      { tok_IgnoreLineEnd__Tag_eb }
  | '\010'
      { tok_IgnoreLineEnd__Tag_eb }
  | [' ' '\t']+
      { tok_Ignore__Tag_eb }
  | name
      { Name (lexeme lexobj lexbuf ), Tag_eb }
  | '='
      { tok_Eq__Tag_eb }
  | '"'
      { tok_DQuote__Tag_eb_att_true }
  | "'"
      { tok_SQuote__Tag_eb_att_false }
  | '>'
      { tok_Rangle__Content }
  | "/>"
      { tok_Rangle_empty__Content }
  | eof
      { tok_Eof__Tag_eb }
  | ""
      { (* Nothing matches: If at least a valid character follows, raise
         * the exception that this character is not allowed here. Otherwise,
         * raise Malformed_code.
         *)
        lexobj # scan_character ();
 	raise (WF_error ("Illegal inside tags"))
      }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This lexer is used to expand and normalize attribute values: *)

(* TODO: Use ERef_att instead of ERef *)

scan_content_string lexobj = parse
    '&' name ';'
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 1 (l-2) in
	ERef s }
  | "&#" ascii_digit+ ";"
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 2 (l-3) in
	CRef (int_of_string s) }
  | "&#x" ascii_hexdigit+ ";"
      { let l = lexeme_len lexobj lexbuf in
        let s = sub_lexeme lexobj lexbuf 3 (l-4) in
	CRef (int_of_string ("0x" ^ s)) }
  | '&'
      { raise(WF_error("The character '&' must be written as '&amp;'")) }
  | pchar_but_amp_lt+
      { CharData ""  (* TODO *) (* (lexeme lexbuf) *) }
  | '\009'
      { CRef 32 }
  | '\013' '\010'
      { CRef(-1)   (* A special case *)
      }
  | '\013'
      { CRef 32 }
  | '\010'
      { CRef 32 }
  | '<'
      {
	(* Depending on the situation, '<' may be legal or not: *)
	tok_CharDataLT
      }
  | eof
      { Eof }
  | _
      { raise Netconversion.Malformed_code }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* The following lexer is an alternative. It is used if event-based
 * attribute parsing is enabled.
 *
 * Note: The argument of the functions is whether the attribute value
 * is delimited by double quotes (true) or single quotes (false).
 *)

scan_tag_eb_att lexobj = parse
    '&' name ';'
      { fun d ->
          let l = lexeme_len lexobj lexbuf in
	  let s = sub_lexeme lexobj lexbuf 1 (l-2) in
	  (ERef_att s), Tag_eb_att d
      }
  | "&#" ascii_digit+ ";"
      { fun d ->
          let l = lexeme_len lexobj lexbuf in
	  let s = sub_lexeme lexobj lexbuf 2 (l-3) in
	  (CRef (int_of_string s), Tag_eb_att d)
      }
  | "&#x" ascii_hexdigit+ ";"
      { fun d ->
          let l = lexeme_len lexobj lexbuf in
	  let s = sub_lexeme lexobj lexbuf 3 (l-4) in
	  (CRef (int_of_string ("0x" ^ s)), Tag_eb_att d)
      }
  | '&'
      { fun _ ->
          raise(WF_error("The character '&' must be written as '&amp;'")) }
  | '\009'
      { fun d ->
          tok_CharDataSPACE, Tag_eb_att d }
  | '\013' '\010'
      { fun d ->
	  LineEnd_att "  ", Tag_eb_att d }
  | '\013'
      { fun d ->
          LineEnd_att " ", Tag_eb_att d }
  | '\010'
      { fun d ->
          LineEnd_att " ", Tag_eb_att d }
  | '<'
      { fun d ->
	  (* Depending on the situation, '<' may be legal or not: *)
	  tok_CharDataLT, Tag_eb_att d
      }
  | '"'
      { function
          true  -> DQuote, Tag_eb
        | false -> tok_CharDataQUOT, Tag_eb_att false
      }
  | '\''
      { function
          true  -> tok_CharDataAPOS, Tag_eb_att true
        | false -> SQuote, Tag_eb
      }
  | "{{"
      { fun d -> LLcurly, Tag_eb_att d }
  | "{"
      { fun d -> Lcurly, Tag_eb_att d }
  | "}}"
      { fun d -> RRcurly, Tag_eb_att d }
  | "}"
      { fun d -> Rcurly, Tag_eb_att d }
  | pchar_ebatt+
      { fun d -> CharData (lexeme lexobj lexbuf), Tag_eb_att d }
  | eof
      { fun d -> Eof, Tag_eb_att d }
  | _
      { fun _ -> raise Netconversion.Malformed_code }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This rule is used to parse NMTOKEN or NMTOKENS attribute values *)

(* Note: This lexer is also used by WDialog. *)

scan_name_string lexobj = parse
    name
      { Name (lexeme lexobj lexbuf) }
  | ws+
      { Ignore }
  | nmtoken
      { Nametoken (lexeme lexobj lexbuf) }
  | eof
      { Eof }
  | character
      { CharData (lexeme lexobj lexbuf) }
  | _
      { raise Netconversion.Malformed_code }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This rule is used to skip over ignored sections <![IGNORE[ ... ]]> *)

scan_ignored_section lexobj = parse
  | "<!["
      { tok_Conditional_begin__Ignored }
  | "]]>"
      { tok_Conditional_end__Ignored }
  | "<!--" comment_string "-->"
      { tok_Ignore__Ignored }
  | '"' char_but_quot* '"'
      { tok_Ignore__Ignored }
  | "'" char_but_apos* "'"
      { tok_Ignore__Ignored }
  | eof
      { tok_Eof__Ignored }
  | char_ignore+
      { tok_Ignore__Ignored }
  | "<"
      { tok_Ignore__Ignored }
  | "]"
      { tok_Ignore__Ignored }
  | "'"
      { tok_Ignore__Ignored }
  | "\""
      { tok_Ignore__Ignored }
  | _
      { raise Netconversion.Malformed_code }

(* ---------------------------------------------------------------------- *)
(* [RULE] *)

(* This rule locates CR, CRLF, and LF characters inside strings. *)

scan_for_crlf lexobj = parse
  | '\013' '\010'
      { tok_CharDataLF }
  | '\013'
      { tok_CharDataLF }
  | '\010'
      { tok_CharDataLF }
  | [^ '\010' '\013' ]+
      { CharData (lexeme lexobj lexbuf) }
  | eof
      { Eof }

(* [END] *)


