(* antlr-tokens-sig.sml
 *
 * COPYRIGHT (c) 2006-2015
 * John Reppy (http://www.cs.uchicago.edu/~jhr)
 * Aaron Turon (http://www.cs.uchicago.edu/~adrassi)
 * All rights reserved.
 *
 * Signature for tokens module generated by ml-antlr
 *)

signature ANTLR_TOKENS =
  sig

    type token

  (* return a string representation of a token that is suitable for error messages *)
    val toString : token -> string

  (* a list of all the tokens in the grammar *)
    val allToks : token list

  (* is a token marked as a keyword?  Error repairs that do not involve keywords
   * are preferred.
   *)
    val isKW    : token -> bool

  (* is a token the end-of-file marker *)
    val isEOF   : token -> bool

end

(* an extension of the TOKENS signature for when the grammar specifies
 * preferred insertions (%prefer) and substitutions (%change).
 *)
signature ANTLR_TOKENS_WITH_CHANGES =
  sig

    include ANTLR_TOKENS

  (* an element in a stream of preferred changes.  An item
   *
   *	CHANGE(strm, n, new, next)
   *
   * has components
   *	s	the underlying token stream after consuming the old
   *		tokens.
   *	n	the number of old tokens removed
   *	new	the list of new tokens to replace the old tokens
   *	next	the function for getting the next change in the stream
   *		of preferred changes.
   *)
    datatype 'strm changes
      = CHANGE of 'strm * int * token list * ('strm -> 'strm changes)
      | NOCHANGE;

  (* given a function for reading tokens from the token stream, return
   * a function for getting the first change in the preferred changes
   * stream.
   *)
    val changes : ('strm -> token * 'strm) -> 'strm -> 'strm changes

  end
