/*
 * Copyright 2007 The Closure Compiler Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.javascript.jscomp.parsing;

import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.javascript.jscomp.parsing.Config.LanguageMode;
import com.google.javascript.rhino.IR;
import com.google.javascript.rhino.JSDocInfo;
import com.google.javascript.rhino.JSDocInfo.Visibility;
import com.google.javascript.rhino.JSDocInfoBuilder;
import com.google.javascript.rhino.JSTypeExpression;
import com.google.javascript.rhino.Node;
import com.google.javascript.rhino.ScriptRuntime;
import com.google.javascript.rhino.Token;
import com.google.javascript.rhino.head.ErrorReporter;
import com.google.javascript.rhino.head.ast.Comment;
import com.google.javascript.rhino.jstype.StaticSourceFile;

import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * A parser for JSDoc comments.
 *
 */
// TODO(nicksantos): Unify all the JSDocInfo stuff into one package, instead of
// spreading it across multiple packages.
public final class JsDocInfoParser {

  private final JsDocTokenStream stream;
  private final JSDocInfoBuilder jsdocBuilder;
  private final StaticSourceFile sourceFile;
  private final Node associatedNode;
  private final ErrorReporter errorReporter;
  private final ErrorReporterParser parser = new ErrorReporterParser();

  // Use a template node for properties set on all nodes to minimize the
  // memory footprint associated with these (similar to IRFactory).
  private final Node templateNode;

  private class ErrorReporterParser {
    void addParserWarning(String messageId, String messageArg, int lineno,
        int charno) {
      errorReporter.warning(ScriptRuntime.getMessage1(messageId, messageArg),
          getSourceName(), lineno, null, charno);
    }

    void addParserWarning(String messageId, int lineno, int charno) {
      errorReporter.warning(ScriptRuntime.getMessage0(messageId),
          getSourceName(), lineno, null, charno);
    }

    void addTypeWarning(String messageId, String messageArg, int lineno,
                    int charno) {
      errorReporter.warning(
          "Bad type annotation. " +
          ScriptRuntime.getMessage1(messageId, messageArg),
          getSourceName(), lineno, null, charno);
    }

    void addTypeWarning(String messageId, int lineno, int charno) {
      errorReporter.warning(
          "Bad type annotation. " +
          ScriptRuntime.getMessage0(messageId),
          getSourceName(), lineno, null, charno);
    }
  }

  // The DocInfo with the fileoverview tag for the whole file.
  private JSDocInfo fileOverviewJSDocInfo = null;
  private State state;

  private final Map<String, Annotation> annotationNames;
  private final Set<String> suppressionNames;
  static private final Set<String> modifiesAnnotationKeywords =
      ImmutableSet.<String>of("this", "arguments");

  private Node.FileLevelJsDocBuilder fileLevelJsDocBuilder;

  /**
   * Sets the JsDocBuilder for the file-level (root) node of this parse. The
   * parser uses the builder to append any preserve annotations it encounters
   * in JsDoc comments.
   *
   * @param fileLevelJsDocBuilder
   */
  void setFileLevelJsDocBuilder(
      Node.FileLevelJsDocBuilder fileLevelJsDocBuilder) {
    this.fileLevelJsDocBuilder = fileLevelJsDocBuilder;
  }

  /**
   * Sets the file overview JSDocInfo, in order to warn about multiple uses of
   * the @fileoverview tag in a file.
   */
  void setFileOverviewJSDocInfo(JSDocInfo fileOverviewJSDocInfo) {
    this.fileOverviewJSDocInfo = fileOverviewJSDocInfo;
  }

  private enum State {
    SEARCHING_ANNOTATION,
    SEARCHING_NEWLINE,
    NEXT_IS_ANNOTATION
  }

  JsDocInfoParser(JsDocTokenStream stream,
                  Comment commentNode,
                  Node associatedNode,
                  Config config,
                  ErrorReporter errorReporter) {
    this.stream = stream;
    this.associatedNode = associatedNode;

    // Sometimes this will be null in tests.
    this.sourceFile = associatedNode == null
        ? null : associatedNode.getStaticSourceFile();

    this.jsdocBuilder = new JSDocInfoBuilder(config.parseJsDocDocumentation);
    if (commentNode != null) {
      this.jsdocBuilder.recordOriginalCommentString(commentNode.getValue());
    }
    this.annotationNames = config.annotationNames;
    this.suppressionNames = config.suppressionNames;

    this.errorReporter = errorReporter;
    this.templateNode = this.createTemplateNode();
  }

  private String getSourceName() {
    return sourceFile == null ? null : sourceFile.getName();
  }

  /**
   * Parse a description as a {@code @type}.
   */
  public JSDocInfo parseInlineTypeDoc() {
    skipEOLs();

    Node typeAst = parseAndRecordTypeNode(next());
    JSTypeExpression expr = createJSTypeExpression(typeAst);
    if (expr != null) {
      jsdocBuilder.recordType(expr);
      return retrieveAndResetParsedJSDocInfo();
    }
    return null;
  }

  /**
   * Parses a string containing a JsDoc type declaration, returning the
   * type if the parsing succeeded or {@code null} if it failed.
   */
  public static Node parseTypeString(String typeString) {
    Config config = new Config(
        Sets.<String>newHashSet(),
        Sets.<String>newHashSet(),
        false,
        LanguageMode.ECMASCRIPT3,
        false);
    JsDocInfoParser parser = new JsDocInfoParser(
        new JsDocTokenStream(typeString),
        null,
        null,
        config,
        NullErrorReporter.forNewRhino());

    return parser.parseTopLevelTypeExpression(parser.next());
  }

  /**
   * Parses a {@link JSDocInfo} object. This parsing method reads all tokens
   * returned by the {@link JsDocTokenStream#getJsDocToken()} method until the
   * {@link JsDocToken#EOC} is returned.
   *
   * @return {@code true} if JSDoc information was correctly parsed,
   *     {@code false} otherwise
   */
  boolean parse() {
    state = State.SEARCHING_ANNOTATION;
    skipEOLs();

    JsDocToken token = next();

    // Always record that we have a comment.
    if (jsdocBuilder.shouldParseDocumentation()) {
      ExtractionInfo blockInfo = extractBlockComment(token);
      token = blockInfo.token;
      if (!blockInfo.string.isEmpty()) {
        jsdocBuilder.recordBlockDescription(blockInfo.string);
      }
    } else {
      if (token != JsDocToken.ANNOTATION &&
          token != JsDocToken.EOC) {
        // Mark that there was a description, but don't bother marking
        // what it was.
        jsdocBuilder.recordBlockDescription("");
      }
    }

    return parseHelperLoop(token, Lists.<ExtendedTypeInfo>newArrayList());
  }

  private boolean parseHelperLoop(JsDocToken token,
                                  List<ExtendedTypeInfo> extendedTypes) {
    while (true) {
      switch (token) {
        case ANNOTATION:
          if (state == State.SEARCHING_ANNOTATION) {
            state = State.SEARCHING_NEWLINE;
            token = parseAnnotation(token, extendedTypes);
          } else {
            token = next();
          }
          break;

        case EOC:
          if (hasParsedFileOverviewDocInfo()) {
            fileOverviewJSDocInfo = retrieveAndResetParsedJSDocInfo();
          }
          checkExtendedTypes(extendedTypes);
          return true;

        case EOF:
          // discard any accumulated information
          jsdocBuilder.build(null);
          parser.addParserWarning("msg.unexpected.eof",
              stream.getLineno(), stream.getCharno());
          checkExtendedTypes(extendedTypes);
          return false;

        case EOL:
          if (state == State.SEARCHING_NEWLINE) {
            state = State.SEARCHING_ANNOTATION;
          }
          token = next();
          break;

        default:
          if (token == JsDocToken.STAR && state == State.SEARCHING_ANNOTATION) {
            token = next();
          } else {
            state = State.SEARCHING_NEWLINE;
            token = eatTokensUntilEOL();
          }
          break;
      }
    }
  }

  private JsDocToken parseAnnotation(JsDocToken token,
      List<ExtendedTypeInfo> extendedTypes) {
    // JSTypes are represented as Rhino AST nodes, and then resolved later.
    JSTypeExpression type;
    int lineno = stream.getLineno();
    int charno = stream.getCharno();

    String annotationName = stream.getString();
    Annotation annotation = annotationNames.get(annotationName);
    if (annotation == null) {
      parser.addParserWarning("msg.bad.jsdoc.tag", annotationName,
          stream.getLineno(), stream.getCharno());
    } else {
      // Mark the beginning of the annotation.
      jsdocBuilder.markAnnotation(annotationName, lineno, charno);

      switch (annotation) {
        case NG_INJECT:
          if (jsdocBuilder.isNgInjectRecorded()) {
            parser.addParserWarning("msg.jsdoc.nginject.extra",
              stream.getLineno(), stream.getCharno());
          } else {
            jsdocBuilder.recordNgInject(true);
          }
          return eatTokensUntilEOL();

        case AUTHOR:
          if (jsdocBuilder.shouldParseDocumentation()) {
            ExtractionInfo authorInfo = extractSingleLineBlock();
            String author = authorInfo.string;

            if (author.length() == 0) {
              parser.addParserWarning("msg.jsdoc.authormissing",
                  stream.getLineno(), stream.getCharno());
            } else {
              jsdocBuilder.addAuthor(author);
            }
            token = authorInfo.token;
          } else {
            token = eatTokensUntilEOL(token);
          }
          return token;

        case CONSISTENTIDGENERATOR:
          if (!jsdocBuilder.recordConsistentIdGenerator()) {
            parser.addParserWarning("msg.jsdoc.consistidgen",
              stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case STRUCT:
          if (!jsdocBuilder.recordStruct()) {
            parser.addTypeWarning("msg.jsdoc.incompat.type",
                                  stream.getLineno(),
                                  stream.getCharno());
          }
          return eatTokensUntilEOL();

        case DICT:
          if (!jsdocBuilder.recordDict()) {
            parser.addTypeWarning("msg.jsdoc.incompat.type",
                                  stream.getLineno(),
                                  stream.getCharno());
          }
          return eatTokensUntilEOL();

        case CONSTRUCTOR:
          if (!jsdocBuilder.recordConstructor()) {
            if (jsdocBuilder.isInterfaceRecorded()) {
              parser.addTypeWarning("msg.jsdoc.interface.constructor",
                  stream.getLineno(), stream.getCharno());
            } else {
              parser.addTypeWarning("msg.jsdoc.incompat.type",
                  stream.getLineno(), stream.getCharno());
            }
          }
          return eatTokensUntilEOL();

        case DEPRECATED:
          if (!jsdocBuilder.recordDeprecated()) {
            parser.addParserWarning("msg.jsdoc.deprecated",
                stream.getLineno(), stream.getCharno());
          }

          // Find the reason/description, if any.
          ExtractionInfo reasonInfo =
              extractMultilineTextualBlock(token);

          String reason = reasonInfo.string;

          if (reason.length() > 0) {
            jsdocBuilder.recordDeprecationReason(reason);
          }

          token = reasonInfo.token;
          return token;

        case INTERFACE:
          if (!jsdocBuilder.recordInterface()) {
            if (jsdocBuilder.isConstructorRecorded()) {
              parser.addTypeWarning("msg.jsdoc.interface.constructor",
                  stream.getLineno(), stream.getCharno());
            } else {
              parser.addTypeWarning("msg.jsdoc.incompat.type",
                  stream.getLineno(), stream.getCharno());
            }
          }
          return eatTokensUntilEOL();

        case DESC:
          if (jsdocBuilder.isDescriptionRecorded()) {
            parser.addParserWarning("msg.jsdoc.desc.extra",
                stream.getLineno(), stream.getCharno());
            return eatTokensUntilEOL();
          } else {
            ExtractionInfo descriptionInfo =
                extractMultilineTextualBlock(token);

            String description = descriptionInfo.string;

            jsdocBuilder.recordDescription(description);
            token = descriptionInfo.token;
            return token;
          }

        case FILE_OVERVIEW:
          String fileOverview = "";
          if (jsdocBuilder.shouldParseDocumentation()) {
            ExtractionInfo fileOverviewInfo =
                extractMultilineTextualBlock(token,
                    WhitespaceOption.TRIM);

            fileOverview = fileOverviewInfo.string;

            token = fileOverviewInfo.token;
          } else {
            token = eatTokensUntilEOL(token);
          }

          if (!jsdocBuilder.recordFileOverview(fileOverview)) {
            parser.addParserWarning("msg.jsdoc.fileoverview.extra",
                stream.getLineno(), stream.getCharno());
          }
          return token;

        case LICENSE:
        case PRESERVE:
          ExtractionInfo preserveInfo =
              extractMultilineTextualBlock(token,
                                           WhitespaceOption.PRESERVE);

          String preserve = preserveInfo.string;

          if (preserve.length() > 0) {
            if (fileLevelJsDocBuilder != null) {
              fileLevelJsDocBuilder.append(preserve);
            }
          }

          token = preserveInfo.token;
          return token;

        case ENUM:
          token = next();
          lineno = stream.getLineno();
          charno = stream.getCharno();

          type = null;
          if (token != JsDocToken.EOL && token != JsDocToken.EOC) {
            type = createJSTypeExpression(
                parseAndRecordTypeNode(token));
          }

          if (type == null) {
            type = createJSTypeExpression(newStringNode("number"));
          }
          if (!jsdocBuilder.recordEnumParameterType(type)) {
            parser.addTypeWarning(
                "msg.jsdoc.incompat.type", lineno, charno);
          }
          token = eatTokensUntilEOL(token);
          return token;

        case EXPORT:
          if (!jsdocBuilder.recordExport()) {
            parser.addParserWarning("msg.jsdoc.export",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case EXPOSE:
          if (!jsdocBuilder.recordExpose()) {
            parser.addParserWarning("msg.jsdoc.expose",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case EXTERNS:
          if (!jsdocBuilder.recordExterns()) {
            parser.addParserWarning("msg.jsdoc.externs",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case JAVA_DISPATCH:
          if (!jsdocBuilder.recordJavaDispatch()) {
            parser.addParserWarning("msg.jsdoc.javadispatch",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case EXTENDS:
        case IMPLEMENTS:
          skipEOLs();
          token = next();
          lineno = stream.getLineno();
          charno = stream.getCharno();
          boolean matchingRc = false;

          if (token == JsDocToken.LC) {
            token = next();
            matchingRc = true;
          }

          if (token == JsDocToken.STRING) {
            Node typeNode = parseAndRecordTypeNameNode(
                token, lineno, charno, matchingRc);

            lineno = stream.getLineno();
            charno = stream.getCharno();

            typeNode = wrapNode(Token.BANG, typeNode);
            type = createJSTypeExpression(typeNode);

            if (annotation == Annotation.EXTENDS) {
              // record the extended type, check later
              extendedTypes.add(new ExtendedTypeInfo(
                  type, stream.getLineno(), stream.getCharno()));
            } else {
              Preconditions.checkState(
                  annotation == Annotation.IMPLEMENTS);
              if (!jsdocBuilder.recordImplementedInterface(type)) {
                parser.addTypeWarning("msg.jsdoc.implements.duplicate",
                    lineno, charno);
              }
            }
            token = next();
            if (matchingRc) {
              if (token != JsDocToken.RC) {
                parser.addTypeWarning("msg.jsdoc.missing.rc",
                    stream.getLineno(), stream.getCharno());
              }
            } else if (token != JsDocToken.EOL &&
                token != JsDocToken.EOF && token != JsDocToken.EOC) {
              parser.addTypeWarning("msg.end.annotation.expected",
                  stream.getLineno(), stream.getCharno());
            }
          } else {
            parser.addTypeWarning("msg.no.type.name", lineno, charno);
          }
          token = eatTokensUntilEOL(token);
          return token;

        case HIDDEN:
          if (!jsdocBuilder.recordHiddenness()) {
            parser.addParserWarning("msg.jsdoc.hidden",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case LENDS:
          skipEOLs();

          matchingRc = false;
          if (match(JsDocToken.LC)) {
            token = next();
            matchingRc = true;
          }

          if (match(JsDocToken.STRING)) {
            token = next();
            if (!jsdocBuilder.recordLends(stream.getString())) {
              parser.addTypeWarning("msg.jsdoc.lends.incompatible",
                  stream.getLineno(), stream.getCharno());
            }
          } else {
            parser.addTypeWarning("msg.jsdoc.lends.missing",
                stream.getLineno(), stream.getCharno());
          }

          if (matchingRc && !match(JsDocToken.RC)) {
            parser.addTypeWarning("msg.jsdoc.missing.rc",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case MEANING:
          ExtractionInfo meaningInfo =
              extractMultilineTextualBlock(token);
          String meaning = meaningInfo.string;
          token = meaningInfo.token;
          if (!jsdocBuilder.recordMeaning(meaning)) {
            parser.addParserWarning("msg.jsdoc.meaning.extra",
                stream.getLineno(), stream.getCharno());
          }
          return token;

        case NO_ALIAS:
          if (!jsdocBuilder.recordNoAlias()) {
            parser.addParserWarning("msg.jsdoc.noalias",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case NO_COMPILE:
          if (!jsdocBuilder.recordNoCompile()) {
            parser.addParserWarning("msg.jsdoc.nocompile",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case NO_TYPE_CHECK:
          if (!jsdocBuilder.recordNoTypeCheck()) {
            parser.addParserWarning("msg.jsdoc.nocheck",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case NOT_IMPLEMENTED:
          return eatTokensUntilEOL();

        case INHERIT_DOC:
        case OVERRIDE:
          if (!jsdocBuilder.recordOverride()) {
            parser.addTypeWarning("msg.jsdoc.override",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case THROWS:
          skipEOLs();
          token = next();
          lineno = stream.getLineno();
          charno = stream.getCharno();
          type = null;

          if (token == JsDocToken.LC) {
            type = createJSTypeExpression(
                parseAndRecordTypeNode(token));

            if (type == null) {
              // parsing error reported during recursive descent
              // recovering parsing
              return eatTokensUntilEOL();
            }
          }

          // *Update* the token to that after the type annotation.
          token = current();

          // Save the throw type.
          jsdocBuilder.recordThrowType(type);

          // Find the throw's description (if applicable).
          if (jsdocBuilder.shouldParseDocumentation()) {
            ExtractionInfo descriptionInfo =
                extractMultilineTextualBlock(token);

            String description = descriptionInfo.string;

            if (description.length() > 0) {
              jsdocBuilder.recordThrowDescription(type, description);
            }

            token = descriptionInfo.token;
          } else {
            token = eatTokensUntilEOL(token);
          }
          return token;

        case PARAM:
          skipEOLs();
          token = next();
          lineno = stream.getLineno();
          charno = stream.getCharno();
          type = null;

          if (token == JsDocToken.LC) {
            type = createJSTypeExpression(
                parseAndRecordParamTypeNode(token));

            if (type == null) {
              // parsing error reported during recursive descent
              // recovering parsing
              return eatTokensUntilEOL();
            }
            skipEOLs();
            token = next();
            lineno = stream.getLineno();
            charno = stream.getCharno();
          }

          String name = null;
          boolean isBracketedParam = JsDocToken.LB == token;
          if (isBracketedParam) {
            token = next();
          }

          if (JsDocToken.STRING != token) {
            parser.addTypeWarning("msg.missing.variable.name",
                lineno, charno);
          } else {
            name = stream.getString();

            if (isBracketedParam) {
              token = next();

              // Throw out JsDocToolkit's "default" parameter
              // annotation.  It makes no sense under our type
              // system.
              if (JsDocToken.EQUALS == token) {
                token = next();
                if (JsDocToken.STRING == token) {
                  token = next();
                }
              }

              if (JsDocToken.RB != token) {
                reportTypeSyntaxWarning("msg.jsdoc.missing.rb");
              } else if (type != null) {
                // Make the type expression optional, if it isn't
                // already.
                type = JSTypeExpression.makeOptionalArg(type);
              }
            }

            // If the param name has a DOT in it, just throw it out
            // quietly. We do not handle the JsDocToolkit method
            // for handling properties of params.
            if (name.indexOf('.') > -1) {
              name = null;
            } else if (!jsdocBuilder.recordParameter(name, type)) {
              if (jsdocBuilder.hasParameter(name)) {
                parser.addTypeWarning("msg.dup.variable.name", name,
                    lineno, charno);
              } else {
                parser.addTypeWarning("msg.jsdoc.incompat.type", name,
                    lineno, charno);
              }
            }
          }

          if (name == null) {
            token = eatTokensUntilEOL(token);
            return token;
          }

          jsdocBuilder.markName(name, sourceFile, lineno, charno);

          // Find the parameter's description (if applicable).
          if (jsdocBuilder.shouldParseDocumentation()) {
            ExtractionInfo paramDescriptionInfo =
                extractMultilineTextualBlock(token);

            String paramDescription = paramDescriptionInfo.string;

            if (paramDescription.length() > 0) {
              jsdocBuilder.recordParameterDescription(name,
                  paramDescription);
            }

            token = paramDescriptionInfo.token;
          } else {
            token = eatTokensUntilEOL(token);
          }
          return token;

        case PRESERVE_TRY:
          if (!jsdocBuilder.recordPreserveTry()) {
            parser.addParserWarning("msg.jsdoc.preservertry",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case NO_SHADOW:
          if (!jsdocBuilder.recordNoShadow()) {
            parser.addParserWarning("msg.jsdoc.noshadow",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case NO_SIDE_EFFECTS:
          if (!jsdocBuilder.recordNoSideEffects()) {
            parser.addParserWarning("msg.jsdoc.nosideeffects",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case MODIFIES:
          token = parseModifiesTag(next());
          return token;

        case IMPLICIT_CAST:
          if (!jsdocBuilder.recordImplicitCast()) {
            parser.addTypeWarning("msg.jsdoc.implicitcast",
                stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case SEE:
          if (jsdocBuilder.shouldParseDocumentation()) {
            ExtractionInfo referenceInfo = extractSingleLineBlock();
            String reference = referenceInfo.string;

            if (reference.length() == 0) {
              parser.addParserWarning("msg.jsdoc.seemissing",
                  stream.getLineno(), stream.getCharno());
            } else {
              jsdocBuilder.addReference(reference);
            }

            token = referenceInfo.token;
          } else {
            token = eatTokensUntilEOL(token);
          }
          return token;

        case STABLEIDGENERATOR:
          if (!jsdocBuilder.recordStableIdGenerator()) {
            parser.addParserWarning("msg.jsdoc.stableidgen",
              stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case SUPPRESS:
          token = parseSuppressTag(next());
          return token;

        case TEMPLATE: {
          ExtractionInfo templateInfo = extractSingleLineBlock();
          List<String> names = Lists.newArrayList(
              Splitter.on(',')
                  .trimResults()
                  .split(templateInfo.string));

          if (names.size() == 0 || names.get(0).length() == 0) {
            parser.addTypeWarning("msg.jsdoc.templatemissing",
                  stream.getLineno(), stream.getCharno());
          } else if (!jsdocBuilder.recordTemplateTypeNames(names)) {
            parser.addTypeWarning("msg.jsdoc.template.at.most.once",
                stream.getLineno(), stream.getCharno());
          }

          token = templateInfo.token;
          return token;
        }

        case CLASS_TEMPLATE: {
          ExtractionInfo classTemplateInfo = extractSingleLineBlock();
          List<String> names = Lists.newArrayList(
              Splitter.on(',')
                  .trimResults()
                  .split(classTemplateInfo.string));

          if (names.size() == 0 || names.get(0).length() == 0) {
            parser.addTypeWarning(
                "msg.jsdoc.classtemplate.missing.type.name",
                stream.getLineno(), stream.getCharno());
          } else if (!jsdocBuilder.recordClassTemplateTypeNames(names)) {
            parser.addTypeWarning(
                "msg.jsdoc.classtemplate.at.most.once",
                stream.getLineno(), stream.getCharno());
          }

          token = classTemplateInfo.token;
          return token;
        }

        case IDGENERATOR:
          if (!jsdocBuilder.recordIdGenerator()) {
            parser.addParserWarning("msg.jsdoc.idgen",
              stream.getLineno(), stream.getCharno());
          }
          return eatTokensUntilEOL();

        case VERSION:
          ExtractionInfo versionInfo = extractSingleLineBlock();
          String version = versionInfo.string;

          if (version.length() == 0) {
            parser.addParserWarning("msg.jsdoc.versionmissing",
                  stream.getLineno(), stream.getCharno());
          } else {
            if (!jsdocBuilder.recordVersion(version)) {
               parser.addParserWarning("msg.jsdoc.extraversion",
                  stream.getLineno(), stream.getCharno());
            }
          }

          token = versionInfo.token;
          return token;

        case CONSTANT:
        case DEFINE:
        case RETURN:
        case PRIVATE:
        case PROTECTED:
        case PUBLIC:
        case THIS:
        case TYPE:
        case TYPEDEF:
          lineno = stream.getLineno();
          charno = stream.getCharno();

          Node typeNode = null;
          boolean hasType = lookAheadForTypeAnnotation();
          boolean isAlternateTypeAnnotation =
              (annotation == Annotation.PRIVATE ||
               annotation == Annotation.PROTECTED ||
               annotation == Annotation.PUBLIC ||
               annotation == Annotation.CONSTANT);
          boolean canSkipTypeAnnotation =
              (isAlternateTypeAnnotation ||
               annotation == Annotation.RETURN);
          type = null;
          if (hasType || !canSkipTypeAnnotation) {
            skipEOLs();
            token = next();
            typeNode = parseAndRecordTypeNode(token);

            if (annotation == Annotation.THIS) {
              typeNode = wrapNode(Token.BANG, typeNode);
            }
            type = createJSTypeExpression(typeNode);
          }

          // The error was reported during recursive descent
          // recovering parsing
          boolean hasError = type == null && !canSkipTypeAnnotation;
          if (!hasError) {
            // Record types for @type.
            // If the @private, @protected, or @public annotations
            // have a type attached, pretend that they actually wrote:
            // @type {type}\n@private
            // This will have some weird behavior in some cases
            // (for example, @private can now be used as a type-cast),
            // but should be mostly OK.
            if ((type != null && isAlternateTypeAnnotation)
                || annotation == Annotation.TYPE) {
              if (!jsdocBuilder.recordType(type)) {
                parser.addTypeWarning(
                    "msg.jsdoc.incompat.type", lineno, charno);
              }
            }

            switch (annotation) {
              case CONSTANT:
                if (!jsdocBuilder.recordConstancy()) {
                  parser.addParserWarning("msg.jsdoc.const",
                      stream.getLineno(), stream.getCharno());
                }
                break;

              case DEFINE:
                if (!jsdocBuilder.recordDefineType(type)) {
                  parser.addParserWarning("msg.jsdoc.define",
                      lineno, charno);
                }
                break;

              case PRIVATE:
                if (!jsdocBuilder.recordVisibility(Visibility.PRIVATE)) {
                  parser.addParserWarning(
                      "msg.jsdoc.visibility.private",
                      lineno, charno);
                }
                break;

              case PROTECTED:
                if (!jsdocBuilder.recordVisibility(Visibility.PROTECTED)) {
                  parser.addParserWarning(
                      "msg.jsdoc.visibility.protected",
                      lineno, charno);
                }
                break;

              case PUBLIC:
                if (!jsdocBuilder.recordVisibility(Visibility.PUBLIC)) {
                  parser.addParserWarning(
                      "msg.jsdoc.visibility.public",
                      lineno, charno);
                }
                break;

              case RETURN:
                if (type == null) {
                  type = createJSTypeExpression(newNode(Token.QMARK));
                }

                if (!jsdocBuilder.recordReturnType(type)) {
                  parser.addTypeWarning(
                      "msg.jsdoc.incompat.type", lineno, charno);
                  break;
                }

                // Find the return's description (if applicable).
                if (jsdocBuilder.shouldParseDocumentation()) {
                  ExtractionInfo returnDescriptionInfo =
                      extractMultilineTextualBlock(token);

                  String returnDescription =
                      returnDescriptionInfo.string;

                  if (returnDescription.length() > 0) {
                    jsdocBuilder.recordReturnDescription(
                        returnDescription);
                  }

                  token = returnDescriptionInfo.token;
                } else {
                  token = eatTokensUntilEOL(token);
                }
                return token;

              case THIS:
                if (!jsdocBuilder.recordThisType(type)) {
                  parser.addTypeWarning(
                      "msg.jsdoc.incompat.type", lineno, charno);
                }
                break;

              case TYPEDEF:
                if (!jsdocBuilder.recordTypedef(type)) {
                  parser.addTypeWarning(
                      "msg.jsdoc.incompat.type", lineno, charno);
                }
                break;
            }
          }

          return eatTokensUntilEOL();
      }
    }

    return next();
  }

  private void checkExtendedTypes(List<ExtendedTypeInfo> extendedTypes) {
    for (ExtendedTypeInfo typeInfo : extendedTypes) {
      // If interface, record the multiple extended interfaces
      if (jsdocBuilder.isInterfaceRecorded()) {
        if (!jsdocBuilder.recordExtendedInterface(typeInfo.type)) {
          parser.addParserWarning("msg.jsdoc.extends.duplicate",
              typeInfo.lineno, typeInfo.charno);
        }
      } else {
        if (!jsdocBuilder.recordBaseType(typeInfo.type)) {
          parser.addTypeWarning("msg.jsdoc.incompat.type",
              typeInfo.lineno, typeInfo.charno);
        }
      }
    }
  }

  /**
   * Parse a {@code @suppress} tag of the form
   * {@code @suppress&#123;warning1|warning2&#125;}.
   *
   * @param token The current token.
   */
  private JsDocToken parseSuppressTag(JsDocToken token) {
    if (token == JsDocToken.LC) {
      Set<String> suppressions = new HashSet<String>();
      while (true) {
        if (match(JsDocToken.STRING)) {
          String name = stream.getString();
          if (!suppressionNames.contains(name)) {
            parser.addParserWarning("msg.jsdoc.suppress.unknown", name,
                stream.getLineno(), stream.getCharno());
          }

          suppressions.add(stream.getString());
          token = next();
        } else {
          parser.addParserWarning("msg.jsdoc.suppress",
              stream.getLineno(), stream.getCharno());
          return token;
        }

        if (match(JsDocToken.PIPE, JsDocToken.COMMA)) {
          token = next();
        } else {
          break;
        }
      }

      if (!match(JsDocToken.RC)) {
        parser.addParserWarning("msg.jsdoc.suppress",
            stream.getLineno(), stream.getCharno());
      } else {
        token = next();
        if (!jsdocBuilder.recordSuppressions(suppressions)) {
          parser.addParserWarning("msg.jsdoc.suppress.duplicate",
              stream.getLineno(), stream.getCharno());
        }
      }
    }
    return token;
  }

  /**
   * Parse a {@code @modifies} tag of the form
   * {@code @modifies&#123;this|arguments|param&#125;}.
   *
   * @param token The current token.
   */
  private JsDocToken parseModifiesTag(JsDocToken token) {
    if (token == JsDocToken.LC) {
      Set<String> modifies = new HashSet<String>();
      while (true) {
        if (match(JsDocToken.STRING)) {
          String name = stream.getString();
          if (!modifiesAnnotationKeywords.contains(name)
              && !jsdocBuilder.hasParameter(name)) {
              parser.addParserWarning("msg.jsdoc.modifies.unknown", name,
                  stream.getLineno(), stream.getCharno());
          }

          modifies.add(stream.getString());
          token = next();
        } else {
          parser.addParserWarning("msg.jsdoc.modifies",
              stream.getLineno(), stream.getCharno());
          return token;
        }

        if (match(JsDocToken.PIPE)) {
          token = next();
        } else {
          break;
        }
      }

      if (!match(JsDocToken.RC)) {
        parser.addParserWarning("msg.jsdoc.modifies",
            stream.getLineno(), stream.getCharno());
      } else {
        token = next();
        if (!jsdocBuilder.recordModifies(modifies)) {
          parser.addParserWarning("msg.jsdoc.modifies.duplicate",
              stream.getLineno(), stream.getCharno());
        }
      }
    }
    return token;
  }

  /**
   * Looks for a type expression at the current token and if found,
   * returns it. Note that this method consumes input.
   *
   * @param token The current token.
   * @return The type expression found or null if none.
   */
  Node parseAndRecordTypeNode(JsDocToken token) {
    return parseAndRecordTypeNode(token, token == JsDocToken.LC);
  }

  /**
   * Looks for a type expression at the current token and if found,
   * returns it. Note that this method consumes input.
   *
   * @param token The current token.
   * @param matchingLC Whether the type expression starts with a "{".
   * @return The type expression found or null if none.
   */
  private Node parseAndRecordTypeNode(JsDocToken token, boolean matchingLC) {
    return parseAndRecordTypeNode(token, stream.getLineno(), stream.getCharno(),
        matchingLC, false);
  }

  /**
   * Looks for a type expression at the current token and if found,
   * returns it. Note that this method consumes input.
   *
   * @param token The current token.
   * @param lineno The line of the type expression.
   * @param startCharno The starting character position of the type expression.
   * @param matchingLC Whether the type expression starts with a "{".
   * @return The type expression found or null if none.
   */
  private Node parseAndRecordTypeNameNode(JsDocToken token, int lineno,
                                          int startCharno, boolean matchingLC) {
    return parseAndRecordTypeNode(token, lineno, startCharno, matchingLC, true);
  }

  /**
   * Looks for a type expression at the current token and if found,
   * returns it. Note that this method consumes input.
   *
   * Parameter type expressions are special for two reasons:
   * <ol>
   *   <li>They must begin with '{', to distinguish type names from param names.
   *   <li>They may end in '=', to denote optionality.
   * </ol>
   *
   * @param token The current token.
   * @return The type expression found or null if none.
   */
  private Node parseAndRecordParamTypeNode(JsDocToken token) {
    Preconditions.checkArgument(token == JsDocToken.LC);
    int lineno = stream.getLineno();
    int startCharno = stream.getCharno();

    Node typeNode = parseParamTypeExpressionAnnotation(token);
    if (typeNode != null) {
      int endLineno = stream.getLineno();
      int endCharno = stream.getCharno();

      jsdocBuilder.markTypeNode(typeNode, lineno, startCharno,
          endLineno, endCharno, true);
    }
    return typeNode;
  }

  /**
   * Looks for a parameter type expression at the current token and if found,
   * returns it. Note that this method consumes input.
   *
   * @param token The current token.
   * @param lineno The line of the type expression.
   * @param startCharno The starting character position of the type expression.
   * @param matchingLC Whether the type expression starts with a "{".
   * @param onlyParseSimpleNames If true, only simple type names are parsed
   *     (via a call to parseTypeNameAnnotation instead of
   *     parseTypeExpressionAnnotation).
   * @return The type expression found or null if none.
   */
  private Node parseAndRecordTypeNode(JsDocToken token, int lineno,
                                      int startCharno,
                                      boolean matchingLC,
                                      boolean onlyParseSimpleNames) {
    Node typeNode = null;

    if (onlyParseSimpleNames) {
      typeNode = parseTypeNameAnnotation(token);
    } else {
      typeNode = parseTypeExpressionAnnotation(token);
    }

    if (typeNode != null) {
      int endLineno = stream.getLineno();
      int endCharno = stream.getCharno();

      jsdocBuilder.markTypeNode(
          typeNode, lineno, startCharno, endLineno, endCharno, matchingLC);
    }

    return typeNode;
  }

  /**
   * Converts a JSDoc token to its string representation.
   */
  private String toString(JsDocToken token) {
    switch (token) {
      case ANNOTATION:
        return "@" + stream.getString();

      case BANG:
        return "!";

      case COMMA:
        return ",";

      case COLON:
        return ":";

      case GT:
        return ">";

      case LB:
        return "[";

      case LC:
        return "{";

      case LP:
        return "(";

      case LT:
        return ".<";

      case QMARK:
        return "?";

      case PIPE:
        return "|";

      case RB:
        return "]";

      case RC:
        return "}";

      case RP:
        return ")";

      case STAR:
        return "*";

      case ELLIPSIS:
        return "...";

      case EQUALS:
        return "=";

      case STRING:
        return stream.getString();

      default:
        throw new IllegalStateException(token.toString());
    }
  }

  /**
   * Constructs a new {@code JSTypeExpression}.
   * @param n A node. May be null.
   */
  JSTypeExpression createJSTypeExpression(Node n) {
    return n == null ? null :
        new JSTypeExpression(n, getSourceName());
  }

  /**
   * Tuple for returning both the string extracted and the
   * new token following a call to any of the extract*Block
   * methods.
   */
  private static class ExtractionInfo {
    private final String string;
    private final JsDocToken token;

    public ExtractionInfo(String string, JsDocToken token) {
      this.string = string;
      this.token = token;
    }
  }

  /**
   * Tuple for recording extended types
   */
  private static class ExtendedTypeInfo {
    final JSTypeExpression type;
    final int lineno;
    final int charno;

    public ExtendedTypeInfo(JSTypeExpression type, int lineno, int charno) {
      this.type = type;
      this.lineno = lineno;
      this.charno = charno;
    }
  }

  /**
   * Extracts the text found on the current line starting at token. Note that
   * token = token.info; should be called after this method is used to update
   * the token properly in the parser.
   *
   * @return The extraction information.
   */
  private ExtractionInfo extractSingleLineBlock() {

    // Get the current starting point.
    stream.update();
    int lineno = stream.getLineno();
    int charno = stream.getCharno() + 1;

    String line = getRemainingJSDocLine().trim();

    // Record the textual description.
    if (line.length() > 0) {
      jsdocBuilder.markText(line, lineno, charno, lineno,
                            charno + line.length());
    }

    return new ExtractionInfo(line, next());
  }

  private ExtractionInfo extractMultilineTextualBlock(JsDocToken token) {
    return extractMultilineTextualBlock(token, WhitespaceOption.SINGLE_LINE);
  }

  private enum WhitespaceOption {
    /**
     * Preserves all whitespace and formatting. Needed for licenses and
     * purposely formatted text.
     */
    PRESERVE,

    /** Preserves newlines but trims the output. */
    TRIM,

    /** Removes newlines and turns the output into a single line string. */
    SINGLE_LINE
  }

  /**
   * Extracts the text found on the current line and all subsequent
   * until either an annotation, end of comment or end of file is reached.
   * Note that if this method detects an end of line as the first token, it
   * will quit immediately (indicating that there is no text where it was
   * expected).  Note that token = info.token; should be called after this
   * method is used to update the token properly in the parser.
   *
   * @param token The start token.
   * @param option How to handle whitespace.
   *
   * @return The extraction information.
   */
  @SuppressWarnings("fallthrough")
  private ExtractionInfo extractMultilineTextualBlock(JsDocToken token,
                                                      WhitespaceOption option) {

    if (token == JsDocToken.EOC || token == JsDocToken.EOL ||
        token == JsDocToken.EOF) {
      return new ExtractionInfo("", token);
    }

    stream.update();
    int startLineno = stream.getLineno();
    int startCharno = stream.getCharno() + 1;

    // Read the content from the first line.
    String line = getRemainingJSDocLine();
    if (option != WhitespaceOption.PRESERVE) {
      line = line.trim();
    }

    StringBuilder builder = new StringBuilder();
    builder.append(line);

    state = State.SEARCHING_ANNOTATION;
    token = next();

    boolean ignoreStar = false;

    // Track the start of the line to count whitespace that
    // the tokenizer skipped. Because this case is rare, it's easier
    // to do this here than in the tokenizer.
    int lineStartChar = -1;

    do {
      switch (token) {
        case STAR:
          if (ignoreStar) {
            // Mark the position after the star as the new start of the line.
            lineStartChar = stream.getCharno() + 1;
          } else {
            // The star is part of the comment.
            if (builder.length() > 0) {
              builder.append(' ');
            }

            builder.append('*');
          }

          token = next();
          continue;

        case EOL:
          if (option != WhitespaceOption.SINGLE_LINE) {
            builder.append("\n");
          }

          ignoreStar = true;
          lineStartChar = 0;
          token = next();
          continue;

        default:
          ignoreStar = false;
          state = State.SEARCHING_ANNOTATION;

          boolean isEOC = token == JsDocToken.EOC;
          if (!isEOC) {
            if (lineStartChar != -1 && option == WhitespaceOption.PRESERVE) {
              int numSpaces = stream.getCharno() - lineStartChar;
              for (int i = 0; i < numSpaces; i++) {
                builder.append(' ');
              }
              lineStartChar = -1;
            } else if (builder.length() > 0) {
              // All tokens must be separated by a space.
              builder.append(' ');
            }
          }

          if (token == JsDocToken.EOC ||
              token == JsDocToken.EOF ||
              // When we're capturing a license block, annotations
              // in the block are OK.
              (token == JsDocToken.ANNOTATION &&
               option != WhitespaceOption.PRESERVE)) {
            String multilineText = builder.toString();

            if (option != WhitespaceOption.PRESERVE) {
              multilineText = multilineText.trim();
            }

            int endLineno = stream.getLineno();
            int endCharno = stream.getCharno();

            if (multilineText.length() > 0) {
              jsdocBuilder.markText(multilineText, startLineno, startCharno,
                  endLineno, endCharno);
            }

            return new ExtractionInfo(multilineText, token);
          }

          builder.append(toString(token));

          line = getRemainingJSDocLine();

          if (option != WhitespaceOption.PRESERVE) {
            line = trimEnd(line);
          }

          builder.append(line);
          token = next();
      }
    } while (true);
  }


  /**
   * Extracts the top-level block comment from the JsDoc comment, if any.
   * This method differs from the extractMultilineTextualBlock in that it
   * terminates under different conditions (it doesn't have the same
   * prechecks), it does not first read in the remaining of the current
   * line and its conditions for ignoring the "*" (STAR) are different.
   *
   * @param token The starting token.
   *
   * @return The extraction information.
   */
  private ExtractionInfo extractBlockComment(JsDocToken token) {
    StringBuilder builder = new StringBuilder();

    boolean ignoreStar = true;

    do {
      switch (token) {
        case ANNOTATION:
        case EOC:
        case EOF:
          return new ExtractionInfo(builder.toString().trim(), token);

        case STAR:
          if (!ignoreStar) {
            if (builder.length() > 0) {
              builder.append(' ');
            }

            builder.append('*');
          }

          token = next();
          continue;

        case EOL:
          ignoreStar = true;
          builder.append('\n');
          token = next();
          continue;

        default:
          if (!ignoreStar && builder.length() > 0) {
            builder.append(' ');
          }

          ignoreStar = false;

          builder.append(toString(token));

          String line = getRemainingJSDocLine();
          line = trimEnd(line);
          builder.append(line);
          token = next();
      }
    } while (true);
  }

  /**
   * Trim characters from only the end of a string.
   * This method will remove all whitespace characters
   * (defined by Character.isWhitespace(char), in addition to the characters
   * provided, from the end of the provided string.
   *
   * @param s String to be trimmed
   * @return String with whitespace and characters in extraChars removed
   *                   from the end.
   */
  private static String trimEnd(String s) {
    int trimCount = 0;
    while (trimCount < s.length()) {
      char ch = s.charAt(s.length() - trimCount - 1);
      if (Character.isWhitespace(ch)) {
        trimCount++;
      } else {
        break;
      }
    }

    if (trimCount == 0) {
      return s;
    }
    return s.substring(0, s.length() - trimCount);
  }

  // Based on ES4 grammar proposed on July 10, 2008.
  // http://wiki.ecmascript.org/doku.php?id=spec:spec
  // Deliberately written to line up with the actual grammar rules,
  // for maximum flexibility.

  // TODO(nicksantos): The current implementation tries to maintain backwards
  // compatibility with previous versions of the spec whenever we can.
  // We should try to gradually withdraw support for these.

  /**
   * TypeExpressionAnnotation := TypeExpression |
   *     '{' TopLevelTypeExpression '}'
   */
  private Node parseTypeExpressionAnnotation(JsDocToken token) {
    if (token == JsDocToken.LC) {
      skipEOLs();
      Node typeNode = parseTopLevelTypeExpression(next());
      if (typeNode != null) {
        skipEOLs();
        if (!match(JsDocToken.RC)) {
          reportTypeSyntaxWarning("msg.jsdoc.missing.rc");
        } else {
          next();
        }
      }

      return typeNode;
    } else {
      return parseTypeExpression(token);
    }
  }

  /**
   * ParamTypeExpressionAnnotation :=
   *     '{' OptionalParameterType '}' |
   *     '{' TopLevelTypeExpression '}' |
   *     '{' '...' TopLevelTypeExpression '}'
   *
   * OptionalParameterType :=
   *     TopLevelTypeExpression '='
   */
  private Node parseParamTypeExpressionAnnotation(JsDocToken token) {
    Preconditions.checkArgument(token == JsDocToken.LC);

    skipEOLs();

    boolean restArg = false;
    token = next();
    if (token == JsDocToken.ELLIPSIS) {
      token = next();
      if (token == JsDocToken.RC) {
        // EMPTY represents the UNKNOWN type in the Type AST.
        return wrapNode(Token.ELLIPSIS, IR.empty());
      }
      restArg = true;
    }

    Node typeNode = parseTopLevelTypeExpression(token);
    if (typeNode != null) {
      skipEOLs();
      if (restArg) {
        typeNode = wrapNode(Token.ELLIPSIS, typeNode);
      } else if (match(JsDocToken.EQUALS)) {
        next();
        skipEOLs();
        typeNode = wrapNode(Token.EQUALS, typeNode);
      }

      if (!match(JsDocToken.RC)) {
        reportTypeSyntaxWarning("msg.jsdoc.missing.rc");
      } else {
        next();
      }
    }

    return typeNode;
  }

  /**
   * TypeNameAnnotation := TypeName | '{' TypeName '}'
   */
  private Node parseTypeNameAnnotation(JsDocToken token) {
    if (token == JsDocToken.LC) {
      skipEOLs();
      Node typeNode = parseTypeName(next());
      if (typeNode != null) {
        skipEOLs();
        if (!match(JsDocToken.RC)) {
          reportTypeSyntaxWarning("msg.jsdoc.missing.rc");
        } else {
          next();
        }
      }

      return typeNode;
    } else {
      return parseTypeName(token);
    }
  }

  /**
   * TopLevelTypeExpression := TypeExpression
   *     | TypeUnionList
   *
   * We made this rule up, for the sake of backwards compatibility.
   */
  private Node parseTopLevelTypeExpression(JsDocToken token) {
    Node typeExpr = parseTypeExpression(token);
    if (typeExpr != null) {
      // top-level unions are allowed
      if (match(JsDocToken.PIPE)) {
        next();
        if (match(JsDocToken.PIPE)) {
          // We support double pipes for backwards-compatibility.
          next();
        }
        skipEOLs();
        token = next();
        return parseUnionTypeWithAlternate(token, typeExpr);
      }
    }
    return typeExpr;
  }

  /**
   * TypeExpressionList := TopLevelTypeExpression
   *     | TopLevelTypeExpression ',' TypeExpressionList
   */
  private Node parseTypeExpressionList(JsDocToken token) {
    Node typeExpr = parseTopLevelTypeExpression(token);
    if (typeExpr == null) {
      return null;
    }
    Node typeList = IR.block();
    typeList.addChildToBack(typeExpr);
    while (match(JsDocToken.COMMA)) {
      next();
      skipEOLs();
      typeExpr = parseTopLevelTypeExpression(next());
      if (typeExpr == null) {
        return null;
      }
      typeList.addChildToBack(typeExpr);
    }
    return typeList;
  }

  /**
   * TypeExpression := BasicTypeExpression
   *     | '?' BasicTypeExpression
   *     | '!' BasicTypeExpression
   *     | BasicTypeExpression '?'
   *     | BasicTypeExpression '!'
   *     | '?'
   */
  private Node parseTypeExpression(JsDocToken token) {
    if (token == JsDocToken.QMARK) {
      // A QMARK could mean that a type is nullable, or that it's unknown.
      // We use look-ahead 1 to determine whether it's unknown. Otherwise,
      // we assume it means nullable. There are 5 cases:
      // {?} - right curly
      // {?=} - equals
      // {function(?, number)} - comma
      // {function(number, ?)} - right paren
      // {function(number, ...[?])} - right bracket
      // {function(): ?|number} - pipe
      // {Array.<?>} - greater than
      // I'm not a big fan of using look-ahead for this, but it makes
      // the type language a lot nicer.
      token = next();
      if (token == JsDocToken.COMMA ||
          token == JsDocToken.EQUALS ||
          token == JsDocToken.RB ||
          token == JsDocToken.RC ||
          token == JsDocToken.RP ||
          token == JsDocToken.PIPE ||
          token == JsDocToken.GT) {
        restoreLookAhead(token);
        return newNode(Token.QMARK);
      }

      return wrapNode(Token.QMARK, parseBasicTypeExpression(token));
    } else if (token == JsDocToken.BANG) {
      return wrapNode(Token.BANG, parseBasicTypeExpression(next()));
    } else {
      Node basicTypeExpr = parseBasicTypeExpression(token);
      if (basicTypeExpr != null) {
        if (match(JsDocToken.QMARK)) {
          next();
          return wrapNode(Token.QMARK, basicTypeExpr);
        } else if (match(JsDocToken.BANG)) {
          next();
          return wrapNode(Token.BANG, basicTypeExpr);
        }
      }

      return basicTypeExpr;
    }
  }

  /**
   * BasicTypeExpression := '*' | 'null' | 'undefined' | TypeName
   *     | FunctionType | UnionType | RecordType | ArrayType
   */
  private Node parseBasicTypeExpression(JsDocToken token) {
    if (token == JsDocToken.STAR) {
      return newNode(Token.STAR);
    } else if (token == JsDocToken.LB) {
      skipEOLs();
      return parseArrayType(next());
    } else if (token == JsDocToken.LC) {
      skipEOLs();
      return parseRecordType(next());
    } else if (token == JsDocToken.LP) {
      skipEOLs();
      return parseUnionType(next());
    } else if (token == JsDocToken.STRING) {
      String string = stream.getString();
      if ("function".equals(string)) {
        skipEOLs();
        return parseFunctionType(next());
      } else if ("null".equals(string) || "undefined".equals(string)) {
        return newStringNode(string);
      } else {
        return parseTypeName(token);
      }
    }

    restoreLookAhead(token);
    return reportGenericTypeSyntaxWarning();
  }

  /**
   * TypeName := NameExpression | NameExpression TypeApplication
   * TypeApplication := '.<' TypeExpressionList '>'
   */
  private Node parseTypeName(JsDocToken token) {
    if (token != JsDocToken.STRING) {
      return reportGenericTypeSyntaxWarning();
    }

    String typeName = stream.getString();
    int lineno = stream.getLineno();
    int charno = stream.getCharno();
    while (match(JsDocToken.EOL) &&
        typeName.charAt(typeName.length() - 1) == '.') {
      skipEOLs();
      if (match(JsDocToken.STRING)) {
        next();
        typeName += stream.getString();
      }
    }

    Node typeNameNode = newStringNode(typeName, lineno, charno);

    if (match(JsDocToken.LT)) {
      next();
      skipEOLs();
      Node memberType = parseTypeExpressionList(next());
      if (memberType != null) {
        typeNameNode.addChildToFront(memberType);

        skipEOLs();
        if (!match(JsDocToken.GT)) {
          return reportTypeSyntaxWarning("msg.jsdoc.missing.gt");
        }

        next();
      }
    }
    return typeNameNode;
  }

  /**
   * FunctionType := 'function' FunctionSignatureType
   * FunctionSignatureType :=
   *    TypeParameters '(' 'this' ':' TypeName, ParametersType ')' ResultType
   */
  private Node parseFunctionType(JsDocToken token) {
    // NOTE(nicksantos): We're not implementing generics at the moment, so
    // just throw out TypeParameters.
    if (token != JsDocToken.LP) {
      restoreLookAhead(token);
      return reportTypeSyntaxWarning("msg.jsdoc.missing.lp");
    }

    Node functionType = newNode(Token.FUNCTION);
    Node parameters = null;
    skipEOLs();
    if (!match(JsDocToken.RP)) {
      token = next();

      boolean hasParams = true;
      if (token == JsDocToken.STRING) {
        String tokenStr = stream.getString();
        boolean isThis = "this".equals(tokenStr);
        boolean isNew = "new".equals(tokenStr);
        if (isThis || isNew) {
          if (match(JsDocToken.COLON)) {
            next();
            skipEOLs();
            Node contextType = wrapNode(
                isThis ? Token.THIS : Token.NEW,
                parseTypeName(next()));
            if (contextType == null) {
              return null;
            }

            functionType.addChildToFront(contextType);
          } else {
            return reportTypeSyntaxWarning("msg.jsdoc.missing.colon");
          }

          if (match(JsDocToken.COMMA)) {
            next();
            skipEOLs();
            token = next();
          } else {
            hasParams = false;
          }
        }
      }

      if (hasParams) {
        parameters = parseParametersType(token);
        if (parameters == null) {
          return null;
        }
      }
    }

    if (parameters != null) {
      functionType.addChildToBack(parameters);
    }

    skipEOLs();
    if (!match(JsDocToken.RP)) {
      return reportTypeSyntaxWarning("msg.jsdoc.missing.rp");
    }

    skipEOLs();
    Node resultType = parseResultType(next());
    if (resultType == null) {
      return null;
    } else {
      functionType.addChildToBack(resultType);
    }
    return functionType;
  }

  /**
   * ParametersType := RestParameterType | NonRestParametersType
   *     | NonRestParametersType ',' RestParameterType
   * RestParameterType := '...' Identifier
   * NonRestParametersType := ParameterType ',' NonRestParametersType
   *     | ParameterType
   *     | OptionalParametersType
   * OptionalParametersType := OptionalParameterType
   *     | OptionalParameterType, OptionalParametersType
   * OptionalParameterType := ParameterType=
   * ParameterType := TypeExpression | Identifier ':' TypeExpression
   */
  // NOTE(nicksantos): The official ES4 grammar forces optional and rest
  // arguments to come after the required arguments. Our parser does not
  // enforce this. Instead we allow them anywhere in the function at parse-time,
  // and then warn about them during type resolution.
  //
  // In theory, it might be mathematically nicer to do the order-checking here.
  // But in practice, the order-checking for structural functions is exactly
  // the same as the order-checking for @param annotations. And the latter
  // has to happen during type resolution. Rather than duplicate the
  // order-checking in two places, we just do all of it in type resolution.
  private Node parseParametersType(JsDocToken token) {
    Node paramsType = newNode(Token.PARAM_LIST);
    boolean isVarArgs = false;
    Node paramType = null;
    if (token != JsDocToken.RP) {
      do {
        if (paramType != null) {
          // skip past the comma
          next();
          skipEOLs();
          token = next();
        }

        if (token == JsDocToken.ELLIPSIS) {
          // In the latest ES4 proposal, there are no type constraints allowed
          // on variable arguments. We support the old syntax for backwards
          // compatibility, but we should gradually tear it out.
          skipEOLs();
          if (match(JsDocToken.RP)) {
            paramType = newNode(Token.ELLIPSIS);
          } else {
            skipEOLs();
            if (!match(JsDocToken.LB)) {
              return reportTypeSyntaxWarning("msg.jsdoc.missing.lb");
            }

            next();
            skipEOLs();
            paramType = wrapNode(Token.ELLIPSIS, parseTypeExpression(next()));
            skipEOLs();
            if (!match(JsDocToken.RB)) {
              return reportTypeSyntaxWarning("msg.jsdoc.missing.rb");
            }
            skipEOLs();
            next();
          }

          isVarArgs = true;
        } else {
          paramType = parseTypeExpression(token);
          if (match(JsDocToken.EQUALS)) {
            skipEOLs();
            next();
            paramType = wrapNode(Token.EQUALS, paramType);
          }
        }

        if (paramType == null) {
          return null;
        }
        paramsType.addChildToBack(paramType);
        if (isVarArgs) {
          break;
        }
      } while (match(JsDocToken.COMMA));
    }

    if (isVarArgs && match(JsDocToken.COMMA)) {
      return reportTypeSyntaxWarning("msg.jsdoc.function.varargs");
    }

    // The right paren will be checked by parseFunctionType

    return paramsType;
  }

  /**
   * ResultType := <empty> | ':' void | ':' TypeExpression
   */
  private Node parseResultType(JsDocToken token) {
    skipEOLs();
    if (!match(JsDocToken.COLON)) {
      return newNode(Token.EMPTY);
    }

    token = next();
    skipEOLs();
    if (match(JsDocToken.STRING) && "void".equals(stream.getString())) {
      next();
      return newNode(Token.VOID);
    } else {
      return parseTypeExpression(next());
    }
  }

  /**
   * UnionType := '(' TypeUnionList ')'
   * TypeUnionList := TypeExpression | TypeExpression '|' TypeUnionList
   *
   * We've removed the empty union type.
   */
  private Node parseUnionType(JsDocToken token) {
    return parseUnionTypeWithAlternate(token, null);
  }

  /**
   * Create a new union type, with an alternate that has already been
   * parsed. The alternate may be null.
   */
  private Node parseUnionTypeWithAlternate(JsDocToken token, Node alternate) {
    Node union = newNode(Token.PIPE);
    if (alternate != null) {
      union.addChildToBack(alternate);
    }

    Node expr = null;
    do {
      if (expr != null) {
        skipEOLs();
        token = next();
        Preconditions.checkState(
            token == JsDocToken.PIPE || token == JsDocToken.COMMA);

        boolean isPipe = token == JsDocToken.PIPE;
        if (isPipe && match(JsDocToken.PIPE)) {
          // We support double pipes for backwards compatibility.
          next();
        }
        skipEOLs();
        token = next();
      }
      expr = parseTypeExpression(token);
      if (expr == null) {
        return null;
      }

      union.addChildToBack(expr);
      // We support commas for backwards compatibility.
    } while (match(JsDocToken.PIPE, JsDocToken.COMMA));

    if (alternate == null) {
      skipEOLs();
      if (!match(JsDocToken.RP)) {
        return reportTypeSyntaxWarning("msg.jsdoc.missing.rp");
      }
      next();
    }
    return union;
  }

  /**
   * ArrayType := '[' ElementTypeList ']'
   * ElementTypeList := <empty> | TypeExpression | '...' TypeExpression
   *     | TypeExpression ',' ElementTypeList
   */
  private Node parseArrayType(JsDocToken token) {
    Node array = newNode(Token.LB);
    Node arg = null;
    boolean hasVarArgs = false;

    do {
      if (arg != null) {
        next();
        skipEOLs();
        token = next();
      }
      if (token == JsDocToken.ELLIPSIS) {
        arg = wrapNode(Token.ELLIPSIS, parseTypeExpression(next()));
        hasVarArgs = true;
      } else {
        arg = parseTypeExpression(token);
      }

      if (arg == null) {
        return null;
      }

      array.addChildToBack(arg);
      if (hasVarArgs) {
        break;
      }
      skipEOLs();
    } while (match(JsDocToken.COMMA));

    if (!match(JsDocToken.RB)) {
      return reportTypeSyntaxWarning("msg.jsdoc.missing.rb");
    }
    next();
    return array;
  }

  /**
   * RecordType := '{' FieldTypeList '}'
   */
  private Node parseRecordType(JsDocToken token) {
    Node recordType = newNode(Token.LC);
    Node fieldTypeList = parseFieldTypeList(token);

    if (fieldTypeList == null) {
      return reportGenericTypeSyntaxWarning();
    }

    skipEOLs();
    if (!match(JsDocToken.RC)) {
      return reportTypeSyntaxWarning("msg.jsdoc.missing.rc");
    }

    next();

    recordType.addChildToBack(fieldTypeList);
    return recordType;
  }

  /**
   * FieldTypeList := FieldType | FieldType ',' FieldTypeList
   */
  private Node parseFieldTypeList(JsDocToken token) {
    Node fieldTypeList = newNode(Token.LB);

    do {
      Node fieldType = parseFieldType(token);

      if (fieldType == null) {
        return null;
      }

      fieldTypeList.addChildToBack(fieldType);

      skipEOLs();
      if (!match(JsDocToken.COMMA)) {
        break;
      }

      // Move to the comma token.
      next();

      // Move to the token passed the comma.
      skipEOLs();
      token = next();
    } while (true);

    return fieldTypeList;
  }

  /**
   * FieldType := FieldName | FieldName ':' TypeExpression
   */
  private Node parseFieldType(JsDocToken token) {
    Node fieldName = parseFieldName(token);

    if (fieldName == null) {
      return null;
    }

    skipEOLs();
    if (!match(JsDocToken.COLON)) {
      return fieldName;
    }

    // Move to the colon.
    next();

    // Move to the token after the colon and parse
    // the type expression.
    skipEOLs();
    Node typeExpression = parseTypeExpression(next());

    if (typeExpression == null) {
      return null;
    }

    Node fieldType = newNode(Token.COLON);
    fieldType.addChildToBack(fieldName);
    fieldType.addChildToBack(typeExpression);
    return fieldType;
  }

  /**
   * FieldName := NameExpression | StringLiteral | NumberLiteral |
   * ReservedIdentifier
   */
  private Node parseFieldName(JsDocToken token) {
    switch (token) {
      case STRING:
        String string = stream.getString();
        return newStringNode(string);

      default:
        return null;
    }
  }

  private Node wrapNode(int type, Node n) {
    return n == null ? null :
        new Node(type, n, stream.getLineno(),
            stream.getCharno()).clonePropsFrom(templateNode);
  }

  private Node newNode(int type) {
    return new Node(type, stream.getLineno(),
        stream.getCharno()).clonePropsFrom(templateNode);
  }

  private Node newStringNode(String s) {
    return newStringNode(s, stream.getLineno(), stream.getCharno());
  }

  private Node newStringNode(String s, int lineno, int charno) {
    Node n = Node.newString(s, lineno, charno).clonePropsFrom(templateNode);
    n.setLength(s.length());
    return n;
  }

  // This is similar to IRFactory.createTemplateNode to share common props
  // e.g., source-name, between all nodes.
  private Node createTemplateNode() {
    // The Node type choice is arbitrary.
    Node templateNode = IR.script();
    templateNode.setStaticSourceFile(
      this.associatedNode != null ?
      this.associatedNode.getStaticSourceFile() :
      null);
    return templateNode;
  }

  private Node reportTypeSyntaxWarning(String warning) {
    parser.addTypeWarning(warning, stream.getLineno(), stream.getCharno());
    return null;
  }

  private Node reportGenericTypeSyntaxWarning() {
    return reportTypeSyntaxWarning("msg.jsdoc.type.syntax");
  }

  /**
   * Eats tokens until {@link JsDocToken#EOL} included, and switches back the
   * state to {@link State#SEARCHING_ANNOTATION}.
   */
  private JsDocToken eatTokensUntilEOL() {
    return eatTokensUntilEOL(next());
  }

  /**
   * Eats tokens until {@link JsDocToken#EOL} included, and switches back the
   * state to {@link State#SEARCHING_ANNOTATION}.
   */
  private JsDocToken eatTokensUntilEOL(JsDocToken token) {
    do {
      if (token == JsDocToken.EOL || token == JsDocToken.EOC ||
          token == JsDocToken.EOF) {
        state = State.SEARCHING_ANNOTATION;
        return token;
      }
      token = next();
    } while (true);
  }

  /**
   * Specific value indicating that the {@link #unreadToken} contains no token.
   */
  private static final JsDocToken NO_UNREAD_TOKEN = null;

  /**
   * One token buffer.
   */
  private JsDocToken unreadToken = NO_UNREAD_TOKEN;

  /** Restores the lookahead token to the token stream */
  private void restoreLookAhead(JsDocToken token) {
    unreadToken = token;
  }

  /**
   * Tests whether the next symbol of the token stream matches the specific
   * token.
   */
  private boolean match(JsDocToken token) {
    unreadToken = next();
    return unreadToken == token;
  }

  /**
   * Tests that the next symbol of the token stream matches one of the specified
   * tokens.
   */
  private boolean match(JsDocToken token1, JsDocToken token2) {
    unreadToken = next();
    return unreadToken == token1 || unreadToken == token2;
  }

  /**
   * Gets the next token of the token stream or the buffered token if a matching
   * was previously made.
   */
  private JsDocToken next() {
    if (unreadToken == NO_UNREAD_TOKEN) {
      return stream.getJsDocToken();
    } else {
      return current();
    }
  }

  /**
   * Gets the current token, invalidating it in the process.
   */
  private JsDocToken current() {
    JsDocToken t = unreadToken;
    unreadToken = NO_UNREAD_TOKEN;
    return t;
  }

  /**
   * Skips all EOLs and all empty lines in the JSDoc. Call this method if you
   * want the JSDoc entry to span multiple lines.
   */
  private void skipEOLs() {
    while (match(JsDocToken.EOL)) {
      next();
      if (match(JsDocToken.STAR)) {
        next();
      }
    }
  }

  /**
   * Returns the remainder of the line.
   */
  private String getRemainingJSDocLine() {
    String result = stream.getRemainingJSDocLine();
    unreadToken = NO_UNREAD_TOKEN;
    return result;
  }

  /**
   * Determines whether the parser has been populated with docinfo with a
   * fileoverview tag.
   */
  private boolean hasParsedFileOverviewDocInfo() {
    return jsdocBuilder.isPopulatedWithFileOverview();
  }

  boolean hasParsedJSDocInfo() {
    return jsdocBuilder.isPopulated();
  }

  JSDocInfo retrieveAndResetParsedJSDocInfo() {
    return jsdocBuilder.build(associatedNode);
  }

  /**
   * Gets the fileoverview JSDocInfo, if any.
   */
  JSDocInfo getFileOverviewJSDocInfo() {
    return fileOverviewJSDocInfo;
  }

  /**
   * Look ahead for a type annotation by advancing the character stream.
   * Does not modify the token stream.
   * This is kind of a hack, and is only necessary because we use the token
   * stream to parse types, but need the underlying character stream to get
   * JsDoc descriptions.
   * @return Whether we found a type annotation.
   */
  private boolean lookAheadForTypeAnnotation() {
    boolean matchedLc = false;
    int c;
    while (true) {
      c = stream.getChar();
      if (c == ' ') {
        continue;
      } else if (c == '{') {
        matchedLc = true;
        break;
      } else {
        break;
      }
    }
    stream.ungetChar(c);
    return matchedLc;
  }
}
