% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/unnest_tokens.R
\name{unnest_tokens}
\alias{unnest_tokens}
\title{Split a column into tokens}
\usage{
unnest_tokens(
  tbl,
  output,
  input,
  token = "words",
  format = c("text", "man", "latex", "html", "xml"),
  to_lower = TRUE,
  drop = TRUE,
  collapse = NULL,
  ...
)
}
\arguments{
\item{tbl}{A data frame}

\item{output}{Output column to be created as string or symbol.}

\item{input}{Input column that gets split as string or symbol.

The output/input arguments are passed by expression and support
\link[rlang:topic-inject]{quasiquotation}; you can unquote strings and symbols.}

\item{token}{Unit for tokenizing, or a custom tokenizing function. Built-in
options are "words" (default), "characters", "character_shingles", "ngrams",
"skip_ngrams", "sentences", "lines", "paragraphs", "regex", and
"ptb" (Penn Treebank). If a function, should take a character vector and
return a list of character vectors of the same length.}

\item{format}{Either "text", "man", "latex", "html", or "xml". When the
format is "text", this function uses the tokenizers package. If not "text",
this uses the hunspell tokenizer, and can tokenize only by "word".}

\item{to_lower}{Whether to convert tokens to lowercase.}

\item{drop}{Whether original input column should get dropped. Ignored
if the original input and new output column have the same name.}

\item{collapse}{A character vector of variables to collapse text across,
or \code{NULL}.

For tokens like n-grams or sentences, text can be collapsed across rows
within variables specified by \code{collapse} before tokenization. At tidytext
0.2.7, the default behavior for \code{collapse = NULL} changed to be more
consistent. The new behavior is that text is \emph{not} collapsed for \code{NULL}.

Grouping data specifies variables to collapse across in the same way as
\code{collapse} but you \strong{cannot} use both the \code{collapse} argument and
grouped data. Collapsing applies mostly to \code{token} options of "ngrams",
"skip_ngrams", "sentences", "lines", "paragraphs", or "regex".}

\item{...}{Extra arguments passed on to \link[tokenizers:tokenizers]{tokenizers}, such
as \code{strip_punct} for "words", \code{n} and \code{k} for "ngrams" and "skip_ngrams",
and \code{pattern} for "regex".}
}
\description{
Split a column into tokens, flattening the table into one-token-per-row.
This function supports non-standard evaluation through the tidyeval framework.
}
\details{
If format is anything other than "text", this uses the
\code{\link[hunspell:hunspell]{hunspell::hunspell_parse()}} tokenizer instead of the tokenizers package.
This does not yet have support for tokenizing by any unit other than words.

Support for \code{token = "tweets"} was removed in tidytext 0.4.0 because of
changes in upstream dependencies.
}
\examples{
\dontshow{if (rlang::is_installed("hunspell")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf}

library(dplyr)
library(janeaustenr)

d <- tibble(txt = prideprejudice)
d

d \%>\%
  unnest_tokens(output = word, input = txt)

d \%>\%
  unnest_tokens(output = sentence, input = txt, token = "sentences")

d \%>\%
  unnest_tokens(output = ngram, input = txt, token = "ngrams", n = 2)

d \%>\%
  unnest_tokens(chapter, txt, token = "regex", pattern = "Chapter [\\\\\\\\d]")

d \%>\%
  unnest_tokens(shingle, txt, token = "character_shingles", n = 4)

# custom function
d \%>\%
  unnest_tokens(word, txt, token = stringr::str_split, pattern = " ")

# tokenize HTML
h <- tibble(row = 1:2,
                text = c("<h1>Text <b>is</b>", "<a href='example.com'>here</a>"))

h \%>\%
  unnest_tokens(word, text, format = "html")
\dontshow{\}) # examplesIf}
}
