Field Detail

• ampersandPattern

  public static Pattern ampersandPattern

• ampersandRegex

  public static String ampersandRegex

• beginOrEndPattern

  public static Pattern beginOrEndPattern

• beginOrEndRegex

  public static String beginOrEndRegex

• bracesPattern

  public static Pattern bracesPattern

• bracesRegex

  public static String bracesRegex

• closedBracesRegex

  public static String closedBracesRegex

• colonPattern

  public static Pattern colonPattern

• colonRegex

  public static String colonRegex

• commaPattern

  public static Pattern commaPattern

• commaRegex

  public static String commaRegex

• dashPattern

  public static Pattern dashPattern

• dashRegex

  public static String dashRegex

• dollarSignPattern

  public static Pattern dollarSignPattern

• dollarSignRegex

  public static String dollarSignRegex

• doubleQuotePattern

  public static Pattern doubleQuotePattern

• doubleQuoteRegex

  public static String doubleQuoteRegex

• ellipsisPattern

  public static Pattern ellipsisPattern

• ellipsisRegex

  public static String ellipsisRegex

• extraSpacePattern

  public static Pattern extraSpacePattern

• extraSpaceRegex

  public static String extraSpaceRegex

• multipleWhitespacePattern

  public static Pattern multipleWhitespacePattern

• multipleWhitespaceRegex

  public static String multipleWhitespaceRegex

• nonFinalPunctPattern

  public static Pattern nonFinalPunctPattern

• nonFinalPunctRegex

  public static String nonFinalPunctRegex

• nonPeriodPunctPattern

  public static Pattern nonPeriodPunctPattern

• nonPeriodPunctRegex

  public static String nonPeriodPunctRegex

• oneWordAbbreviationPattern

  public static Pattern oneWordAbbreviationPattern

• oneWordAbbreviationRegex

  public static String oneWordAbbreviationRegex

• openBracesRegex

  public static String openBracesRegex

• patterns

  protected Pattern[] patterns

• periodPattern

  public static Pattern periodPattern

• periodRegex

  public static String periodRegex

• quotePattern

  public static Pattern quotePattern

• quoteRegex

  public static String quoteRegex

• singleQuotePattern

  public static Pattern singleQuotePattern

• singleQuoteRegex

  public static String singleQuoteRegex

• tAbbreviationPattern

  public static Pattern tAbbreviationPattern

• tAbbreviationRegex

  public static String tAbbreviationRegex

• threeWordAbbreviationPatterns

  public static Pattern[] threeWordAbbreviationPatterns

• threeWordAbbreviationRegexes

  public static String[] threeWordAbbreviationRegexes

• tripleQuotePattern

  public static Pattern tripleQuotePattern

• tripleQuoteRegex

  public static String tripleQuoteRegex

• twoWordAbbreviationPatterns

  public static Pattern[] twoWordAbbreviationPatterns

• twoWordAbbreviationRegexes

  public static String[] twoWordAbbreviationRegexes

Constructor Detail

• PennTreebankTokenizer

  public PennTreebankTokenizer()

Method Detail

• getTokenTexts

  public String[] getTokenTexts(String text)

  Tokenizes the input text and returns a string array corresponding to the tokens.

  Specified by:

  getTokenTexts in class Tokenizer_ImplBase