ISourceCodeTokenizer (Gosu :: Core API 1.14.21 API)

```
public interface ISourceCodeTokenizer
```

Field Summary

Fields
Modifier and Type	Field and Description
`static int`	`TT_COMMENT`
`static int`	`TT_EOF`
`static int`	`TT_EOL`
`static int`	`TT_INTEGER`
`static int`	`TT_KEYWORD`
`static int`	`TT_NOTHING`
`static int`	`TT_NUMBER`
`static int`	`TT_OPERATOR`
`static int`	`TT_WHITESPACE`
`static int`	`TT_WORD`

Method Summary

All Methods Instance Methods Abstract Methods
Modifier and Type	Method and Description
`void`	`commentChar(int ch)`
`IToken`	`copy()`
`IToken`	`copyInto(IToken t)`
`int`	`countMatches(String s)`
`int`	`countMatches(String s, int tokenType)`
`void`	`eolIsSignificant(boolean bFlag)`
`IToken`	`getCurrentToken()`
`ITokenizerInstructor`	`getInstructor()`
`int`	`getLineNumber()`
`int`	`getLineOffset()`
`SourceCodeReader`	`getReader()`
`int`	`getRestoreState()`
`String`	`getSource()`
`int`	`getState()`
`String`	`getStringValue()`
`String`	`getTokenAsString()`
`IToken`	`getTokenAt(int iTokenIndex)`
`IToken`	`getTokenAtPosition(int docPosition)`
`int`	`getTokenColumn()`
`int`	`getTokenEnd()`
`Stack<? extends IToken>`	`getTokens()`
`int`	`getTokenStart()`
`int`	`getType()`
`boolean`	`isAnalyzingDirective()`
`boolean`	`isAnalyzingSeparately()`
`boolean`	`isCommentsSignificant()`
`boolean`	`isEOF()`
`boolean`	`isOperator(String strOperator)`
`boolean`	`isParseDotsAsOperators()`
`boolean`	`isPositioned()`
`boolean`	`isSupportsKeywords()`
`boolean`	`isUnterminatedComment()`
`boolean`	`isUnterminatedString()`
`boolean`	`isWhitespaceSignificant()`
`ISourceCodeTokenizer`	`lightweightRestore()`
`void`	`lowerCaseMode(boolean bLowerCaseMode)`
`int`	`mark()`
`void`	`nextToken()`
`void`	`operatorChars(int iLow, int iHigh)`
`void`	`operators(String[] astrOperators)`
`void`	`ordinaryChar(int ch)`
`void`	`ordinaryChars(int iLow, int iHigh)`
`void`	`parseNumbers()`
`void`	`popOffsetMarker(ITokenizerOffsetMarker offsetMarker)`
`void`	`pushOffsetMarker(ITokenizerOffsetMarker offsetMarker)`
`void`	`quoteChar(int ch)`
`void`	`reset()`
`void`	`reset(Reader reader)`
`void`	`reset(SourceCodeReader reader)`
`void`	`restoreToMark(int markedState)`
`void`	`setCommentsSignificant(boolean bCommentsSignificant)`
`void`	`setInstructor(ITokenizerInstructor instructor)`
`void`	`setParseDotsAsOperators(boolean parseDotsAsOperators)`
`void`	`setSupportsKeywords(boolean supportsKeywords)`
`void`	`setWhitespaceSignificant(boolean bWhitespaceSignificant)`
`void`	`whitespaceChars(int iLow, int iHigh)`
`void`	`wordChars(int iLow, int iHigh)`

Field Detail
- TT_EOL
```
static final int TT_EOL
```
  See Also:
  
  Constant Field Values
- TT_EOF
```
static final int TT_EOF
```
  See Also:
  
  Constant Field Values
- TT_WHITESPACE
```
static final int TT_WHITESPACE
```
  See Also:
  
  Constant Field Values
- TT_COMMENT
```
static final int TT_COMMENT
```
  See Also:
  
  Constant Field Values
- TT_NUMBER
```
static final int TT_NUMBER
```
  See Also:
  
  Constant Field Values
- TT_WORD
```
static final int TT_WORD
```
  See Also:
  
  Constant Field Values
- TT_OPERATOR
```
static final int TT_OPERATOR
```
  See Also:
  
  Constant Field Values
- TT_KEYWORD
```
static final int TT_KEYWORD
```
  See Also:
  
  Constant Field Values
- TT_NOTHING
```
static final int TT_NOTHING
```
  See Also:
  
  Constant Field Values
- TT_INTEGER
```
static final int TT_INTEGER
```
  See Also:
  
  Constant Field Values

Method Detail

copy
```
IToken copy()
```

copyInto
```
IToken copyInto(IToken t)
```

lightweightRestore

ISourceCodeTokenizer lightweightRestore()

isPositioned
```
boolean isPositioned()
```

reset
```
void reset()
```

reset
```
void reset(Reader reader)
```

reset
```
void reset(SourceCodeReader reader)
```

getReader
```
SourceCodeReader getReader()
```

getSource
```
String getSource()
```

getInstructor
```
ITokenizerInstructor getInstructor()
```

setInstructor

void setInstructor(ITokenizerInstructor instructor)

isWhitespaceSignificant
```
boolean isWhitespaceSignificant()
```

setWhitespaceSignificant

void setWhitespaceSignificant(boolean bWhitespaceSignificant)

isCommentsSignificant
```
boolean isCommentsSignificant()
```

setCommentsSignificant

void setCommentsSignificant(boolean bCommentsSignificant)

getLineNumber
```
int getLineNumber()
```

getLineOffset
```
int getLineOffset()
```

getTokenColumn
```
int getTokenColumn()
```

getCurrentToken
```
IToken getCurrentToken()
```

getTokenAt
```
IToken getTokenAt(int iTokenIndex)
```

getTokenAtPosition

IToken getTokenAtPosition(int docPosition)

wordChars

void wordChars(int iLow,
               int iHigh)

whitespaceChars

void whitespaceChars(int iLow,
                     int iHigh)

ordinaryChars

void ordinaryChars(int iLow,
                   int iHigh)

ordinaryChar
```
void ordinaryChar(int ch)
```

operators

void operators(String[] astrOperators)

operatorChars

void operatorChars(int iLow,
                   int iHigh)

isOperator

boolean isOperator(String strOperator)

commentChar
```
void commentChar(int ch)
```

quoteChar
```
void quoteChar(int ch)
```

parseNumbers
```
void parseNumbers()
```

eolIsSignificant
```
void eolIsSignificant(boolean bFlag)
```

lowerCaseMode

void lowerCaseMode(boolean bLowerCaseMode)

isUnterminatedString
```
boolean isUnterminatedString()
```

isUnterminatedComment
```
boolean isUnterminatedComment()
```

setParseDotsAsOperators

void setParseDotsAsOperators(boolean parseDotsAsOperators)

isParseDotsAsOperators
```
boolean isParseDotsAsOperators()
```

getTokenStart
```
int getTokenStart()
```

getTokenEnd
```
int getTokenEnd()
```

getTokenAsString
```
String getTokenAsString()
```

mark
```
int mark()
```

restoreToMark
```
void restoreToMark(int markedState)
```

getRestoreState
```
int getRestoreState()
```

nextToken
```
void nextToken()
```

pushOffsetMarker

void pushOffsetMarker(ITokenizerOffsetMarker offsetMarker)

popOffsetMarker

void popOffsetMarker(ITokenizerOffsetMarker offsetMarker)

getState
```
int getState()
```

countMatches
```
int countMatches(String s)
```

countMatches

int countMatches(String s,
                 int tokenType)

getType
```
int getType()
```

getStringValue
```
String getStringValue()
```

isEOF
```
boolean isEOF()
```

getTokens
```
Stack<? extends IToken> getTokens()
```

isAnalyzingSeparately
```
boolean isAnalyzingSeparately()
```

isAnalyzingDirective
```
boolean isAnalyzingDirective()
```

isSupportsKeywords
```
boolean isSupportsKeywords()
```

setSupportsKeywords

void setSupportsKeywords(boolean supportsKeywords)

Interface ISourceCodeTokenizer

Field Summary

Method Summary

Field Detail

TT_EOL

TT_EOF

TT_WHITESPACE

TT_COMMENT

TT_NUMBER

TT_WORD

TT_OPERATOR

TT_KEYWORD

TT_NOTHING

TT_INTEGER

Method Detail

copy

copyInto

lightweightRestore

isPositioned

reset

reset

reset

getReader

getSource

getInstructor

setInstructor

isWhitespaceSignificant

setWhitespaceSignificant

isCommentsSignificant

setCommentsSignificant

getLineNumber

getLineOffset

getTokenColumn

getCurrentToken

getTokenAt

getTokenAtPosition

wordChars

whitespaceChars

ordinaryChars

ordinaryChar

operators

operatorChars

isOperator

commentChar

quoteChar

parseNumbers

eolIsSignificant

lowerCaseMode

isUnterminatedString

isUnterminatedComment

setParseDotsAsOperators

isParseDotsAsOperators

getTokenStart

getTokenEnd

getTokenAsString

mark

restoreToMark

getRestoreState

nextToken

pushOffsetMarker

popOffsetMarker

getState

countMatches

countMatches

getType

getStringValue

isEOF

getTokens

isAnalyzingSeparately

isAnalyzingDirective

isSupportsKeywords

setSupportsKeywords