gw.internal.gosu.parser

Class SourceCodeTokenizer

java.lang.Object

gw.internal.gosu.parser.SourceCodeTokenizer

All Implemented Interfaces:

ISourceCodeTokenizer

public final class SourceCodeTokenizer
extends Object
implements ISourceCodeTokenizer

Field Summary

Fields inherited from interface gw.lang.parser.ISourceCodeTokenizer

TT_COMMENT, TT_EOF, TT_EOL, TT_INTEGER, TT_KEYWORD, TT_NOTHING, TT_NUMBER, TT_OPERATOR, TT_WHITESPACE, TT_WORD

Constructor Summary

Constructors

Constructor and Description
SourceCodeTokenizer(CharSequence sourceCode)
SourceCodeTokenizer(Reader reader)
SourceCodeTokenizer(SourceCodeReader reader)
SourceCodeTokenizer(SourceCodeReader reader, ITokenizerInstructor instructor)

Method Summary

Modifier and Type	Method and Description
void	commentChar(int ch)
IToken	copy()
IToken	copyInto(IToken t)
int	countMatches(String s)
int	countMatches(String s, int tokenType)
void	eolIsSignificant(boolean bFlag)
Token	getCurrentToken()
ITokenizerInstructor	getInstructor()
SourceCodeTokenizerInternal	getInternal()
int	getLineNumber()
int	getLineOffset()
ITokenizerOffsetMarker	getOffsetMarker()
Token	getPriorToken()
Token	getPriorToken(boolean bSkipWhitespace)
Token	getPriorToken(boolean bSkipWhitespace, boolean bSkipComments)
SourceCodeReader	getReader()
int	getRestoreState()
String	getSource()
int	getState()
String	getStringValue()
String	getTokenAsString()
Token	getTokenAt(int iTokenIndex)
int	getTokenColumn()
int	getTokenEnd()
Stack<Token>	getTokens()
int	getTokenStart()
int	getType()
boolean	isAnalyzingDirective()
boolean	isAnalyzingSeparately()
boolean	isCommentsSignificant()
boolean	isEOF()
boolean	isNOTHING()
boolean	isOperator(String strOperator)
boolean	isParseDotsAsOperators()
boolean	isPositioned()
boolean	isSupportsKeywords()
boolean	isUnterminatedComment()
boolean	isUnterminatedString()
boolean	isWhitespaceSignificant()
SourceCodeTokenizer	lightweightRestore()
int	lookaheadType(int iTokens, boolean bSkipSpaces)
void	lowerCaseMode(boolean bLowerCaseMode)
int	mark()
void	nextToken()
void	operatorChars(int iLow, int iHigh)
void	operators(String[] astrOperators)
void	ordinaryChar(int ch)
void	ordinaryChars(int iLow, int iHigh)
void	parseNumbers()
DocCommentBlock	popLastComment()
void	popOffsetMarker(ITokenizerOffsetMarker offsetMarker)
void	pushOffsetMarker(ITokenizerOffsetMarker offsetMarker)
void	quoteChar(int ch)
void	reset()
void	reset(Reader reader)
void	reset(SourceCodeReader reader)
void	resetButKeepTokens()
void	restoreToMark(int iMarkedOffset)
void	setCommentsSignificant(boolean bCommentsSignificant)
void	setInstructor(ITokenizerInstructor instructor)
void	setParseDotsAsOperators(boolean parseDotsAsOperators)
void	setSupportsKeywords(boolean supportsKeywords)
void	setWhitespaceSignificant(boolean bWhitespaceSignificant)
String	toString()
void	whitespaceChars(int iLow, int iHigh)
void	wordChars(int iLow, int iHigh)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Constructor Detail

SourceCodeTokenizer

public SourceCodeTokenizer(CharSequence sourceCode)

SourceCodeTokenizer

public SourceCodeTokenizer(Reader reader)

SourceCodeTokenizer

public SourceCodeTokenizer(SourceCodeReader reader)

SourceCodeTokenizer

public SourceCodeTokenizer(SourceCodeReader reader,
                           ITokenizerInstructor instructor)

Method Detail

copy

public IToken copy()

Specified by:

copy in interface ISourceCodeTokenizer

copyInto

public IToken copyInto(IToken t)

getInternal

public SourceCodeTokenizerInternal getInternal()

lightweightRestore

public SourceCodeTokenizer lightweightRestore()

Specified by:

lightweightRestore in interface ISourceCodeTokenizer

Returns:

a very lightweight restored copy of this tokenizer, without deep cloning anything. This method should be used with care, as it does not clone internal state and cannot actually tokenize. It should be used only for offset information.

isPositioned

public boolean isPositioned()

Specified by:

isPositioned in interface ISourceCodeTokenizer

reset

public void reset()

Specified by:

reset in interface ISourceCodeTokenizer

reset

public void reset(Reader reader)

Specified by:

reset in interface ISourceCodeTokenizer

reset

public void reset(SourceCodeReader reader)

Specified by:

reset in interface ISourceCodeTokenizer

getReader

public SourceCodeReader getReader()

Specified by:

getReader in interface ISourceCodeTokenizer

getSource

public String getSource()

Specified by:

getSource in interface ISourceCodeTokenizer

getInstructor

public ITokenizerInstructor getInstructor()

Specified by:

getInstructor in interface ISourceCodeTokenizer

setInstructor

public void setInstructor(ITokenizerInstructor instructor)

Specified by:

setInstructor in interface ISourceCodeTokenizer

isWhitespaceSignificant

public boolean isWhitespaceSignificant()

Specified by:

isWhitespaceSignificant in interface ISourceCodeTokenizer

setWhitespaceSignificant

public void setWhitespaceSignificant(boolean bWhitespaceSignificant)

Specified by:

setWhitespaceSignificant in interface ISourceCodeTokenizer

isCommentsSignificant

public boolean isCommentsSignificant()

Specified by:

isCommentsSignificant in interface ISourceCodeTokenizer

setCommentsSignificant

public void setCommentsSignificant(boolean bCommentsSignificant)

Specified by:

setCommentsSignificant in interface ISourceCodeTokenizer

getCurrentToken

public final Token getCurrentToken()

Specified by:

getCurrentToken in interface ISourceCodeTokenizer

getTokenAt

public Token getTokenAt(int iTokenIndex)

Specified by:

getTokenAt in interface ISourceCodeTokenizer

getLineNumber

public int getLineNumber()

Specified by:

getLineNumber in interface ISourceCodeTokenizer

getLineOffset

public int getLineOffset()

Specified by:

getLineOffset in interface ISourceCodeTokenizer

getTokenColumn

public int getTokenColumn()

Specified by:

getTokenColumn in interface ISourceCodeTokenizer

wordChars

public void wordChars(int iLow,
                      int iHigh)

Specified by:

wordChars in interface ISourceCodeTokenizer

whitespaceChars

public void whitespaceChars(int iLow,
                            int iHigh)

Specified by:

whitespaceChars in interface ISourceCodeTokenizer

ordinaryChars

public void ordinaryChars(int iLow,
                          int iHigh)

Specified by:

ordinaryChars in interface ISourceCodeTokenizer

ordinaryChar

public void ordinaryChar(int ch)

Specified by:

ordinaryChar in interface ISourceCodeTokenizer

operators

public void operators(String[] astrOperators)

Specified by:

operators in interface ISourceCodeTokenizer

operatorChars

public void operatorChars(int iLow,
                          int iHigh)

Specified by:

operatorChars in interface ISourceCodeTokenizer

isOperator

public boolean isOperator(String strOperator)

Specified by:

isOperator in interface ISourceCodeTokenizer

commentChar

public void commentChar(int ch)

Specified by:

commentChar in interface ISourceCodeTokenizer

quoteChar

public void quoteChar(int ch)

Specified by:

quoteChar in interface ISourceCodeTokenizer

parseNumbers

public void parseNumbers()

Specified by:

parseNumbers in interface ISourceCodeTokenizer

eolIsSignificant

public void eolIsSignificant(boolean bFlag)

Specified by:

eolIsSignificant in interface ISourceCodeTokenizer

lowerCaseMode

public void lowerCaseMode(boolean bLowerCaseMode)

Specified by:

lowerCaseMode in interface ISourceCodeTokenizer

isUnterminatedString

public boolean isUnterminatedString()

Specified by:

isUnterminatedString in interface ISourceCodeTokenizer

isUnterminatedComment

public boolean isUnterminatedComment()

Specified by:

isUnterminatedComment in interface ISourceCodeTokenizer

setParseDotsAsOperators

public void setParseDotsAsOperators(boolean parseDotsAsOperators)

Specified by:

setParseDotsAsOperators in interface ISourceCodeTokenizer

isParseDotsAsOperators

public boolean isParseDotsAsOperators()

Specified by:

isParseDotsAsOperators in interface ISourceCodeTokenizer

popLastComment

public DocCommentBlock popLastComment()

getTokenStart

public int getTokenStart()

Specified by:

getTokenStart in interface ISourceCodeTokenizer

getTokenEnd

public int getTokenEnd()

Specified by:

getTokenEnd in interface ISourceCodeTokenizer

getTokenAsString

public String getTokenAsString()

Specified by:

getTokenAsString in interface ISourceCodeTokenizer

mark

public int mark()

Specified by:

mark in interface ISourceCodeTokenizer

restoreToMark

public void restoreToMark(int iMarkedOffset)

Specified by:

restoreToMark in interface ISourceCodeTokenizer

getRestoreState

public int getRestoreState()

Specified by:

getRestoreState in interface ISourceCodeTokenizer

getTokens

public final Stack<Token> getTokens()

Specified by:

getTokens in interface ISourceCodeTokenizer

nextToken

public void nextToken()

Specified by:

nextToken in interface ISourceCodeTokenizer

getStringValue

public String getStringValue()

Specified by:

getStringValue in interface ISourceCodeTokenizer

toString

public String toString()

Overrides:

toString in class Object

getOffsetMarker

public ITokenizerOffsetMarker getOffsetMarker()

pushOffsetMarker

public void pushOffsetMarker(ITokenizerOffsetMarker offsetMarker)

Specified by:

pushOffsetMarker in interface ISourceCodeTokenizer

popOffsetMarker

public void popOffsetMarker(ITokenizerOffsetMarker offsetMarker)

Specified by:

popOffsetMarker in interface ISourceCodeTokenizer

getState

public int getState()

Specified by:

getState in interface ISourceCodeTokenizer

countMatches

public int countMatches(String s)

Specified by:

countMatches in interface ISourceCodeTokenizer

countMatches

public int countMatches(String s,
                        int tokenType)

Specified by:

countMatches in interface ISourceCodeTokenizer

getType

public int getType()

Specified by:

getType in interface ISourceCodeTokenizer

isEOF

public boolean isEOF()

Specified by:

isEOF in interface ISourceCodeTokenizer

isNOTHING

public boolean isNOTHING()

resetButKeepTokens

public void resetButKeepTokens()

isAnalyzingSeparately

public boolean isAnalyzingSeparately()

Specified by:

isAnalyzingSeparately in interface ISourceCodeTokenizer

isAnalyzingDirective

public boolean isAnalyzingDirective()

Specified by:

isAnalyzingDirective in interface ISourceCodeTokenizer

isSupportsKeywords

public boolean isSupportsKeywords()

Specified by:

isSupportsKeywords in interface ISourceCodeTokenizer

setSupportsKeywords

public void setSupportsKeywords(boolean supportsKeywords)

Specified by:

setSupportsKeywords in interface ISourceCodeTokenizer

getPriorToken

public Token getPriorToken()

getPriorToken

public Token getPriorToken(boolean bSkipWhitespace)

getPriorToken

public Token getPriorToken(boolean bSkipWhitespace,
                           boolean bSkipComments)

lookaheadType

public int lookaheadType(int iTokens,
                         boolean bSkipSpaces)