stardiv.tokenizer
Class Tokenizer
java.lang.Object
|
+--stardiv.tokenizer.Tokenizer
- All Implemented Interfaces:
- stardiv.concepts.QueryData
- Direct Known Subclasses:
- stardiv.js.comp.Tokenizer
- public synchronized class Tokenizer
- extends java.lang.Object
- implements stardiv.concepts.QueryData
Field Summary |
private static short |
ACTION_BEGIN
|
private static short |
ACTION_CPP_COMMENT
|
private static short |
ACTION_C_COMMENT
|
private static short |
ACTION_END_C_COMMENT
|
private static short |
ACTION_EOL
|
private static short |
ACTION_HTMLSTRING
|
private static short |
ACTION_HTML_2_OF_2
|
private static short |
ACTION_HTML_2_OF_3
|
private static short |
ACTION_HTML_3_OF_3
|
private static short |
ACTION_HTML_COMMENT
|
private static short |
ACTION_HTML_COMMENT_1
|
private static short |
ACTION_HTML_COMMENT_2
|
private static short |
ACTION_HTML_COMMENT_3
|
private static short |
ACTION_IDENTIFIER
|
private static short |
ACTION_NUMBER
|
private static short |
ACTION_SKIP
|
private static short |
ACTION_START_COMMENT
|
private static short |
ACTION_STRING
|
private static short |
ACTION_WHITESPACE
|
protected static short |
CHAR_EOL
|
protected static short |
CHAR_IN_HEX_NUMBER
|
protected static short |
CHAR_IN_IDENTIFIER
|
protected static short |
CHAR_IN_NUMBER
|
protected static short |
CHAR_IN_OCT_NUMBER
|
protected static short |
CHAR_SKIP
|
protected static short |
CHAR_SPACE
|
protected static short |
CHAR_START_IDENTIFIER
|
protected static short |
CHAR_START_NUMBER
|
protected static int |
CHAR_TYPE_TAB_SIZE
|
static int |
EID_DOUBLE_FORMAT
|
static int |
EID_LONG_FORMAT
|
static int |
EID_UNTERMINATED_COMMENT
|
static int |
EID_UNTERMINATED_SKIP
|
static int |
EID_UNTERMINATED_STRING
|
private static int |
NUM_TOKEN_MODE_C
|
private static int |
NUM_TOKEN_MODE_JAVA
|
private static int |
NUM_TOKEN_MODE_JSCRIPT
|
private stardiv.memory.CharBuffer |
aASBuffer
|
protected stardiv.memory.AtomUnionCache |
aAtomUnionCache
|
private stardiv.memory.Stack |
aCharTokenCache
|
protected static short[] |
aCharTypeTab
|
private stardiv.memory.Stack |
aTokenCache
|
private boolean |
bBlockEnd
|
private boolean |
bSkipStringEsc
|
private boolean |
bStringEscape
|
private boolean |
bStringIncludesEsc
|
private char |
cFirstChar
|
private char |
cSkipEnd
|
private char |
cSkipStart
|
private int |
iActionState
|
private int |
iBufEnd
|
private int |
iLength
|
private int |
iMakeBufferLen
|
private int |
iMakeBufferPos
|
private int |
iNumTokenMode
|
private int |
iPosition
|
private int |
iScanPos
|
private int |
iSkipActionState
|
private int |
iSkipBlockLevel
|
private int |
iStartCol
|
private int |
iStartPos
|
private int |
iState
|
private int |
iTotalBufferPos
|
private int |
iTotalStartPos
|
private char[] |
pBuffer
|
private stardiv.tokenizer.Token |
pPeekToken
|
Fields inherited from interface stardiv.concepts.QueryData |
STATE_EOF, STATE_NO_SOURCE, STATE_PENDING, STATE_READY |
Constructor Summary |
Tokenizer()
|
Method Summary |
protected stardiv.tokenizer.Token |
baseNextToken()
|
private stardiv.tokenizer.Token |
getCharToken(char c)
|
private stardiv.tokenizer.Token |
getDoubleToken(char[] c,
int i1,
int i2,
double d)
|
java.lang.String |
getErrorText(int i)
|
protected stardiv.tokenizer.Token |
getErrorToken(char[] c,
int i1,
int i2,
int i3)
|
private stardiv.tokenizer.Token |
getLongToken(char[] c,
int i1,
int i2,
long l)
|
int |
getQueryDataState()
|
private stardiv.tokenizer.Token |
getSkipToken(char[] c,
int i1,
int i2)
|
protected stardiv.tokenizer.Token |
getToken(int i1,
int i2,
int i3)
|
protected stardiv.tokenizer.Token |
getToken(int i,
stardiv.memory.AtomUnion au)
|
private stardiv.tokenizer.Token |
getlong2DoubleToken(char[] c,
int i1,
int i2,
double d)
|
private char[] |
makeBuffer()
|
void |
newSource()
|
stardiv.tokenizer.Token |
nextToken()
|
stardiv.tokenizer.Token |
peekToken()
|
void |
queryData()
|
void |
releaseToken(stardiv.tokenizer.Token t)
|
void |
setCNumTokenMode()
|
void |
setData(char[] c,
int i1,
int i2)
|
void |
setJavaNumTokenMode()
|
void |
setQueryDataState(int i)
|
stardiv.tokenizer.Token |
skipToken(char c1,
char c2)
|
stardiv.tokenizer.Token |
skipTokenIgnoreWhites(char c1,
char c2)
|
private stardiv.tokenizer.Token |
skipTokenImpl()
|
Methods inherited from class java.lang.Object |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
ACTION_BEGIN
private static final short ACTION_BEGIN
ACTION_CPP_COMMENT
private static final short ACTION_CPP_COMMENT
ACTION_C_COMMENT
private static final short ACTION_C_COMMENT
ACTION_END_C_COMMENT
private static final short ACTION_END_C_COMMENT
ACTION_EOL
private static final short ACTION_EOL
ACTION_HTMLSTRING
private static final short ACTION_HTMLSTRING
ACTION_HTML_2_OF_2
private static final short ACTION_HTML_2_OF_2
ACTION_HTML_2_OF_3
private static final short ACTION_HTML_2_OF_3
ACTION_HTML_3_OF_3
private static final short ACTION_HTML_3_OF_3
ACTION_HTML_COMMENT
private static final short ACTION_HTML_COMMENT
ACTION_HTML_COMMENT_1
private static final short ACTION_HTML_COMMENT_1
ACTION_HTML_COMMENT_2
private static final short ACTION_HTML_COMMENT_2
ACTION_HTML_COMMENT_3
private static final short ACTION_HTML_COMMENT_3
ACTION_IDENTIFIER
private static final short ACTION_IDENTIFIER
ACTION_NUMBER
private static final short ACTION_NUMBER
ACTION_SKIP
private static final short ACTION_SKIP
ACTION_START_COMMENT
private static final short ACTION_START_COMMENT
ACTION_STRING
private static final short ACTION_STRING
ACTION_WHITESPACE
private static final short ACTION_WHITESPACE
CHAR_EOL
protected static final short CHAR_EOL
CHAR_IN_HEX_NUMBER
protected static final short CHAR_IN_HEX_NUMBER
CHAR_IN_IDENTIFIER
protected static final short CHAR_IN_IDENTIFIER
CHAR_IN_NUMBER
protected static final short CHAR_IN_NUMBER
CHAR_IN_OCT_NUMBER
protected static final short CHAR_IN_OCT_NUMBER
CHAR_SKIP
protected static final short CHAR_SKIP
CHAR_SPACE
protected static final short CHAR_SPACE
CHAR_START_IDENTIFIER
protected static final short CHAR_START_IDENTIFIER
CHAR_START_NUMBER
protected static final short CHAR_START_NUMBER
CHAR_TYPE_TAB_SIZE
protected static final int CHAR_TYPE_TAB_SIZE
EID_DOUBLE_FORMAT
public static final int EID_DOUBLE_FORMAT
EID_LONG_FORMAT
public static final int EID_LONG_FORMAT
EID_UNTERMINATED_COMMENT
public static final int EID_UNTERMINATED_COMMENT
EID_UNTERMINATED_SKIP
public static final int EID_UNTERMINATED_SKIP
EID_UNTERMINATED_STRING
public static final int EID_UNTERMINATED_STRING
NUM_TOKEN_MODE_C
private static final int NUM_TOKEN_MODE_C
NUM_TOKEN_MODE_JAVA
private static final int NUM_TOKEN_MODE_JAVA
NUM_TOKEN_MODE_JSCRIPT
private static final int NUM_TOKEN_MODE_JSCRIPT
aASBuffer
private stardiv.memory.CharBuffer aASBuffer
aAtomUnionCache
protected stardiv.memory.AtomUnionCache aAtomUnionCache
aCharTokenCache
private stardiv.memory.Stack aCharTokenCache
aCharTypeTab
protected static final short[] aCharTypeTab
aTokenCache
private stardiv.memory.Stack aTokenCache
bBlockEnd
private boolean bBlockEnd
bSkipStringEsc
private boolean bSkipStringEsc
bStringEscape
private boolean bStringEscape
bStringIncludesEsc
private boolean bStringIncludesEsc
cFirstChar
private char cFirstChar
cSkipEnd
private char cSkipEnd
cSkipStart
private char cSkipStart
iActionState
private int iActionState
iBufEnd
private int iBufEnd
iLength
private int iLength
iMakeBufferLen
private int iMakeBufferLen
iMakeBufferPos
private int iMakeBufferPos
iNumTokenMode
private int iNumTokenMode
iPosition
private int iPosition
iScanPos
private int iScanPos
iSkipActionState
private int iSkipActionState
iSkipBlockLevel
private int iSkipBlockLevel
iStartCol
private int iStartCol
iStartPos
private int iStartPos
iState
private int iState
iTotalBufferPos
private int iTotalBufferPos
iTotalStartPos
private int iTotalStartPos
pBuffer
private char[] pBuffer
pPeekToken
private stardiv.tokenizer.Token pPeekToken
Tokenizer
public Tokenizer()
baseNextToken
protected stardiv.tokenizer.Token baseNextToken()
getCharToken
private stardiv.tokenizer.Token getCharToken(char c)
getDoubleToken
private stardiv.tokenizer.Token getDoubleToken(char[] c,
int i1,
int i2,
double d)
getErrorText
public java.lang.String getErrorText(int i)
getErrorToken
protected stardiv.tokenizer.Token getErrorToken(char[] c,
int i1,
int i2,
int i3)
getLongToken
private stardiv.tokenizer.Token getLongToken(char[] c,
int i1,
int i2,
long l)
getQueryDataState
public int getQueryDataState()
- Specified by:
getQueryDataState
in interface stardiv.concepts.QueryData
getSkipToken
private stardiv.tokenizer.Token getSkipToken(char[] c,
int i1,
int i2)
getToken
protected stardiv.tokenizer.Token getToken(int i1,
int i2,
int i3)
getToken
protected stardiv.tokenizer.Token getToken(int i,
stardiv.memory.AtomUnion au)
getlong2DoubleToken
private stardiv.tokenizer.Token getlong2DoubleToken(char[] c,
int i1,
int i2,
double d)
makeBuffer
private char[] makeBuffer()
newSource
public void newSource()
nextToken
public stardiv.tokenizer.Token nextToken()
peekToken
public stardiv.tokenizer.Token peekToken()
queryData
public void queryData()
- Specified by:
queryData
in interface stardiv.concepts.QueryData
releaseToken
public void releaseToken(stardiv.tokenizer.Token t)
setCNumTokenMode
public void setCNumTokenMode()
setData
public void setData(char[] c,
int i1,
int i2)
throws java.lang.IllegalArgumentException,
java.lang.NullPointerException
- Specified by:
setData
in interface stardiv.concepts.QueryData
setJavaNumTokenMode
public void setJavaNumTokenMode()
setQueryDataState
public void setQueryDataState(int i)
throws java.lang.IllegalArgumentException
- Specified by:
setQueryDataState
in interface stardiv.concepts.QueryData
skipToken
public stardiv.tokenizer.Token skipToken(char c1,
char c2)
skipTokenIgnoreWhites
public stardiv.tokenizer.Token skipTokenIgnoreWhites(char c1,
char c2)
skipTokenImpl
private stardiv.tokenizer.Token skipTokenImpl()