001/*
002 * ModeShape (http://www.modeshape.org)
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *       http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.modeshape.common.text;
017
018import static org.hamcrest.core.Is.is;
019import static org.junit.Assert.assertThat;
020import java.util.Arrays;
021import org.junit.Before;
022import org.junit.Test;
023import org.modeshape.common.FixFor;
024import org.modeshape.common.text.TokenStream.BasicTokenizer;
025import org.modeshape.common.text.TokenStream.Tokenizer;
026
027public class TokenStreamTest {
028    public static final int WORD = TokenStream.BasicTokenizer.WORD;
029    public static final int SYMBOL = TokenStream.BasicTokenizer.SYMBOL;
030    public static final int DECIMAL = TokenStream.BasicTokenizer.DECIMAL;
031    public static final int SINGLE_QUOTED_STRING = TokenStream.BasicTokenizer.SINGLE_QUOTED_STRING;
032    public static final int DOUBLE_QUOTED_STRING = TokenStream.BasicTokenizer.DOUBLE_QUOTED_STRING;
033    public static final int COMMENT = TokenStream.BasicTokenizer.COMMENT;
034
035    private Tokenizer tokenizer;
036    private String content;
037    private TokenStream tokens;
038
039    @Before
040    public void beforeEach() {
041        tokenizer = TokenStream.basicTokenizer(false);
042        content = "Select all columns from this table";
043        makeCaseInsensitive();
044    }
045
046    public void makeCaseSensitive() {
047        tokens = new TokenStream(content, tokenizer, true);
048        tokens.start();
049    }
050
051    public void makeCaseInsensitive() {
052        tokens = new TokenStream(content, tokenizer, false);
053        tokens.start();
054    }
055
056    @Test( expected = IllegalStateException.class )
057    public void shouldNotAllowConsumeBeforeStartIsCalled() {
058        tokens = new TokenStream(content, TokenStream.basicTokenizer(false), false);
059        tokens.consume("Select");
060    }
061
062    @Test( expected = IllegalStateException.class )
063    public void shouldNotAllowHasNextBeforeStartIsCalled() {
064        tokens = new TokenStream(content, TokenStream.basicTokenizer(false), false);
065        tokens.hasNext();
066    }
067
068    @Test( expected = IllegalStateException.class )
069    public void shouldNotAllowMatchesBeforeStartIsCalled() {
070        tokens = new TokenStream(content, TokenStream.basicTokenizer(false), false);
071        tokens.matches("Select");
072    }
073
074    @Test( expected = IllegalStateException.class )
075    public void shouldNotAllowCanConsumeBeforeStartIsCalled() {
076        tokens = new TokenStream(content, TokenStream.basicTokenizer(false), false);
077        tokens.canConsume("Select");
078    }
079
080    @Test
081    public void shouldReturnTrueFromHasNextIfThereIsACurrentToken() {
082        content = "word";
083        makeCaseSensitive();
084        assertThat(tokens.currentToken().matches("word"), is(true));
085        assertThat(tokens.hasNext(), is(true));
086    }
087
088    @Test
089    public void shouldConsumeInCaseSensitiveMannerWithExpectedValuesWhenMatchingExactCase() {
090        makeCaseSensitive();
091        tokens.consume("Select");
092        tokens.consume("all");
093        tokens.consume("columns");
094        tokens.consume("from");
095        tokens.consume("this");
096        tokens.consume("table");
097        assertThat(tokens.hasNext(), is(false));
098    }
099
100    @Test( expected = ParsingException.class )
101    public void shouldFailToConsumeInCaseSensitiveMannerWithExpectedValuesWhenMatchingIncorrectCase() {
102        makeCaseSensitive();
103        tokens.consume("Select");
104        tokens.consume("all");
105        tokens.consume("Columns");
106    }
107
108    @Test
109    public void shouldConsumeInCaseInsensitiveMannerWithExpectedValuesWhenMatchingNonExactCase() {
110        makeCaseInsensitive();
111        tokens.consume("SELECT");
112        tokens.consume("ALL");
113        tokens.consume("COLUMNS");
114        tokens.consume("FROM");
115        tokens.consume("THIS");
116        tokens.consume("TABLE");
117        assertThat(tokens.hasNext(), is(false));
118    }
119
120    @Test( expected = ParsingException.class )
121    public void shouldFailToConsumeInCaseInsensitiveMannerWithExpectedValuesWhenMatchingStringIsInLowerCase() {
122        makeCaseInsensitive();
123        tokens.consume("SELECT");
124        tokens.consume("ALL");
125        tokens.consume("columns");
126    }
127
128    @FixFor( "MODE-2497" )
129    @Test
130    public void shouldHandleNonAsciiCharactersWhenCaseSensitive() {
131        content = "ü and";
132        makeCaseSensitive();
133        tokens.consume("ü");
134        tokens.consume("and");
135        assertThat(tokens.hasNext(), is(false));
136    }
137
138    @FixFor( "MODE-2497" )
139    @Test
140    public void shouldHandleßCharacterWhenCaseSensitive() {
141        content = "ß and";
142        makeCaseSensitive();
143        tokens.consume("ß");
144        tokens.consume("and");
145        assertThat(tokens.hasNext(), is(false));
146    }
147
148    @FixFor( "MODE-2497" )
149    @Test
150    public void shouldConsumeCaseInsensitiveStringInOriginalCase() {
151        makeCaseInsensitive();
152        String firstToken = tokens.consume();
153
154        assertThat(firstToken, is("Select"));
155    }
156
157    @FixFor( "MODE-2497" )
158    @Test
159    public void shouldMatchUpperCaseVersionOfßCharacterWhenCaseInsensitive() {
160        content = "ß";
161        makeCaseInsensitive();
162        tokens.consume("SS");
163        assertThat(tokens.hasNext(), is(false));
164    }
165
166    @FixFor( "MODE-2497" )
167    @Test
168    public void shouldHandleTokensAfterßCharacterWhenCaseInsensitive() {
169        content = "ß and";
170        makeCaseInsensitive();
171        tokens.consume(TokenStream.ANY_VALUE);
172        tokens.consume("AND");
173        assertThat(tokens.hasNext(), is(false));
174    }
175
176    @Test
177    public void shouldReturnTrueFromCanConsumeWithCaseSensitiveTokenStreamIfMatchStringDoesMatchCaseExactly() {
178        makeCaseSensitive();
179        assertThat(tokens.canConsume("Select"), is(true));
180        assertThat(tokens.canConsume("all"), is(true));
181        assertThat(tokens.canConsume("columns"), is(true));
182        assertThat(tokens.canConsume("from"), is(true));
183        assertThat(tokens.canConsume("this"), is(true));
184        assertThat(tokens.canConsume("table"), is(true));
185        assertThat(tokens.hasNext(), is(false));
186    }
187
188    @Test
189    public void shouldReturnFalseFromCanConsumeWithCaseSensitiveTokenStreamIfMatchStringDoesNotMatchCaseExactly() {
190        makeCaseSensitive();
191        assertThat(tokens.canConsume("Select"), is(true));
192        assertThat(tokens.canConsume("all"), is(true));
193        assertThat(tokens.canConsume("Columns"), is(false));
194        assertThat(tokens.canConsume("COLUMNS"), is(false));
195        assertThat(tokens.canConsume("columns"), is(true));
196        assertThat(tokens.canConsume("from"), is(true));
197        assertThat(tokens.canConsume("THIS"), is(false));
198        assertThat(tokens.canConsume("table"), is(false));
199        assertThat(tokens.canConsume("this"), is(true));
200        assertThat(tokens.canConsume("table"), is(true));
201        assertThat(tokens.hasNext(), is(false));
202    }
203
204    @Test
205    public void shouldReturnTrueFromCanConsumeWithCaseSensitiveTokenStreamIfSuppliedTypeDoesMatch() {
206        makeCaseSensitive();
207        assertThat(tokens.canConsume(WORD), is(true));
208        assertThat(tokens.canConsume(WORD), is(true));
209        assertThat(tokens.canConsume(WORD), is(true));
210        assertThat(tokens.canConsume(WORD), is(true));
211        assertThat(tokens.canConsume(WORD), is(true));
212        assertThat(tokens.canConsume(WORD), is(true));
213        assertThat(tokens.hasNext(), is(false));
214    }
215
216    @Test
217    public void shouldReturnFalseFromCanConsumeWithCaseSensitiveTokenStreamIfSuppliedTypeDoesMatch() {
218        makeCaseSensitive();
219        assertThat(tokens.canConsume(WORD), is(true));
220        assertThat(tokens.canConsume(WORD), is(true));
221        assertThat(tokens.canConsume(COMMENT), is(false));
222        assertThat(tokens.canConsume(SINGLE_QUOTED_STRING), is(false));
223        assertThat(tokens.canConsume(DOUBLE_QUOTED_STRING), is(false));
224        assertThat(tokens.canConsume(DECIMAL), is(false));
225        assertThat(tokens.canConsume(SYMBOL), is(false));
226
227        assertThat(tokens.canConsume(WORD), is(true));
228        assertThat(tokens.canConsume(WORD), is(true));
229        assertThat(tokens.canConsume(WORD), is(true));
230        assertThat(tokens.canConsume(WORD), is(true));
231        assertThat(tokens.hasNext(), is(false));
232    }
233
234    @Test
235    public void shouldReturnTrueFromMatchesWithCaseSensitiveTokenStreamIfMatchStringDoesMatchCaseExactly() {
236        makeCaseSensitive();
237        assertThat(tokens.matches("Select"), is(true));
238        assertThat(tokens.matches("select"), is(false));
239        assertThat(tokens.canConsume("Select"), is(true));
240        assertThat(tokens.matches("all"), is(true));
241        assertThat(tokens.canConsume("all"), is(true));
242    }
243
244    @Test
245    public void shouldReturnFalseFromMatchesWithCaseSensitiveTokenStreamIfMatchStringDoesMatchCaseExactly() {
246        makeCaseSensitive();
247        assertThat(tokens.matches("select"), is(false));
248        assertThat(tokens.matches("SElect"), is(false));
249        assertThat(tokens.matches("Select"), is(true));
250    }
251
252    @Test
253    public void shouldReturnFalseFromCanConsumeWithCaseInsensitiveTokenStreamIfMatchStringIsNotUppercase() {
254        makeCaseInsensitive();
255        assertThat(tokens.canConsume("Select"), is(false));
256        assertThat(tokens.canConsume("SELECT"), is(true));
257        assertThat(tokens.canConsume("aLL"), is(false));
258        assertThat(tokens.canConsume("all"), is(false));
259        assertThat(tokens.canConsume("ALL"), is(true));
260    }
261
262    @Test
263    public void shouldReturnTrueFromCanConsumeWithCaseInsensitiveTokenStreamIfMatchStringDoesNotMatchCaseExactly() {
264        makeCaseInsensitive();
265        assertThat(tokens.canConsume("SELECT"), is(true));
266        assertThat(tokens.canConsume("ALL"), is(true));
267        assertThat(tokens.canConsume("COLUMNS"), is(true));
268        assertThat(tokens.canConsume("FROM"), is(true));
269        assertThat(tokens.canConsume("THIS"), is(true));
270        assertThat(tokens.canConsume("TABLE"), is(true));
271        assertThat(tokens.hasNext(), is(false));
272    }
273
274    @Test
275    public void shouldReturnTrueFromCanConsumeWithCaseInsensitiveTokenStreamIfSuppliedTypeDoesMatch() {
276        makeCaseInsensitive();
277        assertThat(tokens.canConsume(WORD), is(true));
278        assertThat(tokens.canConsume(WORD), is(true));
279        assertThat(tokens.canConsume(WORD), is(true));
280        assertThat(tokens.canConsume(WORD), is(true));
281        assertThat(tokens.canConsume(WORD), is(true));
282        assertThat(tokens.canConsume(WORD), is(true));
283        assertThat(tokens.hasNext(), is(false));
284    }
285
286    @Test
287    public void shouldReturnFalseFromCanConsumeWithCaseInsensitiveTokenStreamIfSuppliedTypeDoesMatch() {
288        makeCaseInsensitive();
289        assertThat(tokens.canConsume(WORD), is(true));
290        assertThat(tokens.canConsume(WORD), is(true));
291        assertThat(tokens.canConsume(COMMENT), is(false));
292        assertThat(tokens.canConsume(SINGLE_QUOTED_STRING), is(false));
293        assertThat(tokens.canConsume(DOUBLE_QUOTED_STRING), is(false));
294        assertThat(tokens.canConsume(DECIMAL), is(false));
295        assertThat(tokens.canConsume(SYMBOL), is(false));
296
297        assertThat(tokens.canConsume(WORD), is(true));
298        assertThat(tokens.canConsume(WORD), is(true));
299        assertThat(tokens.canConsume(WORD), is(true));
300        assertThat(tokens.canConsume(WORD), is(true));
301        assertThat(tokens.hasNext(), is(false));
302    }
303
304    @Test
305    public void shouldReturnTrueFromMatchesWithCaseInsensitiveTokenStreamIfMatchStringIsUppercaseAndMatches() {
306        makeCaseInsensitive();
307        assertThat(tokens.matches("SELECT"), is(true));
308        assertThat(tokens.canConsume("SELECT"), is(true));
309        assertThat(tokens.matches("ALL"), is(true));
310        assertThat(tokens.canConsume("ALL"), is(true));
311    }
312
313    @Test
314    public void shouldReturnFalseFromMatchesWithCaseInsensitiveTokenStreamIfMatchStringIsUppercaseAndDoesNotMatch() {
315        makeCaseInsensitive();
316        assertThat(tokens.matches("ALL"), is(false));
317        assertThat(tokens.matches("SElect"), is(false));
318        assertThat(tokens.matches("SELECT"), is(true));
319    }
320
321    @Test
322    public void shouldConsumeMultipleTokensIfTheyMatch() {
323        makeCaseInsensitive();
324        tokens.consume("SELECT", "ALL", "COLUMNS", "FROM", "THIS", "TABLE");
325        assertThat(tokens.hasNext(), is(false));
326    }
327
328    @Test( expected = ParsingException.class )
329    public void shouldFailToConsumeMultipleTokensIfTheyDoNotMatch() {
330        makeCaseInsensitive();
331        tokens.consume("SELECT", "ALL", "COLUMNS", "FROM", "TABLE");
332    }
333
334    @Test
335    public void shouldReturnTrueFromCanConsumeMultipleTokensIfTheyAllMatch() {
336        makeCaseInsensitive();
337        assertThat(tokens.canConsume("SELECT", "ALL", "COLUMNS", "FROM", "THIS", "TABLE"), is(true));
338        assertThat(tokens.hasNext(), is(false));
339    }
340
341    @Test
342    public void shouldReturnTrueFromCanConsumeArrayOfTokensIfTheyAllMatch() {
343        makeCaseInsensitive();
344        assertThat(tokens.matches(new String[] {"SELECT", "ALL", "COLUMNS", "FROM", "THIS", "TABLE"}), is(true));
345        assertThat(tokens.canConsume(new String[] {"SELECT", "ALL", "COLUMNS", "FROM", "THIS", "TABLE"}), is(true));
346        assertThat(tokens.hasNext(), is(false));
347    }
348
349    @Test
350    public void shouldReturnTrueFromCanConsumeMultipleTokensIfTheyDoNotAllMatch() {
351        makeCaseInsensitive();
352        // Unable to consume unless they all match ...
353        assertThat(tokens.canConsume("SELECT", "ALL", "COLUMNS", "FRM", "THIS", "TABLE"), is(false));
354        assertThat(tokens.canConsume("SELECT", "ALL", "COLUMNS", "FROM", "THIS", "TABLE", "EXTRA"), is(false));
355        assertThat(tokens.canConsume("SELECT", "ALL", "COLUMNS", "FROM", "EXTRA", "THIS", "TABLE"), is(false));
356        assertThat(tokens.hasNext(), is(true));
357        // Should have consumed nothing so far ...
358        assertThat(tokens.canConsume("SELECT", "ALL", "COLUMNS"), is(true));
359        assertThat(tokens.canConsume("FROM", "THIS", "TABLE"), is(true));
360        assertThat(tokens.hasNext(), is(false));
361    }
362
363    @Test
364    public void shouldReturnTrueFromMatchAnyOfIfAnyOfTheTokenValuesMatch() {
365        makeCaseInsensitive();
366        // Unable to consume unless they all match ...
367        assertThat(tokens.matchesAnyOf("ALL", "COLUMNS"), is(false));
368        assertThat(tokens.matchesAnyOf("ALL", "COLUMNS", "SELECT"), is(true));
369        tokens.consume("SELECT");
370        assertThat(tokens.matchesAnyOf("ALL", "COLUMNS", "SELECT"), is(true));
371        tokens.consume("ALL");
372        assertThat(tokens.matchesAnyOf("ALL", "COLUMNS", "SELECT"), is(true));
373        tokens.consume("COLUMNS");
374        assertThat(tokens.canConsume("FROM", "THIS", "TABLE"), is(true));
375        assertThat(tokens.hasNext(), is(false));
376    }
377
378    @Test
379    public void shouldReturnTrueFromMatchIfAllTypeValuesMatch() {
380        makeCaseInsensitive();
381        assertThat(tokens.matches(BasicTokenizer.WORD, BasicTokenizer.WORD), is(true));
382    }
383
384    @Test
385    public void shouldReturnFalseFromMatchIfAllTypeValuesDoNotMatch() {
386        makeCaseInsensitive();
387        assertThat(tokens.matches(BasicTokenizer.WORD, BasicTokenizer.DECIMAL), is(false));
388        assertThat(tokens.matches(BasicTokenizer.DECIMAL, BasicTokenizer.WORD), is(false));
389    }
390
391    @Test
392    public void shouldConsumeMultipleTokensWithAnyValueConstant() {
393        makeCaseInsensitive();
394        // Unable to consume unless they all match ...
395        tokens.consume("SELECT", "ALL", TokenStream.ANY_VALUE);
396        tokens.consume("FROM", "THIS", "TABLE");
397        assertThat(tokens.hasNext(), is(false));
398    }
399
400    @Test
401    public void shouldConsumeTokenWithAnyValueConstant() {
402        makeCaseInsensitive();
403        // Unable to consume unless they all match ...
404        tokens.consume("SELECT", "ALL");
405        tokens.consume(TokenStream.ANY_VALUE);
406        tokens.consume("FROM", "THIS", "TABLE");
407        assertThat(tokens.hasNext(), is(false));
408    }
409
410    @Test
411    public void shouldReturnTrueFromCanConsumeMultipleTokensWithAnyValueConstant() {
412        makeCaseInsensitive();
413        // Unable to consume unless they all match ...
414        assertThat(tokens.canConsume("SELECT", "ALL", TokenStream.ANY_VALUE, "FRM", "THIS", "TABLE"), is(false));
415        assertThat(tokens.canConsume("SELECT", "ALL", "COLUMNS", "FROM", TokenStream.ANY_VALUE, "TABLE"), is(true));
416        assertThat(tokens.hasNext(), is(false));
417    }
418
419    @Test
420    public void shouldCanConsumeSingleAfterTokensCompleteFromCanConsumeStringList() {
421        makeCaseInsensitive();
422        // consume ALL the tokens using canConsume()
423        tokens.canConsume("SELECT", "ALL", "COLUMNS", "FROM", "THIS", "TABLE");
424        // try to canConsume() single word
425        assertThat(tokens.canConsume("SELECT"), is(false));
426        assertThat(tokens.canConsume(TokenStream.ANY_VALUE), is(false));
427        assertThat(tokens.canConsume(BasicTokenizer.SYMBOL), is(false));
428    }
429
430    @Test
431    public void shouldCanConsumeStringAfterTokensCompleteFromCanConsumeStringArray() {
432        makeCaseInsensitive();
433        // consume ALL the tokens using canConsume()
434        tokens.canConsume(new String[] {"SELECT", "ALL", "COLUMNS", "FROM", "THIS", "TABLE"});
435        // try to canConsume() single word
436        assertThat(tokens.canConsume("SELECT"), is(false));
437        assertThat(tokens.canConsume(TokenStream.ANY_VALUE), is(false));
438        assertThat(tokens.canConsume(BasicTokenizer.SYMBOL), is(false));
439    }
440
441    @Test
442    public void shouldCanConsumeStringAfterTokensCompleteFromCanConsumeStringIterator() {
443        makeCaseInsensitive();
444        // consume ALL the tokens using canConsume()
445        tokens.canConsume(Arrays.asList(new String[] {"SELECT", "ALL", "COLUMNS", "FROM", "THIS", "TABLE"}));
446        // try to canConsume() single word
447        assertThat(tokens.canConsume("SELECT"), is(false));
448        assertThat(tokens.canConsume(TokenStream.ANY_VALUE), is(false));
449        assertThat(tokens.canConsume(BasicTokenizer.SYMBOL), is(false));
450    }
451
452    @Test
453    public void shouldFindNextPositionStartIndex() {
454        makeCaseInsensitive();
455        // "Select all columns from this table";
456        tokens.consume();
457        // Next position should be line 1, column 8
458        assertThat(tokens.nextPosition().getIndexInContent(), is(7));
459        assertThat(tokens.nextPosition().getColumn(), is(8));
460        assertThat(tokens.nextPosition().getLine(), is(1));
461    }
462
463    @Test
464    public void shouldFindPreviousPositionStartIndex() {
465        makeCaseInsensitive();
466        // "Select all columns from this table";
467        tokens.consume();
468        tokens.consume();
469        // previous position should be line 1, column 8
470        assertThat(tokens.previousPosition().getIndexInContent(), is(7));
471        assertThat(tokens.previousPosition().getColumn(), is(8));
472        assertThat(tokens.previousPosition().getLine(), is(1));
473    }
474
475    @Test
476    public void shouldParseMultiLineString() {
477        makeCaseInsensitive();
478        String content = "ALTER DATABASE \n" + "DO SOMETHING; \n" + "ALTER DATABASE \n" + "      SET DEFAULT BIGFILE TABLESPACE;";
479        tokens = new TokenStream(content, tokenizer, true);
480        tokens.start();
481
482        tokens.consume(); // LINE
483        tokens.consume(); // ONE
484        tokens.consume(); // DO
485        tokens.consume(); // SOMETHING
486        tokens.consume(); // ;
487
488        assertThat(tokens.nextPosition().getIndexInContent(), is(31));
489        assertThat(tokens.nextPosition().getColumn(), is(1));
490        tokens.consume(); // ALTER
491        assertThat(tokens.nextPosition().getIndexInContent(), is(37));
492        assertThat(tokens.nextPosition().getColumn(), is(7));
493
494    }
495}