@Test public void testWhitespaceLine() throws Exception { FieldSet line = tokenizer.tokenize(" "); // whitespace counts as text assertEquals(1, line.getFieldCount()); }
@Test public void testAnotherTypeOfRecord() throws Exception { tokenizer.setColumns(new Range[] { new Range(1, 5), new Range(6, 15), new Range(16, 25), new Range(26, 27) }); // test another type of record line = "H2 123456 12345 12"; FieldSet tokens = tokenizer.tokenize(line); assertEquals(4, tokens.getFieldCount()); assertEquals(line.substring(0, 5).trim(), tokens.readString(0)); assertEquals(line.substring(5, 15).trim(), tokens.readString(1)); assertEquals(line.substring(15, 25).trim(), tokens.readString(2)); assertEquals(line.substring(25).trim(), tokens.readString(3)); }
@Test public void testDelimitedLineTokenizerChar() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" "); FieldSet line = tokenizer.tokenize("a b c"); assertEquals(3, line.getFieldCount()); }
@Test public void testNormalLength() throws Exception { tokenizer.setColumns(new Range[] { new Range(1, 10), new Range(11, 25), new Range(26, 30) }); // test shorter line as defined by record descriptor line = "H1 12345678 12345"; FieldSet tokens = tokenizer.tokenize(line); assertEquals(3, tokens.getFieldCount()); assertEquals(line.substring(0, 10).trim(), tokens.readString(0)); assertEquals(line.substring(10, 25).trim(), tokens.readString(1)); assertEquals(line.substring(25).trim(), tokens.readString(2)); }
@Test public void testMultiLineField() throws Exception { FieldSet line = tokenizer.tokenize("a,b,c\nrap"); assertEquals(3, line.getFieldCount()); assertEquals("c\nrap", line.readString(2)); }
@Test public void testTokenizeWithQuoteAndSpaceAtEnd() { FieldSet line = tokenizer.tokenize("a,b,\"c\" "); assertEquals(3, line.getFieldCount()); assertEquals("c", line.readString(2)); }
@Test public void testTokenizeWithQuotesEmptyValue() { FieldSet line = tokenizer.tokenize("\"a\",\"b\",\"\",\"d\""); assertEquals(4, line.getFieldCount()); assertEquals("", line.readString(2)); }
@Test public void testTokenizeWithSpaceAtEnd() { FieldSet line = tokenizer.tokenize("a,b,c "); assertEquals(3, line.getFieldCount()); assertEquals("c ", line.readRawString(2)); }
@Test public void testTokenizeWithNotDefaultQuotes() { tokenizer.setQuoteCharacter('\''); FieldSet line = tokenizer.tokenize("a,b,'c'"); assertEquals(3, line.getFieldCount()); assertEquals("c", line.readString(2)); }
@Test public void testTokenizeOverMultipleLines() { tokenizer = new DelimitedLineTokenizer(";"); FieldSet line = tokenizer.tokenize("value1;\"value2\nvalue2cont\";value3;value4"); assertEquals(4, line.getFieldCount()); assertEquals("value2\nvalue2cont", line.readString(1)); }
@Test public void testTokenizeWithIncludedFieldsAndNames() { tokenizer.setIncludedFields(new int[] {1,2}); tokenizer.setNames(new String[] {"foo", "bar"}); FieldSet line = tokenizer.tokenize("\"a\",\"b\",\"c\",\"d\""); assertEquals(2, line.getFieldCount()); assertEquals("c", line.readString("bar")); }
@Test(expected=IncorrectTokenCountException.class) public void testTokenizeWithIncludedFieldsAndTooManyNames() { tokenizer.setIncludedFields(new int[] {1,2}); tokenizer.setNames(new String[] {"foo", "bar", "spam"}); FieldSet line = tokenizer.tokenize("\"a\",\"b\",\"c\",\"d\""); assertEquals(2, line.getFieldCount()); assertEquals("c", line.readString("bar")); }
@Test(expected=IncorrectTokenCountException.class) public void testTokenizeWithIncludedFieldsAndTooFewNames() { tokenizer.setIncludedFields(new int[] {1,2}); tokenizer.setNames(new String[] {"foo"}); FieldSet line = tokenizer.tokenize("\"a\",\"b\",\"c\",\"d\""); assertEquals(2, line.getFieldCount()); assertEquals("c", line.readString("bar")); }
@Test public void testTokenizeWithDelimiterAtEnd() { FieldSet line = tokenizer.tokenize("a,b,c,"); assertEquals(4, line.getFieldCount()); assertEquals("c", line.readString(2)); assertEquals("", line.readString(3)); }
@Test public void testDelimitedLineTokenizerNewlineToken() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer("\n"); FieldSet line = tokenizer.tokenize("a b\n c"); assertEquals(2, line.getFieldCount()); assertEquals("a b", line.readString(0)); assertEquals("c", line.readString(1)); }
@Test public void testDelimitedLineTokenizerStringsOverlapWithoutSeparation() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | "); FieldSet line = tokenizer.tokenize("a | | b"); assertEquals(2, line.getFieldCount()); assertEquals("a", line.readString(0)); assertEquals("| b", line.readString(1)); }
@Test public void testDelimitedLineTokenizerString() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" b "); FieldSet line = tokenizer.tokenize("a b c"); assertEquals(2, line.getFieldCount()); assertEquals("a", line.readString(0)); assertEquals("c", line.readString(1)); }
@Test public void testDelimitedLineTokenizerStringBeginningOfLine() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | "); FieldSet line = tokenizer.tokenize(" | a | b"); assertEquals(3, line.getFieldCount()); assertEquals("", line.readString(0)); assertEquals("a", line.readString(1)); assertEquals("b", line.readString(2)); }
@Test public void testDelimitedLineTokenizerStringEndOfLine() { AbstractLineTokenizer tokenizer = new DelimitedLineTokenizer(" | "); FieldSet line = tokenizer.tokenize("a | b | "); assertEquals(3, line.getFieldCount()); assertEquals("a", line.readString(0)); assertEquals("b", line.readString(1)); assertEquals("", line.readString(2)); }
@Test public void testTokenizeRegularUse() { tokenizer.setColumns(new Range[] { new Range(1, 2), new Range(3, 7), new Range(8, 12) }); // test shorter line as defined by record descriptor line = "H11234512345"; FieldSet tokens = tokenizer.tokenize(line); assertEquals(3, tokens.getFieldCount()); assertEquals("H1", tokens.readString(0)); assertEquals("12345", tokens.readString(1)); assertEquals("12345", tokens.readString(2)); }