@Override public long getMinimumFragmentSize() throws IOException, InterruptedException { return streamFormat.getMinimumFragmentSize(); }
/** * w/ {@code charset}. * @throws Exception if failed */ @Test public void charset() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.csv(", " charset = 'UTF-16LE',", ")", "simple = {", " a : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleCsvTextFormat"); assertThat(support.getMinimumFragmentSize(), is(-1L)); ModelWrapper model = loaded.newModel("Simple") .setOption("a", new StringOption("Hello, world!")); byte[] contents = write(loaded, model); assertThat(new String(contents, StandardCharsets.UTF_16LE), is("\"Hello, world!\"\r\n")); }
/** * w/ {@code charset}. * @throws Exception if failed */ @Test public void charset() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.tabular(", " charset = 'UTF-16LE',", ")", "simple = {", " a : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleTabularTextFormat"); assertThat(support.getMinimumFragmentSize(), is(-1L)); ModelWrapper model = loaded.newModel("Simple") .setOption("a", new StringOption("Hello, world!")); byte[] contents = write(loaded, model); assertThat(new String(contents, StandardCharsets.UTF_16LE), is("Hello, world!\n")); }
/** * w/ {@code directio.json.line_number}. * @throws Exception if failed */ @Test public void field_line_number_int() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.json(", " format = jsonl,", ")", "simple = {", " a : TEXT;", " @directio.json.line_number", " b : INT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject(SEGMENT, "SimpleJsonFormat"); assertThat(support.getMinimumFragmentSize(), is(-1L)); read(loaded, "\n\n{'a':'Hello, world!'}\n\n", wrapper -> { assertThat(wrapper.getOption("a"), is(new StringOption("Hello, world!"))); assertThat(wrapper.getOption("b"), is(new IntOption(3))); }); }
/** * w/ {@code directio.json.line_number}. * @throws Exception if failed */ @Test public void field_line_number() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.json(", " format = jsonl,", ")", "simple = {", " a : TEXT;", " @directio.json.line_number", " b : LONG;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject(SEGMENT, "SimpleJsonFormat"); assertThat(support.getMinimumFragmentSize(), is(-1L)); read(loaded, "\n\n{'a':'Hello, world!'}\n\n", wrapper -> { assertThat(wrapper.getOption("a"), is(new StringOption("Hello, world!"))); assertThat(wrapper.getOption("b"), is(new LongOption(3))); }); }
/** * w/ {@code directio.json.record_number}. * @throws Exception if failed */ @Test public void field_record_number() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.json(", " format = jsonl,", ")", "simple = {", " a : TEXT;", " @directio.json.record_number", " b : LONG;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject(SEGMENT, "SimpleJsonFormat"); assertThat(support.getMinimumFragmentSize(), is(-1L)); read(loaded, "\n\n{'a':'Hello, world!'}\n\n", wrapper -> { assertThat(wrapper.getOption("a"), is(new StringOption("Hello, world!"))); assertThat(wrapper.getOption("b"), is(new LongOption(1))); }); }
/** * w/ {@code allow_linefeed}. * @throws Exception if failed */ @Test public void allow_linefeed() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.csv(", " allow_linefeed = true,", ")", "simple = {", " a : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleCsvTextFormat"); assertThat(support.getMinimumFragmentSize(), is(-1L)); ModelWrapper model = loaded.newModel("Simple") .setOption("a", new StringOption("Hello\nworld!")); byte[] contents = restore(loaded, model); assertThat(text(contents), is("\"Hello\nworld!\"\r\n")); }
/** * w/ {@code directio.json.record_number}. * @throws Exception if failed */ @Test public void field_record_number_int() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.json(", " format = jsonl,", ")", "simple = {", " a : TEXT;", " @directio.json.record_number", " b : INT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject(SEGMENT, "SimpleJsonFormat"); assertThat(support.getMinimumFragmentSize(), is(-1L)); read(loaded, "\n\n{'a':'Hello, world!'}\n\n", wrapper -> { assertThat(wrapper.getOption("a"), is(new StringOption("Hello, world!"))); assertThat(wrapper.getOption("b"), is(new IntOption(1))); }); }
/** * with file name. * @throws Exception if failed */ @Test public void file_name() throws Exception { ModelLoader loaded = generateJava("file_name"); ModelWrapper model = loaded.newModel("Model"); ModelWrapper buffer = loaded.newModel("Model"); BinaryStreamFormat<Object> support = unsafe(loaded.newObject("csv", "ModelCsvFormat")); assertThat(support.getMinimumFragmentSize(), is(greaterThan(0L))); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = support.createOutput(model.unwrap().getClass(), "hello", output)) { model.set("value", new Text("Hello, world!")); writer.write(model.unwrap()); } try (ModelInput<Object> reader = support.createInput(model.unwrap().getClass(), "testing", in(output), 0, size(output))) { assertThat(reader.readTo(buffer.unwrap()), is(true)); assertThat(buffer.getOption("value"), is((Object) new StringOption("Hello, world!"))); assertThat(buffer.getOption("path"), is((Object) new StringOption("testing"))); assertThat(reader.readTo(buffer.unwrap()), is(false)); } }
/** * w/ {@code directio.text.field}. * @throws Exception if failed */ @Test public void field_field() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.tabular", "simple = {", " @directio.text.field", " a : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleTabularTextFormat"); assertThat(support.getMinimumFragmentSize(), is(not(-1L))); byte[] contents = write(loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A"))); read(contents, loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A"))); assertThat(text(contents), is("A\n")); }
/** * w/ {@code directio.text.field}. * @throws Exception if failed */ @Test public void field_field() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.csv", "simple = {", " @directio.text.field", " a : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleCsvTextFormat"); assertThat(support.getMinimumFragmentSize(), is(not(-1L))); byte[] contents = write(loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A"))); read(contents, loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A"))); assertThat(text(contents), is("A\r\n")); }
/** * w/ format. * @throws Exception if failed */ @Test public void format() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.json(", " format = jsonl,", ")", "simple = {", " value : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject(SEGMENT, "SimpleJsonFormat"); assertThat(support.getMinimumFragmentSize(), is(greaterThan(0L))); byte[] contents = restore(loaded, loaded.newModel("Simple").setOption("value", new StringOption("Hello, world!"))); assertThat(parse(contents), contains(map("value", "'Hello, world!'"))); }
/** * w/ {@code directio.text.record_number}. * @throws Exception if failed */ @Test public void field_record_number() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.csv(", " input_transformer = '" + EmptyLineFilter.class.getName() + "'", ")", "simple = {", " a : TEXT;", " @directio.text.record_number", " b : LONG;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleCsvTextFormat"); assertThat(support.getMinimumFragmentSize(), is(-1L)); byte[] contents = write(loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new LongOption(-1))); assertThat(text(contents), is("A\r\n")); read(loaded, "\n\nHello!\n\n", wrapper -> { assertThat(wrapper.getOption("a"), is(new StringOption("Hello!"))); assertThat(wrapper.getOption("b"), is(new LongOption(1))); }); }
/** * w/ {@code directio.text.record_number}. * @throws Exception if failed */ @Test public void field_record_number() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.tabular(", " input_transformer = '" + EmptyLineFilter.class.getName() + "'", ")", "simple = {", " a : TEXT;", " @directio.text.record_number", " b : LONG;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleTabularTextFormat"); assertThat(support.getMinimumFragmentSize(), is(-1L)); byte[] contents = write(loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new LongOption(-1))); assertThat(text(contents), is("A\n")); read(loaded, "\n\nHello, world!\n\n", wrapper -> { assertThat(wrapper.getOption("a"), is(new StringOption("Hello, world!"))); assertThat(wrapper.getOption("b"), is(new LongOption(1))); }); }
/** * w/ {@code directio.text.ignore}. * @throws Exception if failed */ @Test public void field_ignore() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.csv", "simple = {", " a : TEXT;", " @directio.text.ignore", " b : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleCsvTextFormat"); assertThat(support.getMinimumFragmentSize(), is(not(-1L))); byte[] contents = write(loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new StringOption("B"))); read(contents, loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new StringOption())); assertThat(text(contents), is("A\r\n")); }
/** * w/ {@code directio.text.ignore}. * @throws Exception if failed */ @Test public void field_ignore() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.tabular", "simple = {", " a : TEXT;", " @directio.text.ignore", " b : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleTabularTextFormat"); assertThat(support.getMinimumFragmentSize(), is(not(-1L))); byte[] contents = write(loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new StringOption("B"))); read(contents, loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new StringOption())); assertThat(text(contents), is("A\n")); }
/** * w/ {@code directio.text.file_name}. * @throws Exception if failed */ @Test public void field_file_name() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.tabular", "simple = {", " a : TEXT;", " @directio.text.file_name", " b : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleTabularTextFormat"); assertThat(support.getMinimumFragmentSize(), is(not(-1L))); byte[] contents = write(loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new StringOption("B"))); read(contents, loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new StringOption("testing"))); assertThat(text(contents), is("A\n")); }
/** * w/ {@code directio.text.file_name}. * @throws Exception if failed */ @Test public void field_file_name() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.csv", "simple = {", " a : TEXT;", " @directio.text.file_name", " b : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleCsvTextFormat"); assertThat(support.getMinimumFragmentSize(), is(not(-1L))); byte[] contents = write(loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new StringOption("B"))); read(contents, loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A")) .setOption("b", new StringOption("testing"))); assertThat(text(contents), is("A\r\n")); }
/** * w/ {@code directio.json.field}. * @throws Exception if failed */ @Test public void field_field() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.json(", " format = jsonl,", ")", "simple = {", " @directio.json.field", " a : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject(SEGMENT, "SimpleJsonFormat"); assertThat(support.getMinimumFragmentSize(), is(not(-1L))); byte[] contents = write(loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A"))); read(contents, loaded, loaded.newModel("Simple") .setOption("a", new StringOption("A"))); assertThat(parse(contents), contains(map("a", "'A'"))); }
/** * charset may suppress splitting input. * @throws Exception if failed */ @Test public void charset_suppress_splitting() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.json(", " format = jsonl,", " charset = 'UTF-16LE',", ")", "simple = {", " value : TEXT;", "};", }); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject(SEGMENT, "SimpleJsonFormat"); assertThat(support.getMinimumFragmentSize(), is(lessThan(0L))); byte[] contents = restore(loaded, loaded.newModel("Simple").setOption("value", new StringOption("Hello, world!"))); assertThat(parse(contents, StandardCharsets.UTF_16LE), contains(map("value", "'Hello, world!'"))); }