private ModelInput<Object> reader( BinaryStreamFormat<Object> format, byte[] contents, long fragmentSize) throws IOException, InterruptedException { return format.createInput( format.getSupportedType(), "testinig", new ByteArrayInputStream(contents), 0, fragmentSize); }
private ModelOutput<Object> writer( BinaryStreamFormat<Object> format, OutputStream output) throws IOException, InterruptedException { return format.createOutput(format.getSupportedType(), "testing", output); }
@Override public long getMinimumFragmentSize() throws IOException, InterruptedException { return streamFormat.getMinimumFragmentSize(); }
BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("csv", "TupleCsvFormat"); assertThat(support.getSupportedType(), is((Object) model.unwrap().getClass())); try (ModelOutput<Object> writer = unsafe.createOutput(model.unwrap().getClass(), "hello", output)) { for (int line = 0; line < 100; line++) { ModelWrapper buffer = loaded.newModel("Tuple"); InputStream in = new ByteArrayInputStream(bytes, offset, bytes.length - offset); in.mark(bytes.length - offset); try (ModelInput<Object> reader = unsafe.createInput(model.unwrap().getClass(), "hello", in, offset, length)) { while (true) {
/** * simple case. * @throws Exception if failed */ @Test public void simple() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.tabular", "simple = { value : TEXT; };", }); ModelWrapper model = loaded.newModel("Simple") .setOption("value", new StringOption("Hello, world!")); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleTabularTextFormat"); assertThat(support.getSupportedType(), is((Object) model.unwrap().getClass())); assertThat(support.getMinimumFragmentSize(), is(not(-1L))); BinaryStreamFormat<Object> unsafe = unsafe(support); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = unsafe.createOutput(unsafe.getSupportedType(), "hello", output)) { writer.write(model.unwrap()); } assertThat(text(output), is("Hello, world!\n")); Object buffer = loaded.newModel("Simple").unwrap(); try (ModelInput<Object> reader = unsafe.createInput(unsafe.getSupportedType(), "hello", in(output))) { assertThat(reader.readTo(buffer), is(true)); assertThat(buffer, is(model.unwrap())); assertThat(reader.readTo(buffer), is(false)); } }
/** * fragmentation is restricted. * @throws Exception if failed */ @Test public void fragmentation_restricted() throws Exception { ModelLoader loaded = generateJava("fragmentation_restricted"); ModelWrapper model = loaded.newModel("Tuple"); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("csv", "TupleCsvFormat"); BinaryStreamFormat<Object> unsafe = unsafe(support); model.set("f1", new Text("Hello1")); model.set("f2", new Text("Hello1")); model.set("f3", new Text("Hello1")); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = unsafe.createOutput(model.unwrap().getClass(), "hello", output)) { writer.write(model.unwrap()); } try { unsafe.createInput(model.unwrap().getClass(), "hello", in(output), 1, size(output)); fail(); } catch (Exception e) { // ok. } }
/** * with file name. * @throws Exception if failed */ @Test public void file_name() throws Exception { ModelLoader loaded = generateJava("file_name"); ModelWrapper model = loaded.newModel("Model"); ModelWrapper buffer = loaded.newModel("Model"); BinaryStreamFormat<Object> support = unsafe(loaded.newObject("csv", "ModelCsvFormat")); assertThat(support.getMinimumFragmentSize(), is(greaterThan(0L))); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = support.createOutput(model.unwrap().getClass(), "hello", output)) { model.set("value", new Text("Hello, world!")); writer.write(model.unwrap()); } try (ModelInput<Object> reader = support.createInput(model.unwrap().getClass(), "testing", in(output), 0, size(output))) { assertThat(reader.readTo(buffer.unwrap()), is(true)); assertThat(buffer.getOption("value"), is((Object) new StringOption("Hello, world!"))); assertThat(buffer.getOption("path"), is((Object) new StringOption("testing"))); assertThat(reader.readTo(buffer.unwrap()), is(false)); } }
/** * Creates a new {@link ModelInput} for the specified properties. * @param dataType the target data type * @param path the path about the target stream (for label) * @param stream the target stream * @return the created reader * @throws IOException if failed to create reader * @throws InterruptedException if interrupted * @throws IllegalArgumentException if this does not support target property sequence, * or any parameter is {@code null} * @since 0.9.1 */ public ModelInput<T> createInput( Class<? extends T> dataType, String path, InputStream stream) throws IOException, InterruptedException { return createInput(dataType, path, stream, 0L, -1L); }
cstream = new CountOutputStream(stream, counter); ModelOutput<T> output = streamFormat.createOutput(dataType, path.toString(), cstream); succeed = true; return output;
@Override public Class<T> getSupportedType() { return streamFormat.getSupportedType(); }
@Override public long getPreferredFragmentSize() throws IOException, InterruptedException { return streamFormat.getPreferredFragmentSize(); }
/** * unlimited range. * @throws Exception if failed */ @Test public void unlimited() throws Exception { ModelLoader loaded = generateJava("simple"); ModelWrapper model = loaded.newModel("Simple"); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("csv", "SimpleCsvFormat"); assertThat(support.getSupportedType(), is((Object) model.unwrap().getClass())); BinaryStreamFormat<Object> unsafe = unsafe(support); assertThat(unsafe, is(not(instanceOf(Configurable.class)))); model.set("value", new Text("hello-world")); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = unsafe.createOutput(model.unwrap().getClass(), "hello", output)) { writer.write(model.unwrap()); } assertThat(scan(output.toByteArray()), is(Arrays.asList("hello-world"))); Object buffer = loaded.newModel("Simple").unwrap(); try (ModelInput<Object> reader = unsafe.createInput(model.unwrap().getClass(), "hello", in(output), 0, -1)) { assertThat(reader.readTo(buffer), is(true)); assertThat(buffer, is(model.unwrap())); assertThat(reader.readTo(buffer), is(false)); } }
/** * simple case. * @throws Exception if failed */ @Test public void simple() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.text.csv", "simple = { value : TEXT; };", }); ModelWrapper model = loaded.newModel("Simple") .setOption("value", new StringOption("Hello, world!")); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("text", "SimpleCsvTextFormat"); assertThat(support.getSupportedType(), is((Object) model.unwrap().getClass())); assertThat(support.getMinimumFragmentSize(), is(not(-1L))); BinaryStreamFormat<Object> unsafe = unsafe(support); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = unsafe.createOutput(unsafe.getSupportedType(), "hello", output)) { writer.write(model.unwrap()); } assertThat(text(output), is("\"Hello, world!\"\r\n")); Object buffer = loaded.newModel("Simple").unwrap(); try (ModelInput<Object> reader = unsafe.createInput(unsafe.getSupportedType(), "hello", in(output))) { assertThat(reader.readTo(buffer), is(true)); assertThat(buffer, is(model.unwrap())); assertThat(reader.readTo(buffer), is(false)); } }
/** * with ignored property. * @throws Exception if failed */ @Test public void ignore() throws Exception { ModelLoader loaded = generateJava("ignore"); ModelWrapper model = loaded.newModel("Model"); model.set("value", new Text("Hello, world!")); model.set("ignored", new Text("ignored")); ModelWrapper buffer = loaded.newModel("Model"); BinaryStreamFormat<Object> support = unsafe(loaded.newObject("csv", "ModelCsvFormat")); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = support.createOutput(model.unwrap().getClass(), "hello", output)) { writer.write(model.unwrap()); } try (ModelInput<Object> reader = support.createInput(model.unwrap().getClass(), "testing", in(output), 0, size(output))) { assertThat(reader.readTo(buffer.unwrap()), is(true)); assertThat(buffer.getOption("value"), is((Object) new StringOption("Hello, world!"))); assertThat(buffer.getOption("ignored"), is((Object) new StringOption())); assertThat(reader.readTo(buffer.unwrap()), is(false)); } }
private static <T> DataModelSourceFactory load0( DataModelDefinition<T> definition, BinaryStreamFormat<? super T> format, File source) throws IOException, InterruptedException { String path = source.toURI().toString(); try (InputStream stream = new FileInputStream(source); ModelInput<? super T> input = format.createInput(definition.getModelClass(), path, stream)) { return collect(definition, input); } }
/** * with force header. * @throws Exception if failed */ @Test public void force_header() throws Exception { ModelLoader loaded = generateJava("force_header"); ModelWrapper model = loaded.newModel("Model"); BinaryStreamFormat<Object> support = unsafe(loaded.newObject("csv", "ModelCsvFormat")); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = support.createOutput(model.unwrap().getClass(), "hello", output)) { model.set("value", new Text("Hello, world!")); writer.write(model.unwrap()); } String[][] results = parse(1, new String(output.toByteArray(), "UTF-8")); assertThat(results, is(new String[][] { {"title"}, {"Hello, world!"}, })); }
private void check( ModelLoader loader, String name, BinaryStreamFormat<Object> format, long fragmentSize) throws IOException, InterruptedException { ModelWrapper model = loader.newModel(name); assertThat(format.getSupportedType(), equalTo((Object) model.getModelClass())); model.set("value", new Text("Hello, world!")); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = writer(format, output)) { writer.write(model.unwrap()); } Object buffer = loader.newModel(name).unwrap(); try (ModelInput<Object> reader = reader(format, output.toByteArray(), fragmentSize)) { assertThat(reader.readTo(buffer), is(true)); assertThat(buffer, is(model.unwrap())); assertThat(reader.readTo(buffer), is(false)); assertThat(buffer, is(model.unwrap())); } }
private void read(ModelLoader loaded, String contents, Consumer<ModelWrapper> tester) { BinaryStreamFormat<Object> unsafe = unsafe(loaded.newObject(SEGMENT, "SimpleJsonFormat")); ModelWrapper wrapper = loaded.newModel("Simple"); Object buffer = wrapper.unwrap(); try (ModelInput<Object> reader = unsafe.createInput( unsafe.getSupportedType(), "testing", new ByteArrayInputStream(contents.getBytes(StandardCharsets.UTF_8)))) { assertThat(reader.readTo(buffer), is(true)); tester.accept(wrapper); assertThat(reader.readTo(buffer), is(false)); } catch (IOException | InterruptedException e) { throw new AssertionError(e); } }
/** * With compression. * @throws Exception if failed */ @Test public void compression() throws Exception { ModelLoader loaded = generateJava("compress"); ModelWrapper model = loaded.newModel("Compress"); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject("csv", "CompressCsvFormat"); assertThat(support.getSupportedType(), is((Object) model.unwrap().getClass())); BinaryStreamFormat<Object> unsafe = unsafe(support); assertThat(unsafe, is(instanceOf(Configurable.class))); model.set("value", new Text("hello")); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = unsafe.createOutput(model.unwrap().getClass(), "hello", output)) { writer.write(model.unwrap()); } assertThat( scan(dump(new GZIPInputStream(new ByteArrayInputStream(output.toByteArray())))), is(Arrays.asList("hello"))); Object buffer = loaded.newModel("Compress").unwrap(); try (ModelInput<Object> reader = unsafe.createInput(model.unwrap().getClass(), "hello", in(output), 0, size(output))) { assertThat(reader.readTo(buffer), is(true)); assertThat(buffer, is(model.unwrap())); assertThat(reader.readTo(buffer), is(false)); } }
/** * simple case. * @throws Exception if failed */ @Test public void simple() throws Exception { ModelLoader loaded = generateJavaFromLines(new String[] { "@directio.json", "simple = { value : TEXT; };", }); ModelWrapper model = loaded.newModel("Simple") .setOption("value", new StringOption("Hello, world!")); BinaryStreamFormat<?> support = (BinaryStreamFormat<?>) loaded.newObject(SEGMENT, "SimpleJsonFormat"); assertThat(support.getSupportedType(), is((Object) model.unwrap().getClass())); assertThat(support.getMinimumFragmentSize(), is(lessThan(0L))); BinaryStreamFormat<Object> unsafe = unsafe(support); ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ModelOutput<Object> writer = unsafe.createOutput(unsafe.getSupportedType(), "hello", output)) { writer.write(model.unwrap()); } assertThat(parse(output.toByteArray()), contains(map("value", "'Hello, world!'"))); Object buffer = loaded.newModel("Simple").unwrap(); try (ModelInput<Object> reader = unsafe.createInput(unsafe.getSupportedType(), "hello", in(output))) { assertThat(reader.readTo(buffer), is(true)); assertThat(buffer, is(model.unwrap())); assertThat(reader.readTo(buffer), is(false)); } }