/** * Returns the count since have read bytes. * @return the count */ public long getCount() { return counter.get(); }
/** * Returns the count since have read bytes. * @return the count */ public long getCount() { return counter.get(); }
@Override public float getProgress() throws IOException, InterruptedException { if (closed) { return 1.0f; } float progress = (float) (sizeCounter.get() / fragmentSize); return Math.min(progress, 0.99f); }
Constants.putCounts(context, sourceId, outputId, 1, records, counter.get());
/** * simple input. * @throws Exception if failed */ @Test public void input() throws Exception { put(new File(mapping, "input/file.txt"), "Hello, world!"); profile.setMinimumFragmentSize(-1); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); List<DirectInputFragment> fragments = core.findInputFragments( definition, "input", FilePattern.compile("**")); assertThat(fragments.size(), is(1)); List<String> results = consume(core, fragments); assertThat(counter.get(), is(greaterThan(0L))); assertThat(results.size(), is(1)); assertThat(results, hasItem("Hello, world!")); }
/** * simple output. * @throws Exception if failed */ @Test public void output() throws Exception { HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); try (ModelOutput<StringBuilder> output = core.openOutput( context, definition, "output", "file.txt", counter)){ output.write(new StringBuilder("Hello, world!")); } assertThat(counter.get(), is(greaterThan(0L))); File target = new File(mapping, "output/file.txt"); assertThat(target.exists(), is(false)); commitAttempt(core); assertThat(target.exists(), is(false)); commitTransaction(core); assertThat(target.exists(), is(true)); assertThat(get(target), is(Arrays.asList("Hello, world!"))); }
private long delete(DirectIoPath dpath) { try { Counter counter = new Counter(); if (dpath.getResourcePattern().isPresent()) { dpath.getSource().getEntity().delete( dpath.getComponentPath().getPathString(), dpath.getResourcePattern().get(), recursive, counter); } else { dpath.getSource().getEntity().delete( BasePath.EMPTY.getPathString(), dpath.getComponentPath().asFilePattern(), recursive, counter); } return counter.get(); } catch (IOException | InterruptedException e) { throw new CommandExecutionException(MessageFormat.format( "error occurred while deleting file: {0}", dpath.getBarePath()), e); } } }
/** * output without streaming. * @throws Exception if failed */ @Test public void output_nostreaming() throws Exception { profile.setOutputStreaming(false); profile.getLocalFileSystem().getConf().set( HadoopDataSourceUtil.KEY_LOCAL_TEMPDIR, localtemp.getPath()); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); try (ModelOutput<StringBuilder> output = core.openOutput( context, definition, "output", "file.txt", counter)) { output.write(new StringBuilder("Hello, world!")); } assertThat(counter.get(), is(greaterThan(0L))); File target = new File(mapping, "output/file.txt"); assertThat(target.exists(), is(false)); commitAttempt(core); assertThat(target.exists(), is(false)); commitTransaction(core); assertThat(target.exists(), is(true)); assertThat(get(target), is(Arrays.asList("Hello, world!"))); }
/** * input multiple records. * @throws Exception if failed */ @Test public void input_multirecord() throws Exception { put(new File(mapping, "input/file.txt"), "Hello1", "Hello2", "Hello3"); profile.setMinimumFragmentSize(-1); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); List<DirectInputFragment> fragments = core.findInputFragments( definition, "input", FilePattern.compile("**")); assertThat(fragments.size(), is(1)); List<String> results = consume(core, fragments); assertThat(counter.get(), is(greaterThan(0L))); assertThat(results.size(), is(3)); assertThat(results, hasItem("Hello1")); assertThat(results, hasItem("Hello2")); assertThat(results, hasItem("Hello3")); }
/** * output without staging. * @throws Exception if failed */ @Test public void output_nostaging() throws Exception { profile.setOutputStaging(false); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); try (ModelOutput<StringBuilder> output = core.openOutput( context, definition, "output", "file.txt", counter)) { output.write(new StringBuilder("Hello, world!")); } assertThat(counter.get(), is(greaterThan(0L))); File target = new File(mapping, "output/file.txt"); assertThat(target.exists(), is(false)); commitAttempt(core); assertThat(target.exists(), is(true)); commitTransaction(core); assertThat(target.exists(), is(true)); assertThat(get(target), is(Arrays.asList("Hello, world!"))); }
/** * input multiple files. * @throws Exception if failed */ @Test public void input_multifile() throws Exception { put(new File(mapping, "input/file1.txt"), "Hello1"); put(new File(mapping, "input/file2.txt"), "Hello2"); put(new File(mapping, "input/file3.txt"), "Hello3"); profile.setMinimumFragmentSize(-1); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); List<DirectInputFragment> fragments = core.findInputFragments( definition, "input", FilePattern.compile("**")); assertThat(fragments.size(), is(3)); List<String> results = consume(core, fragments); assertThat(counter.get(), is(greaterThan(0L))); assertThat(results.size(), is(3)); assertThat(results, hasItem("Hello1")); assertThat(results, hasItem("Hello2")); assertThat(results, hasItem("Hello3")); }
/** * output without streaming nor staging. * @throws Exception if failed */ @Test public void output_nomove() throws Exception { profile.setOutputStaging(false); profile.setOutputStreaming(false); profile.getLocalFileSystem().getConf().set( HadoopDataSourceUtil.KEY_LOCAL_TEMPDIR, localtemp.getPath()); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); try (ModelOutput<StringBuilder> output = core.openOutput( context, definition, "output", "file.txt", counter)) { output.write(new StringBuilder("Hello, world!")); } assertThat(counter.get(), is(greaterThan(0L))); File target = new File(mapping, "output/file.txt"); assertThat(target.exists(), is(false)); commitAttempt(core); assertThat(target.exists(), is(true)); commitTransaction(core); assertThat(target.exists(), is(true)); assertThat(get(target), is(Arrays.asList("Hello, world!"))); }
@SuppressWarnings({ "unchecked", "rawtypes" }) @Override protected void reduce( AbstractDirectOutputKey key, Iterable<AbstractDirectOutputValue> values, Context context) throws IOException , InterruptedException { DirectOutputGroup group = (DirectOutputGroup) key.getGroupObject().getObject(); String portId = group.getOutputId(); String path = variables.parse(group.getPath(), false); String sourceId = repository.getRelatedId(path); OutputAttemptContext outputContext = BridgeOutputFormat.createContext(context, sourceId); DataDefinition definition = SimpleDataDefinition.newInstance( group.getDataType(), configure(context, group.getFormat())); DirectDataSource datasource = repository.getRelatedDataSource(path); String basePath = repository.getComponentPath(path); String resourcePath = variables.parse(group.getResourcePath()); Counter counter = new Counter(); long records = 0; try (ModelOutput output = datasource.openOutput(outputContext, definition, basePath, resourcePath, counter)) { for (Union union : values) { Object object = union.getObject(); output.write(object); records++; } } recordCounter.increment(records); Constants.putCounts(context, sourceId, portId, 1, records, counter.get()); }