private OutputAttemptContext createOutputContext() { String tx = UUID.randomUUID().toString(); String attempt = UUID.randomUUID().toString(); return new OutputAttemptContext(tx, attempt, id, new Counter()); }
/** * Returns the count since have read bytes. * @return the count */ public long getCount() { return counter.get(); }
private void advanceCounter(long nextCount) { long deltaCount = nextCount - lastCount; if (deltaCount > 0) { counter.add(deltaCount); lastCount = nextCount; } }
/** * Adds count. * @param delta count delta */ public void add(long delta) { entity.add(delta); onChanged(); }
private long delete(DirectIoPath dpath) { try { Counter counter = new Counter(); if (dpath.getResourcePattern().isPresent()) { dpath.getSource().getEntity().delete( dpath.getComponentPath().getPathString(), dpath.getResourcePattern().get(), recursive, counter); } else { dpath.getSource().getEntity().delete( BasePath.EMPTY.getPathString(), dpath.getComponentPath().asFilePattern(), recursive, counter); } return counter.get(); } catch (IOException | InterruptedException e) { throw new CommandExecutionException(MessageFormat.format( "error occurred while deleting file: {0}", dpath.getBarePath()), e); } } }
/** * Creates output context from execution ID and datasource ID. * @param executionId current execution ID * @param datasourceId target datasource ID * @return output context * @throws IllegalArgumentException if some parameters were {@code null} */ public static OutputTransactionContext createContext(String executionId, String datasourceId) { if (executionId == null) { throw new IllegalArgumentException("executionId must not be null"); //$NON-NLS-1$ } if (datasourceId == null) { throw new IllegalArgumentException("datasourceId must not be null"); //$NON-NLS-1$ } return new OutputTransactionContext(executionId, datasourceId, new Counter()); }
@SuppressWarnings({ "unchecked", "rawtypes" }) @Override protected void reduce( AbstractDirectOutputKey key, Iterable<AbstractDirectOutputValue> values, Context context) throws IOException , InterruptedException { DirectOutputGroup group = (DirectOutputGroup) key.getGroupObject().getObject(); String portId = group.getOutputId(); String path = variables.parse(group.getPath(), false); String sourceId = repository.getRelatedId(path); OutputAttemptContext outputContext = BridgeOutputFormat.createContext(context, sourceId); DataDefinition definition = SimpleDataDefinition.newInstance( group.getDataType(), configure(context, group.getFormat())); DirectDataSource datasource = repository.getRelatedDataSource(path); String basePath = repository.getComponentPath(path); String resourcePath = variables.parse(group.getResourcePath()); Counter counter = new Counter(); long records = 0; try (ModelOutput output = datasource.openOutput(outputContext, definition, basePath, resourcePath, counter)) { for (Union union : values) { Object object = union.getObject(); output.write(object); records++; } } recordCounter.increment(records); Constants.putCounts(context, sourceId, portId, 1, records, counter.get()); }
private void advanceCounter() { double last = lastBytes; double next = last + averageBytesPerRecord; long delta = (long) (next - last); if (delta >= 0L) { counter.add(delta); } lastBytes = next; }
/** * Returns the count since have read bytes. * @return the count */ public long getCount() { return counter.get(); }
/** * Truncates the current target. * @param resourcePattern the target resource pattern * @throws IOException if failed to perform by I/O error * @since 0.7.3 */ public void truncate(String resourcePattern) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( "Truncating Direct I/O resources: {0}:{1} (id={2})", //$NON-NLS-1$ fullPath, resourcePattern, id)); } try { dataSource.delete(basePath, FilePattern.compile(resourcePattern), true, new Counter()); } catch (InterruptedException e) { throw (IOException) new InterruptedIOException("interrupted").initCause(e); //$NON-NLS-1$ } }
@Override public int read() throws IOException { int read = target.read(); if (read > 0) { counter.add(1); } return read; }
@Override public float getProgress() throws IOException, InterruptedException { if (closed) { return 1.0f; } float progress = (float) (sizeCounter.get() / fragmentSize); return Math.min(progress, 0.99f); }
private static <T> DataModelSourceFactory load0( DataModelDefinition<T> definition, HadoopFileFormat<? super T> format, File source) throws IOException, InterruptedException { try (ModelInput<? super T> input = format.createInput( definition.getModelClass(), org.apache.hadoop.fs.FileSystem.getLocal(format.getConf()), new org.apache.hadoop.fs.Path(source.toURI()), new Counter())) { return collect(definition, input); } }
@Override public void write(byte[] b) throws IOException { stream.write(b); counter.add(b.length); }
/** * simple input. * @throws Exception if failed */ @Test public void input() throws Exception { put(new File(mapping, "input/file.txt"), "Hello, world!"); profile.setMinimumFragmentSize(-1); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); List<DirectInputFragment> fragments = core.findInputFragments( definition, "input", FilePattern.compile("**")); assertThat(fragments.size(), is(1)); List<String> results = consume(core, fragments); assertThat(counter.get(), is(greaterThan(0L))); assertThat(results.size(), is(1)); assertThat(results, hasItem("Hello, world!")); }
DirectOutputSink( DataModelDefinition<T> definition, HadoopFileFormat<? super T> format, org.apache.hadoop.fs.Path destination) throws IOException, InterruptedException { this.definition = definition; this.output = format.createOutput( definition.getModelClass(), org.apache.hadoop.fs.FileSystem.getLocal(format.getConf()), destination, new Counter()); }
@Override public int read(byte[] b) throws IOException { int read = target.read(b); if (read > 0) { counter.add(read); } return read; }
/** * simple output. * @throws Exception if failed */ @Test public void output() throws Exception { HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); try (ModelOutput<StringBuilder> output = core.openOutput( context, definition, "output", "file.txt", counter)){ output.write(new StringBuilder("Hello, world!")); } assertThat(counter.get(), is(greaterThan(0L))); File target = new File(mapping, "output/file.txt"); assertThat(target.exists(), is(false)); commitAttempt(core); assertThat(target.exists(), is(false)); commitTransaction(core); assertThat(target.exists(), is(true)); assertThat(get(target), is(Arrays.asList("Hello, world!"))); }
private <T> ModelInput<T> open0( Class<? extends DataFormat<T>> formatClass, String originalBasePath, ResourcePattern resourcePattern) throws IOException, InterruptedException { DirectDataSourceRepository repo = prepareRepository(); String basePath = repo.getComponentPath(originalBasePath); DirectDataSource source = repo.getRelatedDataSource(originalBasePath); DataDefinition<T> definition = createDataDefinition(formatClass); List<DirectInputFragment> fragments = source.findInputFragments(definition, basePath, resourcePattern); return new DirectInputFragmentInput<>(source, definition, fragments.iterator(), new Counter()); }