private OutputAttemptContext context() { return new OutputAttemptContext("tx", "at", "o", counter); }
@Override public void close() throws IOException { output.close(); try { dataSource.commitAttemptOutput(outputContext); dataSource.cleanupAttemptOutput(outputContext); dataSource.commitTransactionOutput(outputContext.getTransactionContext()); dataSource.cleanupTransactionOutput(outputContext.getTransactionContext()); } catch (InterruptedException e) { throw (IOException) new InterruptedIOException("interrupted").initCause(e); //$NON-NLS-1$ } } };
Path getAttemptOutput(OutputAttemptContext context) { assert context != null; Path tempPath = getTemporaryOutput(context.getTransactionContext()); String suffix = String.format("%s/%s", //$NON-NLS-1$ ATTEMPT_AREA, context.getAttemptId()); return append(tempPath, suffix); }
Path getLocalAttemptOutput(OutputAttemptContext context) throws IOException { assert context != null; Path tempPath = HadoopDataSourceUtil.getLocalTemporaryDirectory(profile.getLocalFileSystem()); String suffix = String.format("%s-%s-%s", //$NON-NLS-1$ context.getTransactionId(), context.getAttemptId(), context.getOutputId()); return append(tempPath, suffix); }
throw (IOException) new InterruptedIOException(MessageFormat.format( "Interrupted while cleanup attempt: {0}, {1} (path={2})", context.getTransactionId(), context.getAttemptId(), containerPath)).initCause(e); context.getCounter().add(1);
Path target; if (profile.isOutputStaging()) { target = getStagingOutput(context.getTransactionContext()); } else { target = profile.getFileSystemPath(); context.getCounter(), profile.getLocalFileSystem(), profile.getFileSystem(), attempt, target); } else { Path attempt = getAttemptOutput(context); profile.isOutputStaging())); HadoopDataSourceUtil.move(context.getCounter(), profile.getFileSystem(), attempt, target);
@Override public void commitAttemptOutput(OutputAttemptContext context) throws IOException, InterruptedException { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } Counter counter = context.getCounter(); heartbeat.register(counter); try { entity.commitAttemptOutput(context); } finally { heartbeat.unregister(counter); } }
throw (IOException) new InterruptedIOException(MessageFormat.format( "Interrupted while setup attempt: {0}, {1} (path={2})", context.getTransactionId(), context.getAttemptId(), containerPath)).initCause(e); context.getCounter().add(1);
@Override public void cleanupAttemptOutput(OutputAttemptContext context) throws IOException, InterruptedException { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } Counter counter = context.getCounter(); heartbeat.register(counter); try { entity.cleanupAttemptOutput(context); } finally { heartbeat.unregister(counter); } }
private void commitTransaction(HadoopDataSourceCore core) throws IOException, InterruptedException { core.commitTransactionOutput(context.getTransactionContext()); core.cleanupTransactionOutput(context.getTransactionContext()); }
private OutputAttemptContext createOutputContext() { String tx = UUID.randomUUID().toString(); String attempt = UUID.randomUUID().toString(); return new OutputAttemptContext(tx, attempt, id, new Counter()); }
throw (IOException) new InterruptedIOException(MessageFormat.format( "Interrupted while commit task attempt: {0}, {1} (path={2})", context.getTransactionId(), context.getAttemptId(), containerPath)).initCause(e); } catch (RuntimeException e) { throw e; context.getCounter().add(1);
@Override public void setupAttemptOutput(OutputAttemptContext context) throws IOException, InterruptedException { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } Counter counter = context.getCounter(); heartbeat.register(counter); try { entity.setupAttemptOutput(context); } finally { heartbeat.unregister(counter); } }
private void setup(HadoopDataSourceCore core) throws IOException, InterruptedException { core.setupTransactionOutput(context.getTransactionContext()); core.setupAttemptOutput(context); }
/** * Creates output context from Hadoop context. * @param context current context in Hadoop * @param datasourceId datasource ID * @return the created context * @throws IllegalArgumentException if some parameters were {@code null} */ public static OutputAttemptContext createContext(TaskAttemptContext context, String datasourceId) { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } if (datasourceId == null) { throw new IllegalArgumentException("datasourceId must not be null"); //$NON-NLS-1$ } String transactionId = getTransactionId(context, datasourceId); String attemptId = getAttemptId(context, datasourceId); return new OutputAttemptContext(transactionId, attemptId, datasourceId, createCounter(context)); }
private void cleanup(HadoopDataSourceCore core) throws IOException, InterruptedException { core.cleanupAttemptOutput(context); core.cleanupTransactionOutput(context.getTransactionContext()); }
/** * Initializes the test. * @throws Exception if some errors were occurred */ @Before public void setUp() throws Exception { conf = new Configuration(true); if (definition.getDataFormat() instanceof Configurable) { ((Configurable) definition.getDataFormat()).setConf(conf); } mapping = new File(temp.getRoot(), "mapping").getCanonicalFile(); temporary = new File(temp.getRoot(), "temporary").getCanonicalFile(); localtemp = new File(temp.getRoot(), "localtemp").getCanonicalFile(); profile = new HadoopDataSourceProfile( conf, "testing", "testing", new Path(mapping.toURI()), new Path(temporary.toURI())); context = new OutputAttemptContext("tx", "atmpt", profile.getId(), new Counter()); }
dataSource.setupTransactionOutput(outputContext.getTransactionContext()); dataSource.setupAttemptOutput(outputContext); Counter counter = new Counter();
DirectDataSource ds = repo.getRelatedDataSource(path); OutputTransactionContext txContext = HadoopDataSourceUtil.createContext(executionId, id); OutputAttemptContext aContext = new OutputAttemptContext( txContext.getTransactionId(), String.valueOf(index),
/** * Test method for commitTransactionOutput. * @throws Exception if failed */ @Test public void testCommitTransactionOutput() throws Exception { OutputAttemptContext context = context(); long s1 = counter.count; ds.commitTransactionOutput(context.getTransactionContext()); long s2 = counter.count; assertThat(s2, greaterThan(s1)); assertKeepAlive(false); }