public ArrowRecordWriter(OperatorContext context, final EasyWriter writerConfig, ArrowFormatPluginConfig formatConfig) { final FragmentHandle handle = context.getFragmentHandle(); this.writerConfig = writerConfig; this.stats = context.getStats(); this.listOfFilesCreated = Lists.newArrayList(); this.footerBuilder = ArrowFileFooter.newBuilder(); this.location = new Path(writerConfig.getLocation()); this.prefix = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.extension = formatConfig.outputExtension; }
public TextRecordWriter(OperatorContext context, EasyWriter config, TextFormatConfig textConfig) { final FragmentHandle handle = context.getFragmentHandle(); this.conf = new Configuration(config.getFsConf()); this.stats = context.getStats(); this.location = config.getLocation(); this.prefix = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.fieldDelimiter = textConfig.getFieldDelimiterAsString(); this.lineDelimiter = textConfig.getLineDelimiter(); this.extension = textConfig.outputExtension; this.currentRecord = new StringBuilder(); this.index = 0; }
void logClose(){ FragmentHandle h = context.getFragmentHandle(); logger.debug(String.format("Closing operator: %d:%d:%d", h.getMajorFragmentId(), h.getMinorFragmentId(), context.getStats().getOperatorId())); }
@Override public FragmentInfo apply(final FragmentExecutor fragmentExecutor) { final FragmentStatus status = fragmentExecutor.getStatus(); final ExecProtos.FragmentHandle handle = fragmentExecutor.getHandle(); final MinorFragmentProfile profile = status == null ? null : status.getProfile(); Long memoryUsed = profile == null ? 0 : profile.getMemoryUsed(); Long rowsProcessed = profile == null ? 0 : getRowsProcessed(profile); Timestamp startTime = profile == null ? new Timestamp(0) : new Timestamp(profile.getStartTime()); return new FragmentInfo(dbContext.get().getEndpoint().getAddress(), QueryIdHelper.getQueryId(handle.getQueryId()), handle.getMajorFragmentId(), handle.getMinorFragmentId(), memoryUsed, rowsProcessed, startTime, fragmentExecutor.getBlockingStatus(), fragmentExecutor.getTaskDescriptor()); }
private void sendTermination() { final ExecProtos.FragmentHandle handle = context.getFragmentHandle(); stats.startWait(); for (MinorFragmentEndpoint destination : config.getDestinations()) { // don't send termination message if the receiver fragment is already terminated. if (remainingReceivers.get(destination.getId()) == 0) { ExecRPC.FragmentStreamComplete completion = ExecRPC.FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(config.getOppositeMajorFragmentId()) .addReceivingMinorFragmentId(destination.getId()) .build(); tunnelProvider.getExecTunnel(destination.getEndpoint()).sendStreamComplete(completion); } } stats.stopWait(); }
@Override public void receivingFragmentFinished(ExecProtos.FragmentHandle handle) throws Exception { final int id = handle.getMinorFragmentId(); if (remainingReceivers.compareAndSet(id, 0, 1)) { synchronized (batchCreationLock) { if (batches[id] == null) { terminations.add(id); } else { batches[id].terminate(); batches[id + numReceivers].terminate(); } } int remaining = remaingReceiverCount.decrementAndGet(); if (remaining == 0) { nobodyListening = true; } } }
public static String getQueryIdentifier(final FragmentHandle h) { return getQueryId(h.getQueryId()) + ":" + h.getMajorFragmentId() + ":" + h.getMinorFragmentId(); }
@Override public void noMoreToConsume() { for (int i = 0; i < tunnels.length; ++i) { final FragmentStreamComplete completion = FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(config.getOppositeMajorFragmentId()) .addAllReceivingMinorFragmentId(Ints.asList(receivingMinorFragments[i])) .build(); tunnels[i].sendStreamComplete(completion); } state = State.DONE; }
public static String getFragmentId(final FragmentHandle fragmentHandle) { return fragmentHandle.getMajorFragmentId() + ":" + fragmentHandle.getMinorFragmentId(); }
private void setupOutputStream() { try { final String qid = QueryIdHelper.getQueryId(handle.getQueryId()); final int majorFragmentId = handle.getMajorFragmentId(); final int minorFragmentId = handle.getMinorFragmentId(); final String id = String.format("spool-%s.%s.%s.%s.%s", qid, majorFragmentId, minorFragmentId, oppositeId, bufferIndex); this.spillManager = new SpillManager(config, null, id, SPOOLING_CONFIG, spillService, "spooling sorted exchange"); this.spillFile = spillManager.getSpillFile("batches"); outputStream = spillFile.create(); } catch(Exception ex) { throw Throwables.propagate(ex); } }
@Override public void noMoreToConsume() { for (int i = 0; i < tunnels.size(); ++i) { final FragmentStreamComplete completion = FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(config.getOppositeMajorFragmentId()) .addAllReceivingMinorFragmentId(minorFragments.get(i)) .build(); tunnels.get(i).sendStreamComplete(completion); } state = State.DONE; }
private FragmentStatus getStatus(final FragmentState state, final UserException ex) { final FragmentStatus.Builder status = FragmentStatus.newBuilder(); final MinorFragmentProfile.Builder b = MinorFragmentProfile.newBuilder(); stats.addMetricsToStatus(b); b.setState(state); if (ex != null) { b.setError(ex.getOrCreatePBError(true)); } status.setHandle(handle); b.setMemoryUsed(fragmentAllocator.getAllocatedMemory()); b.setMinorFragmentId(handle.getMinorFragmentId()); status.setProfile(b); return status.build(); }
@Override public void noMoreToConsume() { FragmentStreamComplete completionMessage = FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(oppositeHandle.getMajorFragmentId()) .addReceivingMinorFragmentId(oppositeHandle.getMinorFragmentId()) .build(); tunnel.sendStreamComplete(completionMessage); }
public void sendTermination() { final FragmentHandle handle = context.getFragmentHandle(); FragmentStreamComplete completion = FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(operator.getOppositeMajorFragmentId()) .addReceivingMinorFragmentId(oppositeMinorFragmentId) .build(); tunnel.sendStreamComplete(completion); dropAll = true; }
public JsonRecordWriter(OperatorContext context, EasyWriter writer, JSONFormatConfig formatConfig){ final FragmentHandle handle = context.getFragmentHandle(); final String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.conf = new Configuration(writer.getFsConf()); this.stats = context.getStats(); this.location = writer.getLocation(); this.prefix = fragmentId; this.useExtendedOutput = context.getOptions().getOption(ExecConstants.JSON_EXTENDED_TYPES); this.extension = formatConfig.outputExtension; this.useExtendedOutput = context.getOptions().getOption(ExecConstants.JSON_EXTENDED_TYPES); this.uglify = !formatConfig.prettyPrint || context.getOptions().getOption(ExecConstants.JSON_WRITER_UGLIFY); }
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.ExecProtos.FragmentHandle message) throws java.io.IOException { if(message.hasQueryId()) output.writeObject(1, message.getQueryId(), com.dremio.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasMajorFragmentId()) output.writeInt32(2, message.getMajorFragmentId(), false); if(message.hasMinorFragmentId()) output.writeInt32(3, message.getMinorFragmentId(), false); if(message.hasParentQueryId()) output.writeObject(4, message.getParentQueryId(), com.dremio.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); } public boolean isInitialized(com.dremio.exec.proto.ExecProtos.FragmentHandle message)
public static String getFileNameForQueryFragment(FragmentHandle handle, String location, String tag) { String qid = QueryIdHelper.getQueryId(handle.getQueryId()); int majorFragmentId = handle.getMajorFragmentId(); int minorFragmentId = handle.getMinorFragmentId(); String fileName = String.format("%s//%s_%s_%s_%s", location, qid, majorFragmentId, minorFragmentId, tag); return fileName; }
public WriterOperator(OperatorContext context, WriterOptions options, RecordWriter recordWriter) throws OutOfMemoryException { this.context = context; this.stats = context.getStats(); this.output = context.createOutputVectorContainer(RecordWriter.SCHEMA); this.options = options; final FragmentHandle handle = context.getFragmentHandle(); this.fragmentUniqueId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.recordWriter = recordWriter; this.writtenRecordLimit = options.getRecordLimit(); }
public FragmentData(final FragmentHandle handle, final NodeEndpoint endpoint) { this.endpoint = endpoint; final MinorFragmentProfile f = MinorFragmentProfile.newBuilder() .setState(FragmentState.SENDING) .setMinorFragmentId(handle.getMinorFragmentId()) .setEndpoint(endpoint) .build(); status = FragmentStatus.newBuilder() .setHandle(handle) .setProfile(f) .build(); }
public static String getExecutorThreadName(final FragmentHandle fragmentHandle) { return String.format("%s:frag:%s:%s", getQueryId(fragmentHandle.getQueryId()), fragmentHandle.getMajorFragmentId(), fragmentHandle.getMinorFragmentId()); }