public ArrowRecordWriter(OperatorContext context, final EasyWriter writerConfig, ArrowFormatPluginConfig formatConfig) { final FragmentHandle handle = context.getFragmentHandle(); this.writerConfig = writerConfig; this.stats = context.getStats(); this.listOfFilesCreated = Lists.newArrayList(); this.footerBuilder = ArrowFileFooter.newBuilder(); this.location = new Path(writerConfig.getLocation()); this.prefix = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.extension = formatConfig.outputExtension; }
void logClose(){ FragmentHandle h = context.getFragmentHandle(); logger.debug(String.format("Closing operator: %d:%d:%d", h.getMajorFragmentId(), h.getMinorFragmentId(), context.getStats().getOperatorId())); }
public TextRecordWriter(OperatorContext context, EasyWriter config, TextFormatConfig textConfig) { final FragmentHandle handle = context.getFragmentHandle(); this.conf = new Configuration(config.getFsConf()); this.stats = context.getStats(); this.location = config.getLocation(); this.prefix = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.fieldDelimiter = textConfig.getFieldDelimiterAsString(); this.lineDelimiter = textConfig.getLineDelimiter(); this.extension = textConfig.outputExtension; this.currentRecord = new StringBuilder(); this.index = 0; }
@Override public FragmentInfo apply(final FragmentExecutor fragmentExecutor) { final FragmentStatus status = fragmentExecutor.getStatus(); final ExecProtos.FragmentHandle handle = fragmentExecutor.getHandle(); final MinorFragmentProfile profile = status == null ? null : status.getProfile(); Long memoryUsed = profile == null ? 0 : profile.getMemoryUsed(); Long rowsProcessed = profile == null ? 0 : getRowsProcessed(profile); Timestamp startTime = profile == null ? new Timestamp(0) : new Timestamp(profile.getStartTime()); return new FragmentInfo(dbContext.get().getEndpoint().getAddress(), QueryIdHelper.getQueryId(handle.getQueryId()), handle.getMajorFragmentId(), handle.getMinorFragmentId(), memoryUsed, rowsProcessed, startTime, fragmentExecutor.getBlockingStatus(), fragmentExecutor.getTaskDescriptor()); }
private void sendTermination() { final ExecProtos.FragmentHandle handle = context.getFragmentHandle(); stats.startWait(); for (MinorFragmentEndpoint destination : config.getDestinations()) { // don't send termination message if the receiver fragment is already terminated. if (remainingReceivers.get(destination.getId()) == 0) { ExecRPC.FragmentStreamComplete completion = ExecRPC.FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(config.getOppositeMajorFragmentId()) .addReceivingMinorFragmentId(destination.getId()) .build(); tunnelProvider.getExecTunnel(destination.getEndpoint()).sendStreamComplete(completion); } } stats.stopWait(); }
public Builder mergeFrom(com.dremio.exec.proto.ExecProtos.FragmentHandle other) { if (other == com.dremio.exec.proto.ExecProtos.FragmentHandle.getDefaultInstance()) return this; if (other.hasQueryId()) { mergeQueryId(other.getQueryId()); } if (other.hasMajorFragmentId()) { setMajorFragmentId(other.getMajorFragmentId()); } if (other.hasMinorFragmentId()) { setMinorFragmentId(other.getMinorFragmentId()); } if (other.hasParentQueryId()) { mergeParentQueryId(other.getParentQueryId()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
/** * creates a fragment ticket for the passed-in fragment. May create a new query ticket if no query ticket is already * cached. Closing the ticket will return the reservation and eventually close the corresponding query ticket along with * its allocator * * @param queryTicket the query ticket, obtained from the callback from {@link #buildAndStartQuery(PlanFragment, SchedulingInfo, QueryStarter)}, above * @param fragment fragment plan * @param schedulingInfo information about where should 'fragment' run * @return reserved query allocator */ public FragmentTicket newFragmentTicket(final QueryTicket queryTicket, final PlanFragment fragment, final SchedulingInfo schedulingInfo) { // Note: applying query limit to the phase, as that doesn't add any additional restrictions. If an when we have // phase limits on the plan fragment, we could apply them here. PhaseTicket phaseTicket = queryTicket .getOrCreatePhaseTicket(fragment.getHandle().getMajorFragmentId(), queryTicket.getAllocator().getLimit()); return new FragmentTicket(phaseTicket, queryTicket.getSchedulingGroup()); }
public static String getQueryIdentifier(final FragmentHandle h) { return getQueryId(h.getQueryId()) + ":" + h.getMajorFragmentId() + ":" + h.getMinorFragmentId(); }
@Override public void noMoreToConsume() { for (int i = 0; i < tunnels.length; ++i) { final FragmentStreamComplete completion = FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(config.getOppositeMajorFragmentId()) .addAllReceivingMinorFragmentId(Ints.asList(receivingMinorFragments[i])) .build(); tunnels[i].sendStreamComplete(completion); } state = State.DONE; }
public static String getFragmentId(final FragmentHandle fragmentHandle) { return fragmentHandle.getMajorFragmentId() + ":" + fragmentHandle.getMinorFragmentId(); }
private void setupOutputStream() { try { final String qid = QueryIdHelper.getQueryId(handle.getQueryId()); final int majorFragmentId = handle.getMajorFragmentId(); final int minorFragmentId = handle.getMinorFragmentId(); final String id = String.format("spool-%s.%s.%s.%s.%s", qid, majorFragmentId, minorFragmentId, oppositeId, bufferIndex); this.spillManager = new SpillManager(config, null, id, SPOOLING_CONFIG, spillService, "spooling sorted exchange"); this.spillFile = spillManager.getSpillFile("batches"); outputStream = spillFile.create(); } catch(Exception ex) { throw Throwables.propagate(ex); } }
@Override public void noMoreToConsume() { for (int i = 0; i < tunnels.size(); ++i) { final FragmentStreamComplete completion = FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(config.getOppositeMajorFragmentId()) .addAllReceivingMinorFragmentId(minorFragments.get(i)) .build(); tunnels.get(i).sendStreamComplete(completion); } state = State.DONE; }
@Override public void noMoreToConsume() { FragmentStreamComplete completionMessage = FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(oppositeHandle.getMajorFragmentId()) .addReceivingMinorFragmentId(oppositeHandle.getMinorFragmentId()) .build(); tunnel.sendStreamComplete(completionMessage); }
public void sendTermination() { final FragmentHandle handle = context.getFragmentHandle(); FragmentStreamComplete completion = FragmentStreamComplete.newBuilder() .setQueryId(handle.getQueryId()) .setSendingMajorFragmentId(handle.getMajorFragmentId()) .setSendingMinorFragmentId(handle.getMinorFragmentId()) .setReceivingMajorFragmentId(operator.getOppositeMajorFragmentId()) .addReceivingMinorFragmentId(oppositeMinorFragmentId) .build(); tunnel.sendStreamComplete(completion); dropAll = true; }
public JsonRecordWriter(OperatorContext context, EasyWriter writer, JSONFormatConfig formatConfig){ final FragmentHandle handle = context.getFragmentHandle(); final String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.conf = new Configuration(writer.getFsConf()); this.stats = context.getStats(); this.location = writer.getLocation(); this.prefix = fragmentId; this.useExtendedOutput = context.getOptions().getOption(ExecConstants.JSON_EXTENDED_TYPES); this.extension = formatConfig.outputExtension; this.useExtendedOutput = context.getOptions().getOption(ExecConstants.JSON_EXTENDED_TYPES); this.uglify = !formatConfig.prettyPrint || context.getOptions().getOption(ExecConstants.JSON_WRITER_UGLIFY); }
public static SharedDataMap create(List<SharedData> sharedDataList, FragmentHandle fragmentHandle) { Map<Key,SharedData> map = FluentIterable.from(sharedDataList) .filter(s -> s.getMajorFragmentId() == fragmentHandle.getMajorFragmentId()) .uniqueIndex(s -> new Key(s.getOperatorId(), s.getName())); return new SharedDataMap(map); }
public WriterOperator(OperatorContext context, WriterOptions options, RecordWriter recordWriter) throws OutOfMemoryException { this.context = context; this.stats = context.getStats(); this.output = context.createOutputVectorContainer(RecordWriter.SCHEMA); this.options = options; final FragmentHandle handle = context.getFragmentHandle(); this.fragmentUniqueId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.recordWriter = recordWriter; this.writtenRecordLimit = options.getRecordLimit(); }
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.ExecProtos.FragmentHandle message) throws java.io.IOException { if(message.hasQueryId()) output.writeObject(1, message.getQueryId(), com.dremio.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasMajorFragmentId()) output.writeInt32(2, message.getMajorFragmentId(), false); if(message.hasMinorFragmentId()) output.writeInt32(3, message.getMinorFragmentId(), false); if(message.hasParentQueryId()) output.writeObject(4, message.getParentQueryId(), com.dremio.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); } public boolean isInitialized(com.dremio.exec.proto.ExecProtos.FragmentHandle message)
public static String getFileNameForQueryFragment(FragmentHandle handle, String location, String tag) { String qid = QueryIdHelper.getQueryId(handle.getQueryId()); int majorFragmentId = handle.getMajorFragmentId(); int minorFragmentId = handle.getMinorFragmentId(); String fileName = String.format("%s//%s_%s_%s_%s", location, qid, majorFragmentId, minorFragmentId, tag); return fileName; }
public static String getExecutorThreadName(final FragmentHandle fragmentHandle) { return String.format("%s:frag:%s:%s", getQueryId(fragmentHandle.getQueryId()), fragmentHandle.getMajorFragmentId(), fragmentHandle.getMinorFragmentId()); }