public Class<? extends InputFormat> getInputFileFormatClass() { if (inputFileFormatClass == null && tableDesc != null) { setInputFileFormatClass(tableDesc.getInputFileFormatClass()); } return inputFileFormatClass; }
@Override public void resolveDynamicPartitionStoredAsSubDirsMerge(HiveConf conf, Path path, TableDesc tblDesc, ArrayList<String> aliases, PartitionDesc partDesc) { super.resolveDynamicPartitionStoredAsSubDirsMerge(conf, path, tblDesc, aliases, partDesc); // set internal input format for all partition descriptors partDesc.setInputFileFormatClass(internalInputFormat); // Add the DP path to the list of input paths inputPaths.add(path); }
public Class<? extends InputFormat> getInputFileFormatClass() { if (inputFileFormatClass == null && tableDesc != null) { setInputFileFormatClass(tableDesc.getInputFileFormatClass()); } return inputFileFormatClass; }
@Override public void resolveDynamicPartitionStoredAsSubDirsMerge(HiveConf conf, Path path, TableDesc tblDesc, ArrayList<String> aliases, PartitionDesc partDesc) { super.resolveDynamicPartitionStoredAsSubDirsMerge(conf, path, tblDesc, aliases, partDesc); // set internal input format for all partition descriptors partDesc.setInputFileFormatClass(internalInputFormat); // Add the DP path to the list of input paths inputPaths.add(path); }
public ColumnTruncateWork(List<Integer> droppedColumns, Path inputDir, Path outputDir, boolean hasDynamicPartitions, DynamicPartitionCtx dynPartCtx) { super(); this.droppedColumns = droppedColumns; this.inputDir = inputDir; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; this.dynPartCtx = dynPartCtx; PartitionDesc partDesc = new PartitionDesc(); partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); this.addPathToPartitionInfo(inputDir, partDesc); }
public MergeFileWork(List<Path> inputPaths, Path outputDir, boolean hasDynamicPartitions, String srcTblInputFormat) { this.inputPaths = inputPaths; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; this.srcTblInputFormat = srcTblInputFormat; PartitionDesc partDesc = new PartitionDesc(); if (srcTblInputFormat.equals(OrcInputFormat.class.getName())) { this.internalInputFormat = OrcFileStripeMergeInputFormat.class; } else if (srcTblInputFormat.equals(RCFileInputFormat.class.getName())) { this.internalInputFormat = RCFileBlockMergeInputFormat.class; } partDesc.setInputFileFormatClass(internalInputFormat); for (Path path : this.inputPaths) { this.addPathToPartitionInfo(path, partDesc); } this.isListBucketingAlterTableConcatenate = false; }
public PartialScanWork(List<Path> inputPaths) { super(); this.inputPaths = inputPaths; PartitionDesc partDesc = new PartitionDesc(); partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); for(Path path: this.inputPaths) { this.addPathToPartitionInfo(path, partDesc); } }
public MergeFileWork(List<Path> inputPaths, Path outputDir, boolean hasDynamicPartitions, String srcTblInputFormat, TableDesc tbl) { this.inputPaths = inputPaths; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; this.srcTblInputFormat = srcTblInputFormat; PartitionDesc partDesc = new PartitionDesc(); if (srcTblInputFormat.equals(OrcInputFormat.class.getName())) { this.internalInputFormat = OrcFileStripeMergeInputFormat.class; } else if (srcTblInputFormat.equals(RCFileInputFormat.class.getName())) { this.internalInputFormat = RCFileBlockMergeInputFormat.class; } partDesc.setInputFileFormatClass(internalInputFormat); partDesc.setTableDesc(tbl); for (Path path : this.inputPaths) { this.addPathToPartitionInfo(path, partDesc); } this.isListBucketingAlterTableConcatenate = false; }
public ColumnTruncateWork(List<Integer> droppedColumns, Path inputDir, Path outputDir, boolean hasDynamicPartitions, DynamicPartitionCtx dynPartCtx) { super(); this.droppedColumns = droppedColumns; this.inputDir = inputDir; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; this.dynPartCtx = dynPartCtx; PartitionDesc partDesc = new PartitionDesc(); partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); this.addPathToPartitionInfo(inputDir, partDesc); }
private PartitionDesc changePartitionToMetadataOnly(PartitionDesc desc, Path path) { if (desc == null) { return null; } boolean isEmpty = false; try { isEmpty = Utilities.isEmptyPath(physicalContext.getConf(), path); } catch (IOException e) { LOG.error("Cannot determine if the table is empty", e); } desc.setInputFileFormatClass( isEmpty ? ZeroRowsInputFormat.class : OneNullRowInputFormat.class); desc.setOutputFileFormatClass(HiveIgnoreKeyTextOutputFormat.class); desc.getProperties().setProperty(serdeConstants.SERIALIZATION_LIB, NullStructSerDe.class.getName()); return desc; }
private PartitionDesc changePartitionToMetadataOnly(PartitionDesc desc, Path path) { if (desc == null) return null; boolean isEmpty = false; try { isEmpty = Utilities.isEmptyPath(physicalContext.getConf(), path); } catch (IOException e) { LOG.error("Cannot determine if the table is empty", e); } desc.setInputFileFormatClass( isEmpty ? ZeroRowsInputFormat.class : OneNullRowInputFormat.class); desc.setOutputFileFormatClass(HiveIgnoreKeyTextOutputFormat.class); desc.getProperties().setProperty(serdeConstants.SERIALIZATION_LIB, NullStructSerDe.class.getName()); return desc; }
SymlinkTextInputFormat.class)) { partDesc.setInputFileFormatClass(TextInputFormat.class); FileSystem fileSystem = path.getFileSystem(job); FileStatus fStatus = fileSystem.getFileStatus(path); fileSystem.open(symlink.getPath()))); partDesc.setInputFileFormatClass(TextInputFormat.class);
SymlinkTextInputFormat.class)) { partDesc.setInputFileFormatClass(TextInputFormat.class); FileSystem fileSystem = path.getFileSystem(job); FileStatus fStatus = fileSystem.getFileStatus(path); fileSystem.open(symlink.getPath()))); partDesc.setInputFileFormatClass(TextInputFormat.class);
private void PartitionDescConstructorHelper(final Partition part,final TableDesc tblDesc, boolean setInputFileFormat) throws HiveException { this.tableDesc = tblDesc; setPartSpec(part.getSpec()); if (setInputFileFormat) { setInputFileFormatClass(part.getInputFormatClass()); } else { setOutputFileFormatClass(part.getInputFormatClass()); } setOutputFileFormatClass(part.getOutputFormatClass()); }
private void PartitionDescConstructorHelper(final Partition part,final TableDesc tblDesc, boolean setInputFileFormat) throws HiveException { PlanUtils.configureInputJobPropertiesForStorageHandler(tblDesc); this.tableDesc = tblDesc; setPartSpec(part.getSpec()); if (setInputFileFormat) { setInputFileFormatClass(part.getInputFormatClass()); } else { setOutputFileFormatClass(part.getInputFormatClass()); } setOutputFileFormatClass(part.getOutputFormatClass()); }
@Override public PartitionDesc read(Kryo kryo, Input input, Class<PartitionDesc> type) { PartitionDesc partitionDesc = super.read(kryo, input, type); // The set methods in PartitionDesc intern the any duplicate strings which is why we call them // during de-serialization partitionDesc.setBaseFileName(partitionDesc.getBaseFileName()); partitionDesc.setPartSpec(partitionDesc.getPartSpec()); partitionDesc.setInputFileFormatClass(partitionDesc.getInputFileFormatClass()); partitionDesc.setOutputFileFormatClass(partitionDesc.getOutputFileFormatClass()); return partitionDesc; } }
work.setPathToAliases(pathToAliases); PartitionDesc pDesc = new PartitionDesc(tblDesc, null); pDesc.setInputFileFormatClass(internalIFClass); work.addPathToPartitionInfo(inputDir, pDesc); work.setListBucketingCtx(fsInputDesc.getLbCtx());
work.setPathToAliases(pathToAliases); PartitionDesc pDesc = new PartitionDesc(tblDesc, null); pDesc.setInputFileFormatClass(internalIFClass); work.addPathToPartitionInfo(inputDir, pDesc); work.setListBucketingCtx(fsInputDesc.getLbCtx());
private PartitionDesc changePartitionToMetadataOnly(PartitionDesc desc) { if (desc != null) { desc.setInputFileFormatClass(OneNullRowInputFormat.class); desc.setOutputFileFormatClass(HiveIgnoreKeyTextOutputFormat.class); desc.getProperties().setProperty(serdeConstants.SERIALIZATION_LIB, NullStructSerDe.class.getName()); } return desc; }
public PartialScanWork(List<Path> inputPaths) { super(); this.inputPaths = inputPaths; PartitionDesc partDesc = new PartitionDesc(); partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); if(this.getPathToPartitionInfo() == null) { this.setPathToPartitionInfo(new LinkedHashMap<String, PartitionDesc>()); } for(Path path: this.inputPaths) { this.getPathToPartitionInfo().put(path.toString(), partDesc); } }