public static PartitionDesc getPartitionDesc(Partition part, TableDesc tableDesc) throws HiveException { return new PartitionDesc(part, tableDesc); }
public static PartitionDesc getPartitionDescFromTableDesc(TableDesc tblDesc, Partition part, boolean usePartSchemaProperties) throws HiveException { return new PartitionDesc(part, tblDesc, usePartSchemaProperties); }
public static PartitionDesc getPartitionDesc(Partition part) throws HiveException { return new PartitionDesc(part); }
public static PartitionDesc getPartitionDescFromTableDesc(TableDesc tblDesc, Partition part, boolean usePartSchemaProperties) throws HiveException { return new PartitionDesc(part, tblDesc, usePartSchemaProperties); }
public static PartitionDesc getPartitionDesc(Partition part) throws HiveException { return new PartitionDesc(part); }
private PartitionDesc generateDPFullPartSpec(DynamicPartitionCtx dpCtx, FileStatus[] status, TableDesc tblDesc, int i) { LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<>( dpCtx.getPartSpec()); // Require all the directories to be present with some values. if (!Warehouse.makeSpecFromName(fullPartSpec, status[i].getPath(), new HashSet<>(dpCtx.getPartSpec().keySet()))) { return null; } return new PartitionDesc(tblDesc, fullPartSpec); }
private PartitionDesc generateDPFullPartSpec(DynamicPartitionCtx dpCtx, FileStatus[] status, TableDesc tblDesc, int i) { LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<>(dpCtx.getPartSpec()); Warehouse.makeSpecFromName(fullPartSpec, status[i].getPath()); PartitionDesc pDesc = new PartitionDesc(tblDesc, fullPartSpec); return pDesc; }
public ColumnTruncateWork(List<Integer> droppedColumns, Path inputDir, Path outputDir, boolean hasDynamicPartitions, DynamicPartitionCtx dynPartCtx) { super(); this.droppedColumns = droppedColumns; this.inputDir = inputDir; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; this.dynPartCtx = dynPartCtx; PartitionDesc partDesc = new PartitionDesc(); partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); this.addPathToPartitionInfo(inputDir, partDesc); }
public MergeFileWork(List<Path> inputPaths, Path outputDir, boolean hasDynamicPartitions, String srcTblInputFormat) { this.inputPaths = inputPaths; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; this.srcTblInputFormat = srcTblInputFormat; PartitionDesc partDesc = new PartitionDesc(); if (srcTblInputFormat.equals(OrcInputFormat.class.getName())) { this.internalInputFormat = OrcFileStripeMergeInputFormat.class; } else if (srcTblInputFormat.equals(RCFileInputFormat.class.getName())) { this.internalInputFormat = RCFileBlockMergeInputFormat.class; } partDesc.setInputFileFormatClass(internalInputFormat); for (Path path : this.inputPaths) { this.addPathToPartitionInfo(path, partDesc); } this.isListBucketingAlterTableConcatenate = false; }
public PartialScanWork(List<Path> inputPaths) { super(); this.inputPaths = inputPaths; PartitionDesc partDesc = new PartitionDesc(); partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); for(Path path: this.inputPaths) { this.addPathToPartitionInfo(path, partDesc); } }
public ColumnTruncateWork(List<Integer> droppedColumns, Path inputDir, Path outputDir, boolean hasDynamicPartitions, DynamicPartitionCtx dynPartCtx) { super(); this.droppedColumns = droppedColumns; this.inputDir = inputDir; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; this.dynPartCtx = dynPartCtx; PartitionDesc partDesc = new PartitionDesc(); partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); this.addPathToPartitionInfo(inputDir, partDesc); }
public MergeFileWork(List<Path> inputPaths, Path outputDir, boolean hasDynamicPartitions, String srcTblInputFormat, TableDesc tbl) { this.inputPaths = inputPaths; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; this.srcTblInputFormat = srcTblInputFormat; PartitionDesc partDesc = new PartitionDesc(); if (srcTblInputFormat.equals(OrcInputFormat.class.getName())) { this.internalInputFormat = OrcFileStripeMergeInputFormat.class; } else if (srcTblInputFormat.equals(RCFileInputFormat.class.getName())) { this.internalInputFormat = RCFileBlockMergeInputFormat.class; } partDesc.setInputFileFormatClass(internalInputFormat); partDesc.setTableDesc(tbl); for (Path path : this.inputPaths) { this.addPathToPartitionInfo(path, partDesc); } this.isListBucketingAlterTableConcatenate = false; }
@Override public PartitionDesc clone() { PartitionDesc ret = new PartitionDesc(); ret.inputFileFormatClass = inputFileFormatClass; ret.outputFileFormatClass = outputFileFormatClass; if (properties != null) { ret.setProperties((Properties) properties.clone()); } ret.tableDesc = (TableDesc) tableDesc.clone(); // The partition spec is not present if (partSpec != null) { ret.partSpec = new LinkedHashMap<>(partSpec); } if (vectorPartitionDesc != null) { ret.vectorPartitionDesc = vectorPartitionDesc.clone(); } return ret; }
/** * used for bucket map join */ public void setupContext(List<Path> paths) { this.iterPath = paths.iterator(); List<PartitionDesc> partitionDescs; if (!isPartitioned) { this.iterPartDesc = Iterators.cycle(new PartitionDesc(work.getTblDesc(), null)); } else { this.iterPartDesc = work.getPartDescs(paths).iterator(); } this.context = setupExecContext(operator, paths); }
/** * used for bucket map join */ public void setupContext(List<Path> paths) { this.iterPath = paths.iterator(); List<PartitionDesc> partitionDescs; if (!isPartitioned) { this.iterPartDesc = Iterators.cycle(new PartitionDesc(work.getTblDesc(), null)); } else { this.iterPartDesc = work.getPartDescs(paths).iterator(); } this.context = setupExecContext(operator, paths); }
@Override protected void setUp() throws IOException { conf = new Configuration(); job = new JobConf(conf); TableDesc tblDesc = Utilities.defaultTd; PartitionDesc partDesc = new PartitionDesc(tblDesc, null); LinkedHashMap<Path, PartitionDesc> pt = new LinkedHashMap<>(); pt.put(new Path("/tmp/testfolder"), partDesc); MapredWork mrwork = new MapredWork(); mrwork.getMapWork().setPathToPartitionInfo(pt); Utilities.setMapRedWork(job, mrwork,new Path("/tmp/" + System.getProperty("user.name"), "hive")); fileSystem = FileSystem.getLocal(conf); testDir = new Path(System.getProperty("test.tmp.dir", System.getProperty( "user.dir", new File(".").getAbsolutePath())) + "/TestSymlinkTextInputFormat"); reporter = Reporter.NULL; fileSystem.delete(testDir, true); dataDir1 = new Path(testDir, "datadir1"); dataDir2 = new Path(testDir, "datadir2"); symlinkDir = new Path(testDir, "symlinkdir"); }
public void testAvoidSplitCombination() throws Exception { Configuration conf = new Configuration(); JobConf job = new JobConf(conf); TableDesc tblDesc = Utilities.defaultTd; tblDesc.setInputFileFormatClass(TestSkipCombineInputFormat.class); PartitionDesc partDesc = new PartitionDesc(tblDesc, null); LinkedHashMap<Path, PartitionDesc> pt = new LinkedHashMap<>(); pt.put(new Path("/tmp/testfolder1"), partDesc); pt.put(new Path("/tmp/testfolder2"), partDesc); MapredWork mrwork = new MapredWork(); mrwork.getMapWork().setPathToPartitionInfo(pt); Path mapWorkPath = new Path("/tmp/" + System.getProperty("user.name"), "hive"); Utilities.setMapRedWork(conf, mrwork, mapWorkPath); try { Path[] paths = new Path[2]; paths[0] = new Path("/tmp/testfolder1"); paths[1] = new Path("/tmp/testfolder2"); CombineHiveInputFormat combineInputFormat = ReflectionUtils.newInstance(CombineHiveInputFormat.class, conf); combineInputFormat.pathToPartitionInfo = Utilities.getMapWork(conf).getPathToPartitionInfo(); Set results = combineInputFormat.getNonCombinablePathIndices(job, paths, 2); assertEquals("Should have both path indices in the results set", 2, results.size()); } finally { // Cleanup the mapwork path FileSystem.get(conf).delete(mapWorkPath, true); } }
private void init() throws IOException { conf = new JobConf(); resetIOContext(); rcfReader = mock(RCFileRecordReader.class); when(rcfReader.next((LongWritable)anyObject(), (BytesRefArrayWritable )anyObject())).thenReturn(true); // Since the start is 0, and the length is 100, the first call to sync should be with the value // 50 so return that for getPos() when(rcfReader.getPos()).thenReturn(50L); conf.setBoolean("hive.input.format.sorted", true); TableDesc tblDesc = Utilities.defaultTd; PartitionDesc partDesc = new PartitionDesc(tblDesc, null); LinkedHashMap<Path, PartitionDesc> pt = new LinkedHashMap<>(); pt.put(new Path("/tmp/testfolder"), partDesc); MapredWork mrwork = new MapredWork(); mrwork.getMapWork().setPathToPartitionInfo(pt); Utilities.setMapRedWork(conf, mrwork,new Path("/tmp/" + System.getProperty("user.name"), "hive")); hiveSplit = new TestHiveInputSplit(); hbsReader = new TestHiveRecordReader(rcfReader, conf); hbsReader.initIOContext(hiveSplit, conf, Class.class, rcfReader); }
public static void addMapWork(MapredWork mr, Table tbl, String alias, Operator<?> work) { mr.getMapWork().addMapWork(tbl.getDataLocation(), alias, work, new PartitionDesc( Utilities.getTableDesc(tbl), null)); }
private StructObjectInspector setupOutputObjectInspector() throws HiveException { TableDesc tableDesc = work.getTblDesc(); try { tableSerDe = tableDesc.getDeserializer(job, true); tableOI = (StructObjectInspector) tableSerDe.getObjectInspector(); if (!isPartitioned) { return getTableRowOI(tableOI); } partKeyOI = getPartitionKeyOI(tableDesc); PartitionDesc partDesc = new PartitionDesc(tableDesc, null); List<PartitionDesc> listParts = work.getPartDesc(); // Chose the table descriptor if none of the partitions is present. // For eg: consider the query: // select /*+mapjoin(T1)*/ count(*) from T1 join T2 on T1.key=T2.key // Both T1 and T2 and partitioned tables, but T1 does not have any partitions // FetchOperator is invoked for T1, and listParts is empty. In that case, // use T1's schema to get the ObjectInspector. if (listParts == null || listParts.isEmpty() || !needConversion(tableDesc, listParts)) { return getPartitionedRowOI(tableOI); } convertedOI = (StructObjectInspector) ObjectInspectorConverters.getConvertedOI( tableOI, tableOI, null, false); return getPartitionedRowOI(convertedOI); } catch (Exception e) { throw new HiveException("Failed with exception " + e.getMessage() + StringUtils.stringifyException(e)); } }