public void logMessage(LoadTableDesc tbd) { StringBuilder mesg = new StringBuilder("Loading data to table ") .append( tbd.getTable().getTableName()); if (tbd.getPartitionSpec().size() > 0) { mesg.append(" partition ("); Map<String, String> partSpec = tbd.getPartitionSpec(); for (String key: partSpec.keySet()) { mesg.append(key).append('=').append(partSpec.get(key)).append(", "); } mesg.setLength(mesg.length()-2); mesg.append(')'); } String mesg_detail = " from " + tbd.getSourcePath(); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace(mesg.toString() + " " + mesg_detail); } console.printInfo(mesg.toString(), mesg_detail); }
if (tbd.getPartitionSpec() == null || tbd.getPartitionSpec().isEmpty()) { } else { Partition oldPart = db.getPartition(table, tbd.getPartitionSpec(), false); if (oldPart == null) {
public List<Task<? extends Serializable>> generateUpdateTasks() throws HiveException { hive = Hive.get(this.conf); for (LoadTableDesc ltd : loadTableWork) { TableDesc td = ltd.getTable(); Table srcTable = hive.getTable(td.getTableName()); List<Index> tblIndexes = IndexUtils.getAllIndexes(srcTable, (short)-1); Map<String, String> partSpec = ltd.getPartitionSpec(); if (partSpec == null || partSpec.size() == 0) { //unpartitioned table, update whole index doIndexUpdate(tblIndexes); } else { doIndexUpdate(tblIndexes, partSpec); } } return tasks; }
Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); list.add(partn);
Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); list.add(partn);
tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getLoadFileType(), dpCtx.getNumDPCols(),
if (tbd.getPartitionSpec().size() == 0) { dc = new DataContainer(table.getTTable()); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { LOG.info("Partition is: {}", tbd.getPartitionSpec());
private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, TaskInformation ti) throws HiveException, IOException, InvalidOperationException { List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec()); db.validatePartitionNameCharacters(partVals); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath() + " into " + tbd.getTable().getTableName()); } db.loadPartition(tbd.getSourcePath(), db.getTable(tbd.getTable().getTableName()), tbd.getPartitionSpec(), tbd.getLoadFileType(), tbd.getInheritTableSpecs(), tbd.getInheritLocation(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(), work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && !tbd.isMmTable(), resetStatisticsProps(table), tbd.getWriteId(), tbd.getStmtId(), tbd.isInsertOverwrite()); Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); // See the comment inside updatePartitionBucketSortColumns. if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) { updatePartitionBucketSortColumns(db, table, partn, ti.bucketCols, ti.numBuckets, ti.sortCols); } DataContainer dc = new DataContainer(table.getTTable(), partn.getTPartition()); // add this partition to post-execution hook if (work.getOutputs() != null) { DDLTask.addIfAbsentByName(new WriteEntity(partn, getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs()); } return dc; }
StringBuilder mesg = new StringBuilder("Loading data to table ") .append( tbd.getTable().getTableName()); if (tbd.getPartitionSpec().size() > 0) { mesg.append(" partition ("); Map<String, String> partSpec = tbd.getPartitionSpec(); for (String key: partSpec.keySet()) { mesg.append(key).append('=').append(partSpec.get(key)).append(", "); if (tbd.getPartitionSpec() == null || tbd.getPartitionSpec().isEmpty()) { Partition oldPart = db.getPartition(table, tbd.getPartitionSpec(), false); if (oldPart == null) { if (tbd.getPartitionSpec().size() == 0) { dc = new DataContainer(table.getTTable()); db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getReplace(), LOG.info("Partition is: " + tbd.getPartitionSpec().toString()); tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), dpCtx.getNumDPCols(), tbd.getPartitionSpec()); db.validatePartitionNameCharacters(partVals); db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(),
public List<Task<? extends Serializable>> generateUpdateTasks() throws HiveException { hive = Hive.get(this.conf); for (LoadTableDesc ltd : loadTableWork) { TableDesc td = ltd.getTable(); Table srcTable = hive.getTable(td.getTableName()); List<Index> tblIndexes = IndexUtils.getAllIndexes(srcTable, (short)-1); Map<String, String> partSpec = ltd.getPartitionSpec(); if (partSpec == null || partSpec.size() == 0) { //unpartitioned table, update whole index doIndexUpdate(tblIndexes); } else { doIndexUpdate(tblIndexes, partSpec); } } return tasks; }
Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); list.add(partn);
partitionSpecs.add(tbd.getPartitionSpec());
StringBuilder mesg = new StringBuilder("Loading data to table ") .append( tbd.getTable().getTableName()); if (tbd.getPartitionSpec().size() > 0) { mesg.append(" partition ("); Map<String, String> partSpec = tbd.getPartitionSpec(); for (String key: partSpec.keySet()) { mesg.append(key).append('=').append(partSpec.get(key)).append(", "); if (tbd.getPartitionSpec().size() == 0) { dc = new DataContainer(table.getTTable()); db.loadTable(new Path(tbd.getSourceDir()), tbd.getTable() LOG.info("Partition is: " + tbd.getPartitionSpec().toString()); new Path(tbd.getSourceDir()), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), dpCtx.getNumDPCols(), tbd.getPartitionSpec(), tbd.getReplace(), tbd.getHoldDDLTime()); Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); dc = new DataContainer(table.getTTable(), partn.getTPartition());
Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); list.add(partn);
StringBuilder mesg = new StringBuilder("Loading data to table ") .append( tbd.getTable().getTableName()); if (tbd.getPartitionSpec().size() > 0) { mesg.append(" partition ("); Map<String, String> partSpec = tbd.getPartitionSpec(); for (String key: partSpec.keySet()) { mesg.append(key).append('=').append(partSpec.get(key)).append(", "); if (tbd.getPartitionSpec().size() == 0) { dc = new DataContainer(table.getTTable()); db.loadTable(tbd.getSourcePath(), tbd.getTable() LOG.info("Partition is: " + tbd.getPartitionSpec().toString()); tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), dpCtx.getNumDPCols(), tbd.getPartitionSpec()); db.validatePartitionNameCharacters(partVals); db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), tbd.getHoldDDLTime(), tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(), work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID); Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);