private ArrayList<ExprNodeDesc> getPartitionColsFromBucketCols(String dest, QB qb, Table tab, TableDesc table_desc, Operator input, boolean convert) throws SemanticException { List<String> tabBucketCols = tab.getBucketCols(); List<FieldSchema> tabCols = tab.getCols(); // Partition by the bucketing column List<Integer> posns = new ArrayList<Integer>(); for (String bucketCol : tabBucketCols) { int pos = 0; for (FieldSchema tabCol : tabCols) { if (bucketCol.equals(tabCol.getName())) { posns.add(pos); break; } pos++; } } return genConvertCol(dest, qb, tab, table_desc, input, posns, convert); }
@Override public RelDistribution getDistribution() { ImmutableList.Builder<Integer> columnPositions = new ImmutableList.Builder<Integer>(); for (String bucketColumn : this.hiveTblMetadata.getBucketCols()) { for (int i=0; i<this.hiveTblMetadata.getSd().getCols().size(); i++) { FieldSchema field = this.hiveTblMetadata.getSd().getCols().get(i); if (field.getName().equals(bucketColumn)) { columnPositions.add(i); break; } } } return new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED, columnPositions.build()); }
private ArrayList<ExprNodeDesc> getPartitionColsFromBucketCols(String dest, QB qb, Table tab, TableDesc table_desc, Operator input, boolean convert) throws SemanticException { List<String> tabBucketCols = tab.getBucketCols(); List<FieldSchema> tabCols = tab.getCols(); // Partition by the bucketing column List<Integer> posns = new ArrayList<Integer>(); for (String bucketCol : tabBucketCols) { int pos = 0; for (FieldSchema tabCol : tabCols) { if (bucketCol.equals(tabCol.getName())) { posns.add(pos); break; } pos++; } } return genConvertCol(dest, qb, tab, table_desc, input, posns, convert); }
private void analyzeAlterTableBucketNum(ASTNode ast, String tblName, HashMap<String, String> partSpec) throws SemanticException { Table tab = getTable(tblName, true); if (tab.getBucketCols() == null || tab.getBucketCols().isEmpty()) { throw new SemanticException(ErrorMsg.ALTER_BUCKETNUM_NONBUCKETIZED_TBL.getMsg()); } validateAlterTableType(tab, AlterTableTypes.ALTERBUCKETNUM); inputs.add(new ReadEntity(tab)); int bucketNum = Integer.parseInt(ast.getChild(0).getText()); AlterTableDesc alterBucketNum = new AlterTableDesc(tblName, partSpec, bucketNum); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterBucketNum))); }
private void analyzeAlterTableBucketNum(ASTNode ast, String tblName, HashMap<String, String> partSpec) throws SemanticException { Table tab = getTable(tblName, true); if (tab.getBucketCols() == null || tab.getBucketCols().isEmpty()) { throw new SemanticException(ErrorMsg.ALTER_BUCKETNUM_NONBUCKETIZED_TBL.getMsg()); } validateAlterTableType(tab, AlterTableTypes.ALTERBUCKETNUM); inputs.add(new ReadEntity(tab)); int bucketNum = Integer.parseInt(ast.getChild(0).getText()); AlterTableDesc alterBucketNum = new AlterTableDesc(tblName, partSpec, bucketNum); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterBucketNum), conf)); }
public Integer splitCount(HiveTableScan scan, RelMetadataQuery mq) { Integer splitCount; RelOptHiveTable table = (RelOptHiveTable) scan.getTable(); List<String> bucketCols = table.getHiveTableMD().getBucketCols(); if (bucketCols != null && !bucketCols.isEmpty()) { splitCount = table.getHiveTableMD().getNumBuckets(); } else { splitCount = splitCountRepartition(scan, mq); if (splitCount == null) { throw new RuntimeException("Could not get split count for table: " + scan.getTable().getQualifiedName()); } } return splitCount; }
public Integer splitCount(HiveTableScan scan, RelMetadataQuery mq) { Integer splitCount; RelOptHiveTable table = (RelOptHiveTable) scan.getTable(); List<String> bucketCols = table.getHiveTableMD().getBucketCols(); if (bucketCols != null && !bucketCols.isEmpty()) { splitCount = table.getHiveTableMD().getNumBuckets(); } else { splitCount = splitCountRepartition(scan, mq); if (splitCount == null) { throw new RuntimeException("Could not get split count for table: " + scan.getTable().getQualifiedName()); } } return splitCount; }
@Override public RelDistribution getDistribution() { ImmutableList.Builder<Integer> columnPositions = new ImmutableList.Builder<Integer>(); for (String bucketColumn : this.hiveTblMetadata.getBucketCols()) { for (int i=0; i<this.hiveTblMetadata.getSd().getCols().size(); i++) { FieldSchema field = this.hiveTblMetadata.getSd().getCols().get(i); if (field.getName().equals(bucketColumn)) { columnPositions.add(i); break; } } } return new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED, columnPositions.build()); }
private boolean checkTable(Table table, List<Integer> bucketPositionsDest, List<Integer> sortPositionsDest, List<Integer> sortOrderDest, int numBucketsDest) { // The bucketing and sorting positions should exactly match int numBuckets = table.getNumBuckets(); if (numBucketsDest != numBuckets) { return false; } List<Integer> tableBucketPositions = getBucketPositions(table.getBucketCols(), table.getCols()); List<Integer> sortPositions = getSortPositions(table.getSortCols(), table.getCols()); List<Integer> sortOrder = getSortOrder(table.getSortCols(), table.getCols()); return bucketPositionsDest.equals(tableBucketPositions) && sortPositionsDest.equals(sortPositions) && sortOrderDest.equals(sortOrder); }
/** * Assert that we are not asked to update a bucketing column or partition column. * @param colName it's the A in "SET A = B" */ protected void checkValidSetClauseTarget(ASTNode colName, Table targetTable) throws SemanticException { String columnName = normalizeColName(colName.getText()); // Make sure this isn't one of the partitioning columns, that's not supported. for (FieldSchema fschema : targetTable.getPartCols()) { if (fschema.getName().equalsIgnoreCase(columnName)) { throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg()); } } //updating bucket column should move row from one file to another - not supported if (targetTable.getBucketCols() != null && targetTable.getBucketCols().contains(columnName)) { throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE, columnName); } boolean foundColumnInTargetTable = false; for (FieldSchema col : targetTable.getCols()) { if (columnName.equalsIgnoreCase(col.getName())) { foundColumnInTargetTable = true; break; } } if (!foundColumnInTargetTable) { throw new SemanticException(ErrorMsg.INVALID_TARGET_COLUMN_IN_SET_CLAUSE, colName.getText(), targetTable.getFullyQualifiedName()); } }
private boolean checkTable(Table table, List<Integer> bucketPositionsDest, List<Integer> sortPositionsDest, List<Integer> sortOrderDest, int numBucketsDest) { // The bucketing and sorting positions should exactly match int numBuckets = table.getNumBuckets(); if (numBucketsDest != numBuckets) { return false; } List<Integer> tableBucketPositions = getBucketPositions(table.getBucketCols(), table.getCols()); List<Integer> sortPositions = getSortPositions(table.getSortCols(), table.getCols()); List<Integer> sortOrder = getSortOrder(table.getSortCols(), table.getCols()); return bucketPositionsDest.equals(tableBucketPositions) && sortPositionsDest.equals(sortPositions) && sortOrderDest.equals(sortOrder); }
List<String> bucketCols = table.getBucketCols(); if (bucketCols != null && !bucketCols.isEmpty()) {
int numBuckets = -1; if (isBucketed) { bucketColsList.add(table.getBucketCols()); numBuckets = table.getNumBuckets(); List<String> sortCols = new ArrayList<String>();
int numBuckets = -1; if (isBucketed) { bucketColsList.add(table.getBucketCols()); numBuckets = table.getNumBuckets(); List<String> sortCols = new ArrayList<String>();
List<String> bucketCols = table.getBucketCols(); return matchBucketSortCols(groupByCols, bucketCols, sortCols); } else {
/** * Assert that we are not asked to update a bucketing column or partition column * @param colName it's the A in "SET A = B" */ private void checkValidSetClauseTarget(ASTNode colName, Table targetTable) throws SemanticException { String columnName = normalizeColName(colName.getText()); // Make sure this isn't one of the partitioning columns, that's not supported. for (FieldSchema fschema : targetTable.getPartCols()) { if (fschema.getName().equalsIgnoreCase(columnName)) { throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg()); } } //updating bucket column should move row from one file to another - not supported if(targetTable.getBucketCols() != null && targetTable.getBucketCols().contains(columnName)) { throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE,columnName); } boolean foundColumnInTargetTable = false; for(FieldSchema col : targetTable.getCols()) { if(columnName.equalsIgnoreCase(col.getName())) { foundColumnInTargetTable = true; break; } } if(!foundColumnInTargetTable) { throw new SemanticException(ErrorMsg.INVALID_TARGET_COLUMN_IN_SET_CLAUSE, colName.getText(), getDotName(new String[] {targetTable.getDbName(), targetTable.getTableName()})); } } private ASTNode findLHSofAssignment(ASTNode assignment) {
final int nbuckets = tbl.getNumBuckets(); ctxt.setNumBuckets(nbuckets); ctxt.setBucketCols(tbl.getBucketCols()); ctxt.setSchema(tbl.getFields()); if (tbl.isPartitioned()) {
private void alterPartitionSpecInMemory(Table tbl, Map<String, String> partSpec, org.apache.hadoop.hive.metastore.api.Partition tpart, boolean inheritTableSpecs, String partPath) throws HiveException, InvalidOperationException { LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : " + partSpec); if (inheritTableSpecs) { tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); tpart.getSd().getSerdeInfo().setParameters( tbl.getTTable().getSd().getSerdeInfo().getParameters()); tpart.getSd().setBucketCols(tbl.getBucketCols()); tpart.getSd().setNumBuckets(tbl.getNumBuckets()); tpart.getSd().setSortCols(tbl.getSortCols()); } if (partPath == null || partPath.trim().equals("")) { throw new HiveException("new partition path should not be null or empty."); } tpart.getSd().setLocation(partPath); }
private void alterPartitionSpecInMemory(Table tbl, Map<String, String> partSpec, org.apache.hadoop.hive.metastore.api.Partition tpart, boolean inheritTableSpecs, String partPath) throws HiveException, InvalidOperationException { LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : " + partSpec); if (inheritTableSpecs) { tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); tpart.getSd().getSerdeInfo().setParameters( tbl.getTTable().getSd().getSerdeInfo().getParameters()); tpart.getSd().setBucketCols(tbl.getBucketCols()); tpart.getSd().setNumBuckets(tbl.getNumBuckets()); tpart.getSd().setSortCols(tbl.getSortCols()); } if (partPath == null || partPath.trim().equals("")) { throw new HiveException("new partition path should not be null or empty."); } tpart.getSd().setLocation(partPath); }
List<String> bucketCols = tbl.getBucketCols(); bucketCols.add("col1"); try {