private PartitionDescProto getPartitionDesc(Path tablePath, Path partitionPath, FileSystem fs) throws IOException { String partitionName = StringUtils.unescapePathName(partitionPath.toString()); int startIndex = partitionName.indexOf(tablePath.toString()) + tablePath.toString().length(); partitionName = partitionName.substring(startIndex + File.separator.length()); CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); String[] partitionKeyPairs = partitionName.split("/"); for(int i = 0; i < partitionKeyPairs.length; i++) { String partitionKeyPair = partitionKeyPairs[i]; String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); keyBuilder.setPartitionValue(split[1]); builder.addPartitionKeys(keyBuilder.build()); } builder.setPath(partitionPath.toString()); return builder.build(); }
/** * Get partition key/value list and partition name * * ex) partition key/value list : * - col1, 2015-07-01 * - col2, tajo * partition name : col1=2015-07-01/col2=tajo * * @param columns partition column names * @param values partition values * @return partition key/value list and partition name */ public static Pair<List<PartitionKeyProto>, String> getPartitionKeyNamePair(String[] columns, String[] values) { Pair<List<PartitionKeyProto>, String> pair = null; List<PartitionKeyProto> partitionKeyList = new ArrayList<>(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < columns.length; i++) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(columns[i]); builder.setPartitionValue(values[i]); if (i > 0) { sb.append("/"); } sb.append(columns[i]).append("=").append(values[i]); partitionKeyList.add(builder.build()); } pair = new Pair<>(partitionKeyList, sb.toString()); return pair; }
/** * Get partition key/value list and partition name * * ex) partition key/value list : * - col1, 2015-07-01 * - col2, tajo * partition name : col1=2015-07-01/col2=tajo * * @param columns partition column names * @param values partition values * @return partition key/value list and partition name */ public static Pair<List<PartitionKeyProto>, String> getPartitionKeyNamePair(String[] columns, String[] values) { Pair<List<PartitionKeyProto>, String> pair = null; List<PartitionKeyProto> partitionKeyList = TUtil.newList(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < columns.length; i++) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(columns[i]); builder.setPartitionValue(values[i]); if (i > 0) { sb.append("/"); } sb.append(columns[i]).append("=").append(values[i]); partitionKeyList.add(builder.build()); } pair = new Pair<List<PartitionKeyProto>, String>(partitionKeyList, sb.toString()); return pair; }
CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder();
/** * Add partition information to TableStats for storing to CatalogStore. * * @param partition partition name * @throws IOException */ private void addPartition(String partition) throws IOException { PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); for (String partitionKeyPair : partitionKeyPairs) { String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); // Partition path have been escaped to avoid URISyntaxException. But partition value of partition keys table // need to contain unescaped value for comparing filter conditions in select statement. keyBuilder.setPartitionValue(StringUtils.unescapePathName(split[1])); builder.addPartitionKeys(keyBuilder.build()); } if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, it get the uri from staging directory. int endIndex = storeTablePath.toString().indexOf(FileTablespace.TMP_STAGING_DIR_PREFIX); String outputPath = storeTablePath.toString().substring(0, endIndex); builder.setPath(outputPath + partition); } else { builder.setPath(this.plan.getUri().toString() + "/" + partition); } context.addPartition(builder.build()); }
/** * Add partition information to TableStats for storing to CatalogStore. * * @param partition partition name * @throws IOException */ private void addPartition(String partition) throws IOException { PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); for(int i = 0; i < partitionKeyPairs.length; i++) { String partitionKeyPair = partitionKeyPairs[i]; String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); // Partition path have been escaped to avoid URISyntaxException. But partition value of partition keys table // need to contain unescaped value for comparing filter conditions in select statement. keyBuilder.setPartitionValue(StringUtils.unescapePathName(split[1])); builder.addPartitionKeys(keyBuilder.build()); } if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, it get the uri from staging directory. int endIndex = storeTablePath.toString().indexOf(FileTablespace.TMP_STAGING_DIR_PREFIX); String outputPath = storeTablePath.toString().substring(0, endIndex); builder.setPath(outputPath + partition); } else { builder.setPath(this.plan.getUri().toString() + "/" + partition); } context.addPartition(builder.build()); }
private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) { Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; try { String sql = "SELECT "+ COL_COLUMN_NAME + " , "+ COL_PARTITION_VALUE + " FROM " + TB_PARTTION_KEYS + " WHERE " + COL_PARTITIONS_PK + " = ? "; conn = getConnection(); pstmt = conn.prepareStatement(sql); pstmt.setInt(1, pid); res = pstmt.executeQuery(); while (res.next()) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); partitionDesc.addPartitionKeys(builder); } } catch (SQLException se) { throw new TajoInternalError(se); } finally { CatalogUtil.closeQuietly(pstmt, res); } }
private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) { ResultSet res = null; String sql = "SELECT "+ COL_COLUMN_NAME + " , "+ COL_PARTITION_VALUE + " FROM " + TB_PARTTION_KEYS + " WHERE " + COL_PARTITIONS_PK + " = ? "; try (PreparedStatement pstmt = getConnection().prepareStatement(sql)) { pstmt.setInt(1, pid); res = pstmt.executeQuery(); while (res.next()) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); partitionDesc.addPartitionKeys(builder); } } catch (SQLException se) { throw new TajoInternalError(se); } finally { CatalogUtil.closeQuietly(res); } }
private PartitionDescProto getPartitionDesc(Path tablePath, Path partitionPath, FileSystem fs) throws IOException { String partitionName = StringUtils.unescapePathName(partitionPath.toString()); int startIndex = partitionName.indexOf(tablePath.toString()) + tablePath.toString().length(); partitionName = partitionName.substring(startIndex + File.separator.length()); CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); String[] partitionKeyPairs = partitionName.split("/"); for (String partitionKeyPair : partitionKeyPairs) { String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); keyBuilder.setPartitionValue(split[1]); builder.addPartitionKeys(keyBuilder.build()); } builder.setPath(partitionPath.toString()); ContentSummary contentSummary = fs.getContentSummary(partitionPath); builder.setNumBytes(contentSummary.getLength()); return builder.build(); }
public Builder toBuilder() { return newBuilder(this); }
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder(org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto prototype) {
public static Builder newBuilder(org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto prototype) { return newBuilder().mergeFrom(prototype); } public Builder toBuilder() { return newBuilder(this); }