for (Partition hivePartition : hivePartitions) { CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); builder.setPath(hivePartition.getSd().getLocation()); builder.setPartitionName(partitionName.toString()); builder.setNumBytes(Long.parseLong(params.get(StatsSetupConst.TOTAL_SIZE))); partitions.add(builder.build());
private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) { Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; try { String sql = "SELECT "+ COL_COLUMN_NAME + " , "+ COL_PARTITION_VALUE + " FROM " + TB_PARTTION_KEYS + " WHERE " + COL_PARTITIONS_PK + " = ? "; conn = getConnection(); pstmt = conn.prepareStatement(sql); pstmt.setInt(1, pid); res = pstmt.executeQuery(); while (res.next()) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); partitionDesc.addPartitionKeys(builder); } } catch (SQLException se) { throw new TajoInternalError(se); } finally { CatalogUtil.closeQuietly(pstmt, res); } }
subBuilder.mergeFrom(partition_); partition_ = subBuilder.buildPartial();
subBuilder.mergeFrom(partitionDesc_); partitionDesc_ = subBuilder.buildPartial();
PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setId(res.getInt(COL_PARTITIONS_PK)); builder.setPartitionName(res.getString("PARTITION_NAME")); builder.setPath(res.getString("PATH")); builder.setNumBytes(res.getLong(COL_PARTITION_BYTES)); partitions.add(builder.build());
subBuilder.mergeFrom(partition_); partition_ = subBuilder.buildPartial();
bitField0_ |= 0x00000001; partitionName_ = other.partitionName_; onChanged(); bitField0_ = (bitField0_ & ~0x00000002); } else { ensurePartitionKeysIsMutable(); partitionKeys_.addAll(other.partitionKeys_); onChanged(); partitionKeysBuilder_ = com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? getPartitionKeysFieldBuilder() : null; } else { partitionKeysBuilder_.addAllMessages(other.partitionKeys_); bitField0_ |= 0x00000004; path_ = other.path_; onChanged(); setId(other.getId()); this.mergeUnknownFields(other.getUnknownFields()); return this;
builder.setPartitionName(partitionName); builder.setPath(partition.getSd().getLocation()); keyBuilder.setColumnName(columnName); keyBuilder.setPartitionValue(value); builder.addPartitionKeys(keyBuilder); builder.setNumBytes(Long.parseLong(params.get(StatsSetupConst.TOTAL_SIZE))); return builder.build();
builder.setPath(res.getString("PATH")); builder.setPartitionName(res.getString("PARTITION_NAME")); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); partitions.add(builder.build());
builder.setId(res.getInt(COL_PARTITIONS_PK)); builder.setPath(res.getString("PATH")); builder.setPartitionName(partitionName); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); } else { CatalogUtil.closeQuietly(pstmt, res); return builder.build();
builder.setId(res.getInt(COL_PARTITIONS_PK)); builder.setPath(res.getString("PATH")); builder.setPartitionName(partitionName); builder.setNumBytes(res.getLong(COL_PARTITION_BYTES)); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); } else { CatalogUtil.closeQuietly(res); return builder.build();
builder.setPath(res.getString("PATH")); builder.setPartitionName(res.getString("PARTITION_NAME")); builder.setNumBytes(res.getLong(COL_PARTITION_BYTES)); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); partitions.add(builder.build());
/** * Add partition information to TableStats for storing to CatalogStore. * * @param partition partition name * @throws IOException */ private void addPartition(String partition) throws IOException { PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); for(int i = 0; i < partitionKeyPairs.length; i++) { String partitionKeyPair = partitionKeyPairs[i]; String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); // Partition path have been escaped to avoid URISyntaxException. But partition value of partition keys table // need to contain unescaped value for comparing filter conditions in select statement. keyBuilder.setPartitionValue(StringUtils.unescapePathName(split[1])); builder.addPartitionKeys(keyBuilder.build()); } if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, it get the uri from staging directory. int endIndex = storeTablePath.toString().indexOf(FileTablespace.TMP_STAGING_DIR_PREFIX); String outputPath = storeTablePath.toString().substring(0, endIndex); builder.setPath(outputPath + partition); } else { builder.setPath(this.plan.getUri().toString() + "/" + partition); } context.addPartition(builder.build()); }
/** * Add partition information to TableStats for storing to CatalogStore. * * @param partition partition name * @throws IOException */ private void addPartition(String partition) throws IOException { PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); for (String partitionKeyPair : partitionKeyPairs) { String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); // Partition path have been escaped to avoid URISyntaxException. But partition value of partition keys table // need to contain unescaped value for comparing filter conditions in select statement. keyBuilder.setPartitionValue(StringUtils.unescapePathName(split[1])); builder.addPartitionKeys(keyBuilder.build()); } if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, it get the uri from staging directory. int endIndex = storeTablePath.toString().indexOf(FileTablespace.TMP_STAGING_DIR_PREFIX); String outputPath = storeTablePath.toString().substring(0, endIndex); builder.setPath(outputPath + partition); } else { builder.setPath(this.plan.getUri().toString() + "/" + partition); } context.addPartition(builder.build()); }
/** * <code>optional .PartitionDescProto partition = 2;</code> */ public Builder mergePartition(org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto value) { if (partitionBuilder_ == null) { if (((bitField0_ & 0x00000002) == 0x00000002) && partition_ != org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto.getDefaultInstance()) { partition_ = org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto.newBuilder(partition_).mergeFrom(value).buildPartial(); } else { partition_ = value; } onChanged(); } else { partitionBuilder_.mergeFrom(value); } bitField0_ |= 0x00000002; return this; } /**
/** * <code>optional .PartitionDescProto partition = 2;</code> */ public Builder mergePartition(org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto value) { if (partitionBuilder_ == null) { if (((bitField0_ & 0x00000002) == 0x00000002) && partition_ != org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto.getDefaultInstance()) { partition_ = org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto.newBuilder(partition_).mergeFrom(value).buildPartial(); } else { partition_ = value; } onChanged(); } else { partitionBuilder_.mergeFrom(value); } bitField0_ |= 0x00000002; return this; } /**
private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) { ResultSet res = null; String sql = "SELECT "+ COL_COLUMN_NAME + " , "+ COL_PARTITION_VALUE + " FROM " + TB_PARTTION_KEYS + " WHERE " + COL_PARTITIONS_PK + " = ? "; try (PreparedStatement pstmt = getConnection().prepareStatement(sql)) { pstmt.setInt(1, pid); res = pstmt.executeQuery(); while (res.next()) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); partitionDesc.addPartitionKeys(builder); } } catch (SQLException se) { throw new TajoInternalError(se); } finally { CatalogUtil.closeQuietly(res); } }
/** * <code>optional .PartitionDescProto partitionDesc = 6;</code> */ public Builder mergePartitionDesc(org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto value) { if (partitionDescBuilder_ == null) { if (((bitField0_ & 0x00000020) == 0x00000020) && partitionDesc_ != org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto.getDefaultInstance()) { partitionDesc_ = org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto.newBuilder(partitionDesc_).mergeFrom(value).buildPartial(); } else { partitionDesc_ = value; } onChanged(); } else { partitionDescBuilder_.mergeFrom(value); } bitField0_ |= 0x00000020; return this; } /**
@Override public CatalogProtos.PartitionDescProto getProto() { if (builder == null) { builder = CatalogProtos.PartitionDescProto.newBuilder(); } if(this.partitionName != null) { builder.setPartitionName(this.partitionName); } builder.clearPartitionKeys(); if (this.partitionKeys != null) { for(PartitionKeyProto partitionKey : this.partitionKeys) { builder.addPartitionKeys(partitionKey); } } if(this.path != null) { builder.setPath(this.path); } if(this.numBytes != null) { builder.setNumBytes(this.numBytes); } return builder.build(); }
@Override public CatalogProtos.PartitionDescProto getProto() { if (builder == null) { builder = CatalogProtos.PartitionDescProto.newBuilder(); } if(this.partitionName != null) { builder.setPartitionName(this.partitionName); } builder.clearPartitionKeys(); if (this.partitionKeys != null) { for(PartitionKeyProto partitionKey : this.partitionKeys) { builder.addPartitionKeys(partitionKey); } } if(this.path != null) { builder.setPath(this.path); } return builder.build(); }