pstmt4.setString(2, partition.getPartitionName()); pstmt4.setInt(3, tableId); pstmt4.setString(4, partitionKey.getColumnName()); pstmt4.setString(5, partitionKey.getPartitionValue());
/** * Add partition information to TableStats for storing to CatalogStore. * * @param partition partition name * @throws IOException */ private void addPartition(String partition) throws IOException { PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); for(int i = 0; i < partitionKeyPairs.length; i++) { String partitionKeyPair = partitionKeyPairs[i]; String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); // Partition path have been escaped to avoid URISyntaxException. But partition value of partition keys table // need to contain unescaped value for comparing filter conditions in select statement. keyBuilder.setPartitionValue(StringUtils.unescapePathName(split[1])); builder.addPartitionKeys(keyBuilder.build()); } if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, it get the uri from staging directory. int endIndex = storeTablePath.toString().indexOf(FileTablespace.TMP_STAGING_DIR_PREFIX); String outputPath = storeTablePath.toString().substring(0, endIndex); builder.setPath(outputPath + partition); } else { builder.setPath(this.plan.getUri().toString() + "/" + partition); } context.addPartition(builder.build()); }
public int getSerializedSize() { int size = memoizedSerializedSize; if (size != -1) return size; size = 0; if (((bitField0_ & 0x00000001) == 0x00000001)) { size += com.google.protobuf.CodedOutputStream .computeBytesSize(1, getColumnNameBytes()); } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream .computeBytesSize(2, getParentColumnNameBytes()); } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream .computeBytesSize(3, getPartitionValueBytes()); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; }
pstmt4.setString(2, partition.getPartitionName()); pstmt4.setInt(3, tableId); pstmt4.setString(4, partitionKey.getColumnName()); pstmt4.setString(5, partitionKey.getPartitionValue());
values.add(keyProto.getPartitionValue());
pstmt2.setString(2, partition.getPartitionName()); pstmt2.setInt(3, tableId); pstmt2.setString(4, partitionKey.getColumnName()); pstmt2.setString(5, partitionKey.getPartitionValue()); pstmt2.addBatch(); pstmt2.clearParameters();
CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder();
pstmt2.setString(2, partition.getPartitionName()); pstmt2.setInt(3, tableId); pstmt2.setString(4, partitionKey.getColumnName()); pstmt2.setString(5, partitionKey.getPartitionValue()); pstmt2.addBatch(); pstmt2.clearParameters();
assertEquals(partitions.size(), 1); assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2"); assertEquals(partitions.get(0).getPartitionKeysList().get(0).getColumnName(), "col3"); assertEquals(partitions.get(0).getPartitionKeysList().get(0).getPartitionValue(), "1"); assertEquals(partitions.get(0).getPartitionKeysList().get(1).getColumnName(), "col4"); assertEquals(partitions.get(0).getPartitionKeysList().get(1).getPartitionValue(), "2");
private void addPartition(String databaseName, String tableName, CatalogProtos.PartitionDescProto partitionDescProto) { HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; try { client = clientPool.getClient(); Partition partition = new Partition(); partition.setDbName(databaseName); partition.setTableName(tableName); Map<String, String> params = new HashMap<>(); params.put(StatsSetupConst.TOTAL_SIZE, Long.toString(partitionDescProto.getNumBytes())); partition.setParameters(params); List<String> values = Lists.newArrayList(); for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { values.add(keyProto.getPartitionValue()); } partition.setValues(values); Table table = client.getHiveClient().getTable(databaseName, tableName); StorageDescriptor sd = table.getSd(); sd.setLocation(partitionDescProto.getPath()); partition.setSd(sd); client.getHiveClient().add_partition(partition); } catch (Exception e) { throw new TajoInternalError(e); } finally { if (client != null) { client.release(); } } }
/** * Get partition key/value list and partition name * * ex) partition key/value list : * - col1, 2015-07-01 * - col2, tajo * partition name : col1=2015-07-01/col2=tajo * * @param columns partition column names * @param values partition values * @return partition key/value list and partition name */ public static Pair<List<PartitionKeyProto>, String> getPartitionKeyNamePair(String[] columns, String[] values) { Pair<List<PartitionKeyProto>, String> pair = null; List<PartitionKeyProto> partitionKeyList = new ArrayList<>(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < columns.length; i++) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(columns[i]); builder.setPartitionValue(values[i]); if (i > 0) { sb.append("/"); } sb.append(columns[i]).append("=").append(values[i]); partitionKeyList.add(builder.build()); } pair = new Pair<>(partitionKeyList, sb.toString()); return pair; }
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto)) { return super.equals(obj); } org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto other = (org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto) obj; boolean result = true; result = result && (hasColumnName() == other.hasColumnName()); if (hasColumnName()) { result = result && getColumnName() .equals(other.getColumnName()); } result = result && (hasParentColumnName() == other.hasParentColumnName()); if (hasParentColumnName()) { result = result && getParentColumnName() .equals(other.getParentColumnName()); } result = result && (hasPartitionValue() == other.hasPartitionValue()); if (hasPartitionValue()) { result = result && getPartitionValue() .equals(other.getPartitionValue()); } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; }
/** * Add partition information to TableStats for storing to CatalogStore. * * @param partition partition name * @throws IOException */ private void addPartition(String partition) throws IOException { PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); for (String partitionKeyPair : partitionKeyPairs) { String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); // Partition path have been escaped to avoid URISyntaxException. But partition value of partition keys table // need to contain unescaped value for comparing filter conditions in select statement. keyBuilder.setPartitionValue(StringUtils.unescapePathName(split[1])); builder.addPartitionKeys(keyBuilder.build()); } if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, it get the uri from staging directory. int endIndex = storeTablePath.toString().indexOf(FileTablespace.TMP_STAGING_DIR_PREFIX); String outputPath = storeTablePath.toString().substring(0, endIndex); builder.setPath(outputPath + partition); } else { builder.setPath(this.plan.getUri().toString() + "/" + partition); } context.addPartition(builder.build()); }
/** * Get partition key/value list and partition name * * ex) partition key/value list : * - col1, 2015-07-01 * - col2, tajo * partition name : col1=2015-07-01/col2=tajo * * @param columns partition column names * @param values partition values * @return partition key/value list and partition name */ public static Pair<List<PartitionKeyProto>, String> getPartitionKeyNamePair(String[] columns, String[] values) { Pair<List<PartitionKeyProto>, String> pair = null; List<PartitionKeyProto> partitionKeyList = TUtil.newList(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < columns.length; i++) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(columns[i]); builder.setPartitionValue(values[i]); if (i > 0) { sb.append("/"); } sb.append(columns[i]).append("=").append(values[i]); partitionKeyList.add(builder.build()); } pair = new Pair<List<PartitionKeyProto>, String>(partitionKeyList, sb.toString()); return pair; }
@java.lang.Override public int hashCode() { if (memoizedHashCode != 0) { return memoizedHashCode; } int hash = 41; hash = (19 * hash) + getDescriptorForType().hashCode(); if (hasColumnName()) { hash = (37 * hash) + COLUMNNAME_FIELD_NUMBER; hash = (53 * hash) + getColumnName().hashCode(); } if (hasParentColumnName()) { hash = (37 * hash) + PARENTCOLUMNNAME_FIELD_NUMBER; hash = (53 * hash) + getParentColumnName().hashCode(); } if (hasPartitionValue()) { hash = (37 * hash) + PARTITIONVALUE_FIELD_NUMBER; hash = (53 * hash) + getPartitionValue().hashCode(); } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; }
public org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto buildPartial() { org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto result = new org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0; if (((from_bitField0_ & 0x00000001) == 0x00000001)) { to_bitField0_ |= 0x00000001; } result.columnName_ = columnName_; if (((from_bitField0_ & 0x00000002) == 0x00000002)) { to_bitField0_ |= 0x00000002; } result.parentColumnName_ = parentColumnName_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { to_bitField0_ |= 0x00000004; } result.partitionValue_ = partitionValue_; result.bitField0_ = to_bitField0_; onBuilt(); return result; }
private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) { Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; try { String sql = "SELECT "+ COL_COLUMN_NAME + " , "+ COL_PARTITION_VALUE + " FROM " + TB_PARTTION_KEYS + " WHERE " + COL_PARTITIONS_PK + " = ? "; conn = getConnection(); pstmt = conn.prepareStatement(sql); pstmt.setInt(1, pid); res = pstmt.executeQuery(); while (res.next()) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); partitionDesc.addPartitionKeys(builder); } } catch (SQLException se) { throw new TajoInternalError(se); } finally { CatalogUtil.closeQuietly(pstmt, res); } }
private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) { ResultSet res = null; String sql = "SELECT "+ COL_COLUMN_NAME + " , "+ COL_PARTITION_VALUE + " FROM " + TB_PARTTION_KEYS + " WHERE " + COL_PARTITIONS_PK + " = ? "; try (PreparedStatement pstmt = getConnection().prepareStatement(sql)) { pstmt.setInt(1, pid); res = pstmt.executeQuery(); while (res.next()) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); partitionDesc.addPartitionKeys(builder); } } catch (SQLException se) { throw new TajoInternalError(se); } finally { CatalogUtil.closeQuietly(res); } }
private PartitionDescProto getPartitionDesc(Path tablePath, Path partitionPath, FileSystem fs) throws IOException { String partitionName = StringUtils.unescapePathName(partitionPath.toString()); int startIndex = partitionName.indexOf(tablePath.toString()) + tablePath.toString().length(); partitionName = partitionName.substring(startIndex + File.separator.length()); CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); String[] partitionKeyPairs = partitionName.split("/"); for(int i = 0; i < partitionKeyPairs.length; i++) { String partitionKeyPair = partitionKeyPairs[i]; String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); keyBuilder.setPartitionValue(split[1]); builder.addPartitionKeys(keyBuilder.build()); } builder.setPath(partitionPath.toString()); return builder.build(); }
public Builder mergeFrom(org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto other) { if (other == org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto.getDefaultInstance()) return this; if (other.hasColumnName()) { bitField0_ |= 0x00000001; columnName_ = other.columnName_; onChanged(); } if (other.hasParentColumnName()) { bitField0_ |= 0x00000002; parentColumnName_ = other.parentColumnName_; onChanged(); } if (other.hasPartitionValue()) { bitField0_ |= 0x00000004; partitionValue_ = other.partitionValue_; onChanged(); } this.mergeUnknownFields(other.getUnknownFields()); return this; }