public HCatPartition next() throws HCatException { return new HCatPartition(hcatTable, iterator.next()); }
public HCatPartition(HCatPartition rhs, Map<String, String> partitionKeyValues, String location) throws HCatException { this.hcatTable = rhs.hcatTable; this.tableName = rhs.tableName; this.dbName = rhs.dbName; this.sd = new StorageDescriptor(rhs.sd); this.sd.setLocation(location); this.columns = getColumns(this.sd); this.createTime = (int) (System.currentTimeMillis() / 1000); this.lastAccessTime = -1; this.values = new ArrayList<String>(hcatTable.getPartCols().size()); for (HCatFieldSchema partField : hcatTable.getPartCols()) { if (!partitionKeyValues.containsKey(partField.getName())) { throw new HCatException("Missing value for partition-key \'" + partField.getName() + "\' in table: " + hcatTable.getDbName() + "." + hcatTable.getTableName()); } else { values.add(partitionKeyValues.get(partField.getName())); } } }
@Override public HCatPartition deserializePartition(String hcatPartitionStringRep) throws HCatException { HCatPartition hcatPartition = MetadataSerializer.get().deserializePartition(hcatPartitionStringRep); hcatPartition.hcatTable(getTable(hcatPartition.getDatabaseName(), hcatPartition.getTableName())); return hcatPartition; }
partitionSpec_1.put("grid", "AB"); partitionSpec_1.put("dt", "2011_12_31"); HCatPartition sourcePartition_1 = new HCatPartition(sourceTable, partitionSpec_1, makePartLocation(sourceTable,partitionSpec_1)); assertEquals("Column schema doesn't match.", sourceTable.getCols(), addedPartition_1.getColumns()); assertEquals("InputFormat doesn't match.", sourceTable.getInputFileFormat(), addedPartition_1.getInputFormat()); assertEquals("OutputFormat doesn't match.", sourceTable.getOutputFileFormat(), addedPartition_1.getOutputFormat()); assertEquals("SerDe doesn't match.", sourceTable.getSerdeLib(), addedPartition_1.getSerDe()); assertEquals("SerDe params don't match.", sourceTable.getSerdeParams(), addedPartition_1.getSerdeParams()); partitionSpec_2.put("grid", "AB"); partitionSpec_2.put("dt", "2012_01_01"); HCatPartition sourcePartition_2 = new HCatPartition(sourceTable, partitionSpec_2, makePartLocation(sourceTable,partitionSpec_2)); sourceMetaStore().addPartition(HCatAddPartitionDesc.create(sourcePartition_2).build()); HCatPartition sourcePartition = sourcePartitions.get(i), targetPartition = targetPartitions.get(i); assertEquals("Column schema doesn't match.", sourcePartition.getColumns(), targetPartition.getColumns()); assertEquals("InputFormat doesn't match.", sourcePartition.getInputFormat(), targetPartition.getInputFormat()); assertEquals("OutputFormat doesn't match.", sourcePartition.getOutputFormat(), targetPartition.getOutputFormat()); assertEquals("SerDe doesn't match.", sourcePartition.getSerDe(), targetPartition.getSerDe()); assertEquals("SerDe params don't match.", sourcePartition.getSerdeParams(), targetPartition.getSerdeParams());
@Override public String serializePartition(HCatPartition hcatPartition) throws HCatException { try { return new TSerializer(new TJSONProtocol.Factory()) .toString(hcatPartition.toHivePartition(), "UTF-8"); } catch (TException exception) { throw new HCatException("Could not serialize HCatPartition: " + hcatPartition, exception); } }
partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2011_12_31"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table, partitionSpec))).build()); partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2012_01_01"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table, partitionSpec))).build()); partitionSpec.put("dt", "2012_01_01"); partitionSpec.put("grid", "OB"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table, partitionSpec))).build()); partitionSpec.put("dt", "2012_01_01"); partitionSpec.put("grid", "XB"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table, partitionSpec))).build()); assertArrayEquals("Mismatched partition.", new String[]{"2011_12_31", "AB"}, partitions.get(0).getValues().toArray()); List<HCatFieldSchema> partColumns = partitions.get(0).getPartColumns(); assertEquals(2, partColumns.size()); assertEquals("dt", partColumns.get(0).getName());
Map<String, String> props1 = new HashMap<String,String>(); props1.put(ReplicationUtils.REPL_STATE_ID,String.valueOf(evid - 5)); HCatPartition ptnToAdd1 = (new HCatPartition(tableCreated, ptnDesc1, TestHCatClient.makePartLocation(tableCreated,ptnDesc1))).parameters(props1); client.addPartition(HCatAddPartitionDesc.create(ptnToAdd1).build()); Map<String, String> props2 = new HashMap<String,String>(); props2.put(ReplicationUtils.REPL_STATE_ID, String.valueOf(evid + 5)); HCatPartition ptnToAdd2 = (new HCatPartition(tableCreated, ptnDesc2, TestHCatClient.makePartLocation(tableCreated,ptnDesc2))).parameters(props2); client.addPartition(HCatAddPartitionDesc.create(ptnToAdd2).build());
partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2011_12_31"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table,partitionSpec))).build()); partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2012_01_01"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table,partitionSpec))).build()); partitionSpec.put("dt", "2012_01_01"); partitionSpec.put("grid", "OB"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table,partitionSpec))).build()); partitionSpec.put("dt", "2012_01_01"); partitionSpec.put("grid", "XB"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table,partitionSpec))).build()); assertArrayEquals("Mismatched partition.", new String[]{"2012_01_01", "AB"}, partitions.get(0).getValues().toArray()); assertArrayEquals("Mismatched partition.", new String[]{"2012_01_01", "OB"}, partitions.get(1).getValues().toArray()); assertArrayEquals("Mismatched partition.", new String[]{"2012_01_01", "XB"}, partitions.get(2).getValues().toArray());
/** * Gets the table name. * * @return the table name */ @Deprecated // @deprecated in favour of {@link HCatPartition.#getTableName()}. To be removed in Hive 0.16. public String getTableName() { return hcatPartition == null? tableName : hcatPartition.getTableName(); }
/** * Gets the partition spec. * * @return the partition spec */ @Deprecated // @deprecated in favour of {@link HCatPartition.#getPartitionKeyValMap()}. To be removed in Hive 0.16. public Map<String, String> getPartitionSpec() { return hcatPartition == null? partitionKeyValues : hcatPartition.getPartitionKeyValMap(); }
/** * Gets the database name. * * @return the database name */ @Deprecated // @deprecated in favour of {@link HCatPartition.#getDatabaseName()}. To be removed in Hive 0.16. public String getDatabaseName() { return hcatPartition == null? dbName : hcatPartition.getDatabaseName(); }
/** * Gets the location. * * @return the location */ @Deprecated // @deprecated in favour of {@link HCatPartition.#getLocation()}. To be removed in Hive 0.16. public String getLocation() { return hcatPartition == null? location : hcatPartition.getLocation(); }
/** * Gets the last known replication state of the provided partition. * This is applicable only if it is the destination of a replication * and has had data replicated into it via imports previously. * If that is not available, but parent table is provided, * defaults to parent table's replication state. If that is also * unknown, defaults to 0. */ public static long getLastReplicationId(HCatPartition ptn, @Nullable HCatTable parentTable) { Map<String,String> parameters = ptn.getParameters(); if (parameters != null){ if (parameters.containsKey(REPL_STATE_ID)){ return Long.parseLong(parameters.get(REPL_STATE_ID)); } } if (parentTable != null){ return getLastReplicationId(parentTable); } return 0l; // default is to return earliest possible state. }
partitionSpec_1.put("grid", "AB"); partitionSpec_1.put("dt", "2011_12_31"); HCatPartition sourcePartition_1 = new HCatPartition(sourceTable, partitionSpec_1, makePartLocation(sourceTable,partitionSpec_1)); assertEquals("Column schema doesn't match.", sourceTable.getCols(), addedPartition_1.getColumns()); assertEquals("InputFormat doesn't match.", sourceTable.getInputFileFormat(), addedPartition_1.getInputFormat()); assertEquals("OutputFormat doesn't match.", sourceTable.getOutputFileFormat(), addedPartition_1.getOutputFormat()); assertEquals("SerDe doesn't match.", sourceTable.getSerdeLib(), addedPartition_1.getSerDe()); assertEquals("SerDe params don't match.", sourceTable.getSerdeParams(), addedPartition_1.getSerdeParams()); partitionSpec_2.put("grid", "AB"); partitionSpec_2.put("dt", "2012_01_01"); HCatPartition sourcePartition_2 = new HCatPartition(sourceTable, partitionSpec_2, makePartLocation(sourceTable,partitionSpec_2)); sourceMetaStore().addPartition(HCatAddPartitionDesc.create(sourcePartition_2).build()); HCatPartition sourcePartition = sourceIterator.next(); HCatPartition targetPartition = targetIterator.next(); assertEquals("Column schema doesn't match.", sourcePartition.getColumns(), targetPartition.getColumns()); assertEquals("InputFormat doesn't match.", sourcePartition.getInputFormat(), targetPartition.getInputFormat()); assertEquals("OutputFormat doesn't match.", sourcePartition.getOutputFormat(), targetPartition.getOutputFormat()); assertEquals("SerDe doesn't match.", sourcePartition.getSerDe(), targetPartition.getSerDe()); assertEquals("SerDe params don't match.", sourcePartition.getSerdeParams(), targetPartition.getSerdeParams());
ptnList.add(hCatPartition.toHivePartition());
assertNotNull(tableCreated); HCatPartition ptnToAdd = (new HCatPartition(tableCreated, ptnDesc, TestHCatClient.makePartLocation(tableCreated,ptnDesc))).parameters(props); client.addPartition(HCatAddPartitionDesc.create(ptnToAdd).build()); props2.put(ReplicationUtils.REPL_STATE_ID,String.valueOf(evid - 5)); HCatPartition ptnToAdd2 = (new HCatPartition(tableCreated, ptnDesc, TestHCatClient.makePartLocation(tableCreated,ptnDesc))).parameters(props2); client.addPartition(HCatAddPartitionDesc.create(ptnToAdd2).build());
/** * Gets the table name. * * @return the table name */ @Deprecated // @deprecated in favour of {@link HCatPartition.#getTableName()}. To be removed in Hive 0.16. public String getTableName() { return hcatPartition == null? tableName : hcatPartition.getTableName(); }
/** * Gets the partition spec. * * @return the partition spec */ @Deprecated // @deprecated in favour of {@link HCatPartition.#getPartitionKeyValMap()}. To be removed in Hive 0.16. public Map<String, String> getPartitionSpec() { return hcatPartition == null? partitionKeyValues : hcatPartition.getPartitionKeyValMap(); }
/** * Gets the database name. * * @return the database name */ @Deprecated // @deprecated in favour of {@link HCatPartition.#getDatabaseName()}. To be removed in Hive 0.16. public String getDatabaseName() { return hcatPartition == null? dbName : hcatPartition.getDatabaseName(); }
/** * Gets the location. * * @return the location */ @Deprecated // @deprecated in favour of {@link HCatPartition.#getLocation()}. To be removed in Hive 0.16. public String getLocation() { return hcatPartition == null? location : hcatPartition.getLocation(); }