State props = hiveTable.getProps(); Table table = new Table(); table.setDbName(hiveTable.getDbName()); table.setTableName(hiveTable.getTableName()); table.setParameters(getParameters(props)); if (hiveTable.getCreateTime().isPresent()) { table.setCreateTime(Ints.checkedCast(hiveTable.getCreateTime().get())); if (hiveTable.getLastAccessTime().isPresent()) { table.setLastAccessTime(Ints.checkedCast(hiveTable.getLastAccessTime().get())); if (hiveTable.getOwner().isPresent()) { table.setOwner(hiveTable.getOwner().get()); if (hiveTable.getRetention().isPresent()) { table.setRetention(Ints.checkedCast(hiveTable.getRetention().get())); if (hiveTable.getTableType().isPresent()) { table.setTableType(hiveTable.getTableType().get()); } else { table.setTableType(DEFAULT_TABLE_TYPE.toString()); table.getParameters().put(EXTERNAL, Boolean.TRUE.toString().toUpperCase()); table.setPartitionKeys(getFieldSchemas(hiveTable.getPartitionKeys())); table.setSd(getStorageDescriptor(hiveTable)); return table;
@Override public HiveTable build() { return new HiveTable(this); } }
private static Map<String, String> getAdditionalMetadata(HiveTable table, Optional<HivePartition> partition, Optional<Exception> error) { ImmutableMap.Builder<String, String> builder = ImmutableMap.<String, String> builder().put(DB_NAME, table.getDbName()).put(TABLE_NAME, table.getTableName()); if (table.getLocation().isPresent()) { builder.put("Location", table.getLocation().get()); } if (partition.isPresent()) { builder.put("Partition", partition.get().toString()); } if (error.isPresent()) { builder.put(ERROR_MESSAGE, error.get().getMessage()); } return builder.build(); }
@Override public String toString() { return Objects.toStringHelper(this).omitNullValues().add("path", this.path.toString()) .add("db", this.table.getDbName()).add("table", this.table.getTableName()) .add("partition", this.partition.orNull()).toString(); }
@Override public void execute() throws IOException { HiveTable hiveTable = HiveMetaStoreUtils.getHiveTable(this.table); try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropPartitionIfExists(this.partition.getDbName(), this.partition.getTableName(), hiveTable.getPartitionKeys(), this.partition.getValues()); } }
@SuppressWarnings("unchecked") public T compareRetention() { if (!this.result) { compare(((HiveTable) this.existingUnit).getRetention(), ((HiveTable) this.newUnit).getRetention()); } return (T) this; }
private Table getTableWithCreateTime(Table table, HiveTable referenceTable) { return gettableWithCreateTime(table, Ints.checkedCast(referenceTable.getCreateTime().or(DateTime.now().getMillis() / 1000))); }
@Override public void execute() throws IOException { if (this.verifyBeforeRegistering) { if (!this.hiveSpec.getTable().getLocation().isPresent()) { throw getException("Table does not have a location parameter."); } Path tablePath = new Path(this.hiveSpec.getTable().getLocation().get()); FileSystem fs = this.hiveSpec.getPath().getFileSystem(new Configuration()); if (!fs.exists(tablePath)) { throw getException(String.format("Table location %s does not exist.", tablePath)); } if (this.hiveSpec.getPartition().isPresent()) { if (!this.hiveSpec.getPartition().get().getLocation().isPresent()) { throw getException("Partition does not have a location parameter."); } Path partitionPath = new Path(this.hiveSpec.getPartition().get().getLocation().get()); if (!fs.exists(this.hiveSpec.getPath())) { throw getException(String.format("Partition location %s does not exist.", partitionPath)); } } } try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { log.info("Registering Hive Spec " + this.hiveSpec); ListenableFuture<Void> future = hiveRegister.register(this.hiveSpec); future.get(); } catch (InterruptedException | ExecutionException ie) { throw new IOException("Hive registration was interrupted.", ie); } }
private String getSpecId(HiveSpec spec) { Optional<HivePartition> partition = spec.getPartition(); if (partition.isPresent()) { return String.format("%s.%s@%s", spec.getTable().getDbName(), spec.getTable().getTableName(), Arrays.toString(partition.get().getValues().toArray())); } else { return String.format("%s.%s", spec.getTable().getDbName(), spec.getTable().getTableName()); } }
@Override public void execute() throws IOException { HiveTable hiveTable = HiveMetaStoreUtils.getHiveTable(this.table); try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropPartitionIfExists(this.partition.getDbName(), this.partition.getTableName(), hiveTable.getPartitionKeys(), this.partition.getValues()); } }
@SuppressWarnings("unchecked") public T compareRetention() { if (!this.result) { compare(((HiveTable) this.existingUnit).getRetention(), ((HiveTable) this.newUnit).getRetention()); } return (T) this; }
private Table getTableWithCreateTime(Table table, HiveTable referenceTable) { return gettableWithCreateTime(table, Ints.checkedCast(referenceTable.getCreateTime().or(DateTime.now().getMillis() / 1000))); }
@Override public void execute() throws IOException { if (this.verifyBeforeRegistering) { if (!this.hiveSpec.getTable().getLocation().isPresent()) { throw getException("Table does not have a location parameter."); } Path tablePath = new Path(this.hiveSpec.getTable().getLocation().get()); FileSystem fs = this.hiveSpec.getPath().getFileSystem(new Configuration()); if (!fs.exists(tablePath)) { throw getException(String.format("Table location %s does not exist.", tablePath)); } if (this.hiveSpec.getPartition().isPresent()) { if (!this.hiveSpec.getPartition().get().getLocation().isPresent()) { throw getException("Partition does not have a location parameter."); } Path partitionPath = new Path(this.hiveSpec.getPartition().get().getLocation().get()); if (!fs.exists(this.hiveSpec.getPath())) { throw getException(String.format("Partition location %s does not exist.", partitionPath)); } } } try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { log.info("Registering Hive Spec " + this.hiveSpec); ListenableFuture<Void> future = hiveRegister.register(this.hiveSpec); future.get(); } catch (InterruptedException | ExecutionException ie) { throw new IOException("Hive registration was interrupted.", ie); } }
State props = hiveTable.getProps(); Table table = new Table(); table.setDbName(hiveTable.getDbName()); table.setTableName(hiveTable.getTableName()); table.setParameters(getParameters(props)); if (hiveTable.getCreateTime().isPresent()) { table.setCreateTime(Ints.checkedCast(hiveTable.getCreateTime().get())); if (hiveTable.getLastAccessTime().isPresent()) { table.setLastAccessTime(Ints.checkedCast(hiveTable.getLastAccessTime().get())); if (hiveTable.getOwner().isPresent()) { table.setOwner(hiveTable.getOwner().get()); if (hiveTable.getRetention().isPresent()) { table.setRetention(Ints.checkedCast(hiveTable.getRetention().get())); if (hiveTable.getTableType().isPresent()) { table.setTableType(hiveTable.getTableType().get()); } else { table.setTableType(DEFAULT_TABLE_TYPE.toString()); table.getParameters().put(EXTERNAL, Boolean.TRUE.toString().toUpperCase()); table.setPartitionKeys(getFieldSchemas(hiveTable.getPartitionKeys())); table.setSd(getStorageDescriptor(hiveTable)); return table;
private static Map<String, String> getAdditionalMetadata(HiveSpec spec, Optional<Exception> error) { ImmutableMap.Builder<String, String> builder = ImmutableMap.<String, String> builder().put(DB_NAME, spec.getTable().getDbName()) .put(TABLE_NAME, spec.getTable().getTableName()).put("Path", spec.getPath().toString()); if(spec.getPartition().isPresent()){ builder.put(PARTITIONS, spec.getPartition().get().toString()); } if (error.isPresent()) { builder.put(ERROR_MESSAGE, error.get().getMessage()); } return builder.build(); }
@Override public String toString() { String table = this.hiveSpec.getTable().getDbName() + "." + this.hiveSpec.getTable().getTableName(); String partitionInfo = this.hiveSpec.getPartition().isPresent() ? " partition " + Arrays.toString(this.hiveSpec.getPartition().get().getValues().toArray()) : ""; String location = this.hiveSpec.getPartition().isPresent() ? this.hiveSpec.getPartition().get().getLocation().get() : this.hiveSpec.getTable().getLocation().get(); return String.format("Register %s%s with location %s in Hive metastore %s.", table, partitionInfo, location, this.metastoreURI.isPresent() ? this.metastoreURI.get() : "default"); } }
@Override public HiveTable build() { return new HiveTable(this); } }
@Override public void addOrAlterPartition(HiveTable table, HivePartition partition) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { addOrAlterPartition(client.get(), HiveMetaStoreUtils.getTable(table), partition); } catch (TException te) { throw new IOException( String.format("Failed to add/alter partition %s.%s@%s", table.getDbName(), table.getTableName(), partition.getValues()), te); } }