public static HiveLocationDescriptor forTable(Table table, FileSystem fs, Properties properties) throws IOException { return new HiveLocationDescriptor(table.getDataLocation(), HiveUtils.getInputFormat(table.getTTable().getSd()), fs, properties); }
public static HiveLocationDescriptor forPartition(Partition partition, FileSystem fs, Properties properties) throws IOException { return new HiveLocationDescriptor(partition.getDataLocation(), HiveUtils.getInputFormat(partition.getTPartition().getSd()), fs, properties); }
int addPartitionDeregisterSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority, Table table, Partition partition) throws IOException { int stepPriority = initialPriority; Collection<Path> partitionPaths = Lists.newArrayList(); if (this.deleteMethod == DeregisterFileDeleteMethod.RECURSIVE) { partitionPaths = Lists.newArrayList(partition.getDataLocation()); } else if (this.deleteMethod == DeregisterFileDeleteMethod.INPUT_FORMAT) { InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(partition.getTPartition().getSd()); HiveLocationDescriptor targetLocation = new HiveLocationDescriptor(partition.getDataLocation(), inputFormat, this.targetFs, this.dataset.getProperties()); partitionPaths = targetLocation.getPaths().keySet(); } else if (this.deleteMethod == DeregisterFileDeleteMethod.NO_DELETE) { partitionPaths = Lists.newArrayList(); } if (!partitionPaths.isEmpty()) { DeleteFileCommitStep deletePaths = DeleteFileCommitStep.fromPaths(this.targetFs, partitionPaths, this.dataset.getProperties(), table.getDataLocation()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deletePaths, stepPriority++)); } PartitionDeregisterStep deregister = new PartitionDeregisterStep(table.getTTable(), partition.getTPartition(), this.targetURI, this.hiveRegProps); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deregister, stepPriority++)); return stepPriority; }
break; case INPUT_FORMAT: InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(table.getSd());
public static HiveLocationDescriptor forPartition(Partition partition, FileSystem fs, Properties properties) throws IOException { return new HiveLocationDescriptor(partition.getDataLocation(), HiveUtils.getInputFormat(partition.getTPartition().getSd()), fs, properties); }
public static HiveLocationDescriptor forTable(Table table, FileSystem fs, Properties properties) throws IOException { return new HiveLocationDescriptor(table.getDataLocation(), HiveUtils.getInputFormat(table.getTTable().getSd()), fs, properties); }
int addPartitionDeregisterSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority, Table table, Partition partition) throws IOException { int stepPriority = initialPriority; Collection<Path> partitionPaths = Lists.newArrayList(); if (this.deleteMethod == DeregisterFileDeleteMethod.RECURSIVE) { partitionPaths = Lists.newArrayList(partition.getDataLocation()); } else if (this.deleteMethod == DeregisterFileDeleteMethod.INPUT_FORMAT) { InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(partition.getTPartition().getSd()); HiveLocationDescriptor targetLocation = new HiveLocationDescriptor(partition.getDataLocation(), inputFormat, this.targetFs, this.dataset.getProperties()); partitionPaths = targetLocation.getPaths().keySet(); } else if (this.deleteMethod == DeregisterFileDeleteMethod.NO_DELETE) { partitionPaths = Lists.newArrayList(); } if (!partitionPaths.isEmpty()) { DeleteFileCommitStep deletePaths = DeleteFileCommitStep.fromPaths(this.targetFs, partitionPaths, this.dataset.getProperties(), table.getDataLocation()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deletePaths, stepPriority++)); } PartitionDeregisterStep deregister = new PartitionDeregisterStep(table.getTTable(), partition.getTPartition(), this.targetURI, this.hiveRegProps); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deregister, stepPriority++)); return stepPriority; }
break; case INPUT_FORMAT: InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(table.getSd());