/** * Get the name of the dataset from the URI * * @param uri The dataset or view URI * @return The dataset name */ private static String uriToName(URI uri) { return Registration.lookupDatasetUri(URI.create( uri.getRawSchemeSpecificPart())).second().get("dataset"); }
private static String uriToName(URI uri) { return Registration.lookupDatasetUri(URI.create( uri.getRawSchemeSpecificPart())).second().get("dataset"); } }
/** * Get the name of the dataset from the URI * * @param uri The dataset or view URI * @return The dataset name */ private static String uriToName(URI uri) { return Registration.lookupDatasetUri(URI.create( uri.getRawSchemeSpecificPart())).second().get("dataset"); }
static void finishMove(FileSystem fs, List<Pair<Path, Path>> staged) { try { for (Pair<Path, Path> pair : staged) { if (!fs.rename(pair.first(), pair.second())) { throw new IOException( "Failed to rename " + pair.first() + " to " + pair.second()); } } } catch (IOException e) { throw new DatasetIOException("Could not finish replacement", e); } }
protected static Map<String, String> optionsForUri(URI uri) { Preconditions.checkArgument(isDatasetOrViewUri(uri.toString()), "Must be a dataset or view URI: " + uri); return Registration.lookupDatasetUri( URI.create(uri.getSchemeSpecificPart())).second(); }
private static boolean deleteWithTrash(URI uri, boolean useTrash){ Preconditions.checkArgument( URIBuilder.DATASET_SCHEME.equals(uri.getScheme()), "Not a dataset URI: " + uri); Pair<DatasetRepository, Map<String, String>> pair = Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart())); DatasetRepository repo = pair.first(); Map<String, String> uriOptions = pair.second(); return useTrash ? repo.moveToTrash(uriOptions.get(URIBuilder.NAMESPACE_OPTION), uriOptions.get(URIBuilder.DATASET_NAME_OPTION)) : repo.delete(uriOptions.get(URIBuilder.NAMESPACE_OPTION), uriOptions.get(URIBuilder.DATASET_NAME_OPTION)); }
@Override public TimeDomain load(Pair<PartitionStrategy, String> entry) { return new TimeDomain(entry.first(), entry.second()); } });
public Pair<String, Class<? extends Comparable>> merge( Pair<String, Class<? extends Comparable>> left, Pair<String, Class<? extends Comparable>> right) { String name = left.first(); if (name == null || name.isEmpty()) { name = right.first(); } if (left.second() == String.class) { return new Pair<String, Class<? extends Comparable>>(name, String.class); } else if (right.second() == String.class) { return new Pair<String, Class<? extends Comparable>>(name, String.class); } else if (left.second() == Long.class) { return new Pair<String, Class<? extends Comparable>>(name, Long.class); } else if (right.second() == Long.class) { return new Pair<String, Class<? extends Comparable>>(name, Long.class); } return new Pair<String, Class<? extends Comparable>>(name, Integer.class); }
/** * Constructs a builder based on the given dataset or view URI. * * @param uri a dataset or view URI * * @since 0.17.0 */ public URIBuilder(URI uri) { Preconditions.checkNotNull(uri, "URI cannot be null"); boolean isViewUri = VIEW_SCHEME.equals(uri.getScheme()); Preconditions.checkArgument(isViewUri || DATASET_SCHEME.equals(uri.getScheme()), "Not a dataset or view URI: " + uri); Pair<URIPattern, Map<String, String>> pair = Registration .lookupDatasetPattern(URI.create(uri.getRawSchemeSpecificPart())); this.pattern = pair.first(); this.isView = isViewUri; options.put(NAMESPACE_OPTION, NAMESPACE_DEFAULT); options.putAll(pair.second()); }
/** * Check whether a {@link Dataset} identified by the given URI exists. * <p> * URIs must begin with {@code dataset:}. The remainder of * the URI is implementation specific, depending on the dataset scheme. * * @param uri a {@code Dataset} URI * @return {@code true} if the dataset exists, {@code false} otherwise * @throws NullPointerException if {@code uri} is null * @throws IllegalArgumentException if {@code uri} is not a dataset URI */ public static boolean exists(URI uri) { Preconditions.checkArgument( URIBuilder.DATASET_SCHEME.equals(uri.getScheme()), "Not a dataset URI: " + uri); Pair<DatasetRepository, Map<String, String>> pair = Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart())); DatasetRepository repo = pair.first(); Map<String, String> uriOptions = pair.second(); return repo.exists( uriOptions.get(URIBuilder.NAMESPACE_OPTION), uriOptions.get(URIBuilder.DATASET_NAME_OPTION)); }
/** * Constructs a builder based on the given repository URI and * {@link Dataset#getName() dataset name}. * * @param repoUri the repository URI * @param namespace A namespace, or logical group name, for the dataset. * @param dataset the {@link Dataset} name * * @since 0.17.0 */ public URIBuilder(URI repoUri, String namespace, String dataset) { Preconditions.checkNotNull(repoUri, "Repository URI cannot be null"); Preconditions.checkNotNull(dataset, "Dataset name cannot be null"); Preconditions.checkArgument(REPO_SCHEME.equals(repoUri.getScheme()), "Repository URI must start with \"repo:\" but was:" + repoUri); Pair<URIPattern, Map<String, String>> pair = Registration .lookupPatternByRepoUri(URI.create(repoUri.getRawSchemeSpecificPart())); this.pattern = pair.first(); options.putAll(pair.second()); options.put(DATASET_NAME_OPTION, dataset); options.put(NAMESPACE_OPTION, namespace); }
public static PartitionStrategy strategy(FileSystem fs, Path location) throws IOException { if (!fs.exists(location)) { return null; } List<Pair<String, Class<? extends Comparable>>> pairs = visit( new GetPartitionInfo(), fs, location); if (pairs == null || pairs.isEmpty() || pairs.size() <= 1) { return null; } PartitionStrategy.Builder builder = new PartitionStrategy.Builder(); // skip the initial partition because it is the containing directory for (int i = 1; i < pairs.size(); i += 1) { Pair<String, Class<? extends Comparable>> pair = pairs.get(i); builder.provided( pair.first() == null ? "partition_" + i : pair.first(), ProvidedFieldPartitioner.valuesString(pair.second())); } return builder.build(); }
Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart())); DatasetRepository repo = pair.first(); Map<String, String> uriOptions = pair.second();
Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart())); DatasetRepository repo = pair.first(); Map<String, String> uriOptions = pair.second();
private static <E> View<E> loadOrCreateTaskAttemptView(TaskAttemptContext taskContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(taskContext); Map<String, String> uriOptions = Registration.lookupDatasetUri( URI.create(URI.create(conf.get(KITE_OUTPUT_URI)).getSchemeSpecificPart())).second(); Dataset<E> dataset = loadOrCreateTaskAttemptDataset(taskContext); if (dataset instanceof AbstractDataset) { DatasetDescriptor descriptor = dataset.getDescriptor(); Schema schema = descriptor.getSchema(); PartitionStrategy strategy = null; if (descriptor.isPartitioned()) { strategy = descriptor.getPartitionStrategy(); } Constraints constraints = Constraints.fromQueryMap( schema, strategy, uriOptions); return ((AbstractDataset<E>) dataset).filter(constraints); } else { return dataset; } }
@Override public MarkerRange.Builder update( MarkerRange.Builder current, Range range) { if (range.hasLowerBound()) { for (Pair<String, Range> pair : fields) { current.addToStart(pair.first(), pair.second().lowerEndpoint()); } } if (range.hasUpperBound()) { for (Pair<String, Range> pair : fields) { current.addToEnd(pair.first(), pair.second().upperEndpoint()); } } return current; } }
private static <E> View<E> loadOrCreateTaskAttemptView(TaskAttemptContext taskContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(taskContext); Map<String, String> uriOptions = Registration.lookupDatasetUri( URI.create(URI.create(conf.get(KITE_OUTPUT_URI)).getSchemeSpecificPart())).second(); Dataset<E> dataset = loadOrCreateTaskAttemptDataset(taskContext); if (dataset instanceof AbstractDataset) { DatasetDescriptor descriptor = dataset.getDescriptor(); Schema schema = descriptor.getSchema(); PartitionStrategy strategy = null; if (descriptor.isPartitioned()) { strategy = descriptor.getPartitionStrategy(); } Constraints constraints = Constraints.fromQueryMap( schema, strategy, uriOptions); return ((AbstractDataset<E>) dataset).filter(constraints); } else { return dataset; } }
Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart())); DatasetRepository repo = pair.first(); Map<String, String> uriOptions = pair.second();
} else if (compatible.size() == 1) { Pair<String, Range> pair = compatible.get(0); return addProjected((Predicate<?>) pair.second(), pair.first(), current); } else { return new RangeGroupIterator(constraint, compatible, current);