/** * Finds the deepest ancestor of input that is not a glob. */ public static Path deepestNonGlobPath(Path input) { Path commonRoot = input; while (commonRoot != null && isGlob(commonRoot)) { commonRoot = commonRoot.getParent(); } return commonRoot; }
/** * A helper to determine if the path description of this {@link DatasetDescriptor} is a superset of paths * accepted by the other {@link DatasetDescriptor}. If the path description of the other {@link DatasetDescriptor} * is a glob pattern, we return false. * * @param otherPath a glob pattern that describes a set of paths. * @return true if the glob pattern described by the otherPath matches the path in this {@link DatasetDescriptor}. */ private boolean isPathContaining(String otherPath) { if (otherPath == null) { return false; } if (DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_ANY.equals(this.getPath())) { return true; } if (PathUtils.isGlob(new Path(otherPath))) { return false; } GlobPattern globPattern = new GlobPattern(this.getPath()); return globPattern.matches(otherPath); }
if (PathUtils.isGlob(input.getPartition().getDataLocation())) { log.error(String.format("%s cannot be applied to globbed location %s. Will not skip.", this.getClass().getSimpleName(), input.getPartition().getDataLocation()));
if (PathUtils.isGlob(input.getPartition().getDataLocation())) { log.error(String.format("%s cannot be applied to globbed location %s. Will not skip.", this.getClass().getSimpleName(), input.getPartition().getDataLocation()));
public HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties, Config datasetConfig) { this.fs = fs; this.clientPool = clientPool; this.table = table; this.properties = properties; this.tableRootPath = PathUtils.isGlob(this.table.getDataLocation()) ? Optional.<Path> absent() : Optional.fromNullable(this.table.getDataLocation()); this.tableIdentifier = this.table.getDbName() + "." + this.table.getTableName(); this.datasetNamePattern = Optional.fromNullable(ConfigUtils.getString(datasetConfig, DATASET_NAME_PATTERN_KEY, null)); this.dbAndTable = new DbAndTable(table.getDbName(), table.getTableName()); if (this.datasetNamePattern.isPresent()) { this.logicalDbAndTable = parseLogicalDbAndTable(this.datasetNamePattern.get(), this.dbAndTable, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN); } else { this.logicalDbAndTable = this.dbAndTable; } this.datasetConfig = resolveConfig(datasetConfig, dbAndTable, logicalDbAndTable); this.metricContext = Instrumented.getMetricContext(new State(properties), HiveDataset.class, Lists.<Tag<?>> newArrayList(new Tag<>(DATABASE, table.getDbName()), new Tag<>(TABLE, table.getTableName()))); }
/** * Finds the deepest ancestor of input that is not a glob. */ public static Path deepestNonGlobPath(Path input) { Path commonRoot = input; while (commonRoot != null && isGlob(commonRoot)) { commonRoot = commonRoot.getParent(); } return commonRoot; }
/** * A helper to determine if the path description of this {@link DatasetDescriptor} is a superset of paths * accepted by the other {@link DatasetDescriptor}. If the path description of the other {@link DatasetDescriptor} * is a glob pattern, we return false. * * @param otherPath a glob pattern that describes a set of paths. * @return true if the glob pattern described by the otherPath matches the path in this {@link DatasetDescriptor}. */ private boolean isPathContaining(String otherPath) { if (otherPath == null) { return false; } if (DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_ANY.equals(this.getPath())) { return true; } if (PathUtils.isGlob(new Path(otherPath))) { return false; } GlobPattern globPattern = new GlobPattern(this.getPath()); return globPattern.matches(otherPath); }
if (PathUtils.isGlob(input.getPartition().getDataLocation())) { log.error(String.format("%s cannot be applied to globbed location %s. Will not skip.", this.getClass().getSimpleName(), input.getPartition().getDataLocation()));
if (PathUtils.isGlob(input.getPartition().getDataLocation())) { log.error(String.format("%s cannot be applied to globbed location %s. Will not skip.", this.getClass().getSimpleName(), input.getPartition().getDataLocation()));
public HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties, Config datasetConfig) { this.fs = fs; this.clientPool = clientPool; this.table = table; this.properties = properties; this.tableRootPath = PathUtils.isGlob(this.table.getDataLocation()) ? Optional.<Path> absent() : Optional.fromNullable(this.table.getDataLocation()); this.tableIdentifier = this.table.getDbName() + "." + this.table.getTableName(); this.datasetNamePattern = Optional.fromNullable(ConfigUtils.getString(datasetConfig, DATASET_NAME_PATTERN_KEY, null)); this.dbAndTable = new DbAndTable(table.getDbName(), table.getTableName()); if (this.datasetNamePattern.isPresent()) { this.logicalDbAndTable = parseLogicalDbAndTable(this.datasetNamePattern.get(), this.dbAndTable, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN); } else { this.logicalDbAndTable = this.dbAndTable; } this.datasetConfig = resolveConfig(datasetConfig, dbAndTable, logicalDbAndTable); this.metricContext = Instrumented.getMetricContext(new State(properties), HiveDataset.class, Lists.<Tag<?>> newArrayList(new Tag<>(DATABASE, table.getDbName()), new Tag<>(TABLE, table.getTableName()))); }