FileGenerator(Context context, FileSystem fs, Path dir, boolean useFileIds, UserGroupInformation ugi) { this(context, fs, dir, Ref.from(useFileIds), ugi); }
public static <T> Ref<T> from(T t) { return new Ref<T>(t); } }
@VisibleForTesting FileGenerator(Context context, FileSystem fs, Path dir, boolean useFileIds, UserGroupInformation ugi) { this(context, fs, dir, Ref.from(useFileIds), ugi); }
private void ensureInfo() { if (info != null) return; Ref<Integer> endpointVersion = new Ref<>(-1); info = node.getAmPluginInfo(endpointVersion); context.setNodeInfo(info, endpointVersion.value); // Give the caller context for future errors. if (info == null) { // RequestManager will catch this and handle like any other error. throw new RuntimeException("No AM plugin info for " + node); } } }
dir, conf, validWriteIdList, Ref.from(false), true, null);
Ref<Integer> endpointVersion = new Ref<>(-1); AmPluginInfo info = session.getAmPluginInfo(endpointVersion); if (info != null && endpointVersion.value > failedEndpointVersion) {
Ref<Boolean> isPastEndTime = Ref.from(false); while (iter.hasNext()) { FileCache<T> fc = iter.next().getValue();
@VisibleForTesting protected void schedulePendingTasks() throws InterruptedException { Ref<TaskInfo> downgradedTask = new Ref<>(null); writeLock.lock(); try {
@Test public void testSubmit() throws Exception { DAG dag = DAG.create("test"); task.submit(conf, dag, Ref.from(sessionState)); // validate close/reopen verify(sessionState, times(1)).reopen(); verify(session, times(2)).submitDAG(any(DAG.class)); }
public static <T> Ref<T> from(T t) { return new Ref<T>(t); } }
public static Directory getAcidState(Path directory, Configuration conf, ValidTxnList txnList, Ref<Boolean> useFileIds, boolean ignoreEmptyFiles ) throws IOException { FileSystem fs = directory.getFileSystem(conf); // The following 'deltas' includes all kinds of delta files including insert & delete deltas. final List<ParsedDelta> deltas = new ArrayList<ParsedDelta>(); List<ParsedDelta> working = new ArrayList<ParsedDelta>(); List<FileStatus> originalDirectories = new ArrayList<FileStatus>(); final List<FileStatus> obsolete = new ArrayList<FileStatus>(); List<HdfsFileStatusWithId> childrenWithId = null; Boolean val = useFileIds.value; if (val == null || val) { try { childrenWithId = SHIMS.listLocatedHdfsStatus(fs, directory, hiddenFileFilter); if (val == null) { useFileIds.value = true; } } catch (Throwable t) { LOG.error("Failed to get files with ID; using regular API: " + t.getMessage()); if (val == null && t instanceof UnsupportedOperationException) { useFileIds.value = false; } } } TxnBase bestBase = new TxnBase(); final List<HdfsFileStatusWithId> original = new ArrayList<>(); if (childrenWithId != null) {
@VisibleForTesting protected void schedulePendingTasks() throws InterruptedException { Ref<TaskInfo> downgradedTask = new Ref<>(null); writeLock.lock(); try {
Ref<Boolean> useFileIds = Ref.from(useFileIdsConfig ? null : false); boolean allowSyntheticFileIds = useFileIdsConfig && HiveConf.getBoolVar( conf, ConfVars.HIVE_ORC_ALLOW_SYNTHETIC_FILE_ID_IN_SPLITS);
public static Directory getAcidState(Path directory, Configuration conf, ValidWriteIdList writeIdList, Ref<Boolean> useFileIds, boolean ignoreEmptyFiles, Map<String, String> tblproperties) throws IOException { ValidTxnList validTxnList = null; String s = conf.get(ValidTxnList.VALID_TXNS_KEY); if(!Strings.isNullOrEmpty(s)) { /** * getAcidState() is sometimes called on non-transactional tables, e.g. * OrcInputFileFormat.FileGenerator.callInternal(). e.g. orc_merge3.q In that case * writeIdList is bogus - doesn't even have a table name. * see https://issues.apache.org/jira/browse/HIVE-20856. * * For now, assert that ValidTxnList.VALID_TXNS_KEY is set only if this is really a read * of a transactional table. * see {@link #getChildState(FileStatus, HdfsFileStatusWithId, ValidWriteIdList, List, List, List, List, TxnBase, boolean, List, Map, FileSystem, ValidTxnList)} */ validTxnList = new ValidReadTxnList(); validTxnList.readFromString(s); } FileSystem fs = directory.getFileSystem(conf); // The following 'deltas' includes all kinds of delta files including insert & delete deltas. final List<ParsedDelta> deltas = new ArrayList<ParsedDelta>(); List<ParsedDelta> working = new ArrayList<ParsedDelta>(); List<FileStatus> originalDirectories = new ArrayList<FileStatus>(); final List<FileStatus> obsolete = new ArrayList<FileStatus>(); final List<FileStatus> abortedDirectories = new ArrayList<>();
Ref<Boolean> useFileIds = Ref.from(useFileIdsConfig ? null : false); boolean allowSyntheticFileIds = useFileIdsConfig && HiveConf.getBoolVar( conf, ConfVars.HIVE_ORC_ALLOW_SYNTHETIC_FILE_ID_IN_SPLITS);
+ t.getDbName() + "." + t.getTableName()); AcidUtils.Directory dir = AcidUtils.getAcidState(new Path(sd.getLocation()), conf, writeIds, Ref.from(false), false, t.getParameters()); removeFilesForMmTable(conf, dir);
if (!processOneFileSplit(split, startTime, Ref.from(0), null)) return null;
long uncachedPrefixEnd = slices.get(0).getKnownTornStart(), uncachedSuffixStart = slices.get(slices.size() - 1).getLastEnd(); Ref<Integer> stripeIx = Ref.from(0); if (uncachedPrefixEnd > split.getStart()) {
boolean cleanContext = false; Context ctx = null; Ref<TezSessionState> sessionRef = Ref.from(null);
Ref<Boolean> isPastEndTime = Ref.from(false); while (iter.hasNext()) { FileCache<T> fc = iter.next().getValue();