@Test(expected = IllegalArgumentException.class) public void testMergePathWithInvalidMoveWorkThrowsException() { final Path condInputPath = new Path("s3a://bucket/scratch/-ext-10000"); final MoveWork mockWork = mock(MoveWork.class); final LineageState lineageState = new LineageState(); when(mockWork.getLoadMultiFilesWork()).thenReturn(new LoadMultiFilesDesc()); GenMapRedUtils.mergeMovePaths(condInputPath, mockWork, lineageState); }
public SessionState(HiveConf conf, String userName) { this.sessionConf = conf; this.userName = userName; this.registry = new Registry(false); if (LOG.isDebugEnabled()) { LOG.debug("SessionState user: " + userName); } isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT); ls = new LineageState(); resourceMaps = new ResourceMaps(); // Must be deterministic order map for consistent q-test output across Java versions overriddenConfigurations = new LinkedHashMap<String, String>(); overriddenConfigurations.putAll(HiveConf.getConfSystemProperties()); // if there isn't already a session name, go ahead and create it. if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) { conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId()); } // Using system classloader as the parent. Using thread context // classloader as parent can pollute the session. See HIVE-11878 parentLoader = SessionState.class.getClassLoader(); // Make sure that each session has its own UDFClassloader. For details see {@link UDFClassLoader} final ClassLoader currentLoader = Utilities.createUDFClassLoader((URLClassLoader) parentLoader, new String[]{}); this.sessionConf.setClassLoader(currentLoader); resourceDownloader = new ResourceDownloader(conf, HiveConf.getVar(conf, ConfVars.DOWNLOADED_RESOURCES_DIR)); }
@Test public void testConditionalMoveOnHdfsIsNotOptimized() throws SemanticException { hiveConf.set(HiveConf.ConfVars.HIVE_BLOBSTORE_OPTIMIZATIONS_ENABLED.varname, "true"); Path sinkDirName = new Path("hdfs://bucket/scratch/-ext-10002"); FileSinkOperator fileSinkOperator = createFileSinkOperator(sinkDirName); Path finalDirName = new Path("hdfs://bucket/scratch/-ext-10000"); Path tableLocation = new Path("hdfs://bucket/warehouse/table"); Task<MoveWork> moveTask = createMoveTask(finalDirName, tableLocation); List<Task<MoveWork>> moveTaskList = Collections.singletonList(moveTask); GenMapRedUtils.createMRWorkForMergingFiles(fileSinkOperator, finalDirName, null, moveTaskList, hiveConf, dummyMRTask, new LineageState()); ConditionalTask conditionalTask = (ConditionalTask)dummyMRTask.getChildTasks().get(0); Task<? extends Serializable> moveOnlyTask = conditionalTask.getListTasks().get(0); Task<? extends Serializable> mergeOnlyTask = conditionalTask.getListTasks().get(1); Task<? extends Serializable> mergeAndMoveTask = conditionalTask.getListTasks().get(2); // Verify moveOnlyTask is NOT optimized assertEquals(1, moveOnlyTask.getChildTasks().size()); verifyMoveTask(moveOnlyTask, sinkDirName, finalDirName); verifyMoveTask(moveOnlyTask.getChildTasks().get(0), finalDirName, tableLocation); // Verify mergeOnlyTask is NOT optimized assertEquals(1, mergeOnlyTask.getChildTasks().size()); verifyMoveTask(mergeOnlyTask.getChildTasks().get(0), finalDirName, tableLocation); // Verify mergeAndMoveTask is NOT optimized assertEquals(1, mergeAndMoveTask.getChildTasks().size()); assertEquals(1, mergeAndMoveTask.getChildTasks().get(0).getChildTasks().size()); verifyMoveTask(mergeAndMoveTask.getChildTasks().get(0), sinkDirName, finalDirName); verifyMoveTask(mergeAndMoveTask.getChildTasks().get(0).getChildTasks().get(0), finalDirName, tableLocation); }
@Test public void testConditionalMoveTaskIsNotOptimized() throws SemanticException { hiveConf.set(HiveConf.ConfVars.HIVE_BLOBSTORE_OPTIMIZATIONS_ENABLED.varname, "false"); Path sinkDirName = new Path("s3a://bucket/scratch/-ext-10002"); FileSinkOperator fileSinkOperator = createFileSinkOperator(sinkDirName); Path finalDirName = new Path("s3a://bucket/scratch/-ext-10000"); Path tableLocation = new Path("s3a://bucket/warehouse/table"); Task<MoveWork> moveTask = createMoveTask(finalDirName, tableLocation); List<Task<MoveWork>> moveTaskList = Collections.singletonList(moveTask); GenMapRedUtils.createMRWorkForMergingFiles(fileSinkOperator, finalDirName, null, moveTaskList, hiveConf, dummyMRTask, new LineageState()); ConditionalTask conditionalTask = (ConditionalTask)dummyMRTask.getChildTasks().get(0); Task<? extends Serializable> moveOnlyTask = conditionalTask.getListTasks().get(0); Task<? extends Serializable> mergeOnlyTask = conditionalTask.getListTasks().get(1); Task<? extends Serializable> mergeAndMoveTask = conditionalTask.getListTasks().get(2); // Verify moveOnlyTask is NOT optimized assertEquals(1, moveOnlyTask.getChildTasks().size()); verifyMoveTask(moveOnlyTask, sinkDirName, finalDirName); verifyMoveTask(moveOnlyTask.getChildTasks().get(0), finalDirName, tableLocation); // Verify mergeOnlyTask is NOT optimized assertEquals(1, mergeOnlyTask.getChildTasks().size()); verifyMoveTask(mergeOnlyTask.getChildTasks().get(0), finalDirName, tableLocation); // Verify mergeAndMoveTask is NOT optimized assertEquals(1, mergeAndMoveTask.getChildTasks().size()); assertEquals(1, mergeAndMoveTask.getChildTasks().get(0).getChildTasks().size()); verifyMoveTask(mergeAndMoveTask.getChildTasks().get(0), sinkDirName, finalDirName); verifyMoveTask(mergeAndMoveTask.getChildTasks().get(0).getChildTasks().get(0), finalDirName, tableLocation); }
moveTaskList, hiveConf, dummyMRTask, new LineageState()); ConditionalTask conditionalTask = (ConditionalTask)dummyMRTask.getChildTasks().get(0); Task<? extends Serializable> moveOnlyTask = conditionalTask.getListTasks().get(0);
@Test public void testMergePathValidMoveWorkReturnsNewMoveWork() { final Path condInputPath = new Path("s3a://bucket/scratch/-ext-10000"); final Path condOutputPath = new Path("s3a://bucket/scratch/-ext-10002"); final Path targetMoveWorkPath = new Path("s3a://bucket/scratch/-ext-10003"); final MoveWork mockWork = mock(MoveWork.class); final LineageState lineageState = new LineageState(); MoveWork newWork; // test using loadFileWork when(mockWork.getLoadFileWork()).thenReturn(new LoadFileDesc( condOutputPath, targetMoveWorkPath, false, "", "", false)); newWork = GenMapRedUtils.mergeMovePaths(condInputPath, mockWork, lineageState); assertNotNull(newWork); assertNotEquals(newWork, mockWork); assertEquals(condInputPath, newWork.getLoadFileWork().getSourcePath()); assertEquals(targetMoveWorkPath, newWork.getLoadFileWork().getTargetDir()); // test using loadTableWork TableDesc tableDesc = new TableDesc(); reset(mockWork); when(mockWork.getLoadTableWork()).thenReturn(new LoadTableDesc( condOutputPath, tableDesc, null)); newWork = GenMapRedUtils.mergeMovePaths(condInputPath, mockWork, lineageState); assertNotNull(newWork); assertNotEquals(newWork, mockWork); assertEquals(condInputPath, newWork.getLoadTableWork().getSourcePath()); assertTrue(newWork.getLoadTableWork().getTable().equals(tableDesc)); }
public SessionState(HiveConf conf) { this.conf = conf; isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT); ls = new LineageState(); }
public SessionState(HiveConf conf, String userName) { this.conf = conf; this.userName = userName; if (LOG.isDebugEnabled()) { LOG.debug("SessionState user: " + userName); } isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT); ls = new LineageState(); resourceMaps = new ResourceMaps(); dependencyResolver = new DependencyResolver(); // Must be deterministic order map for consistent q-test output across Java versions overriddenConfigurations = new LinkedHashMap<String, String>(); overriddenConfigurations.putAll(HiveConf.getConfSystemProperties()); // if there isn't already a session name, go ahead and create it. if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) { conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId()); } parentLoader = JavaUtils.getClassLoader(); }