/** working with target-dir. */ @Test public void testAppendToTargetDir() throws IOException { ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf()); String targetDir = getWarehouseDir() + "/tempTargetDir"; args.add("--target-dir"); args.add(targetDir); // there's no need for a new param // in diff. w/--warehouse-dir there will no be $tablename dir Path output = new Path(targetDir); runAppendTest(args, output); }
FileSystem fs = FileSystem.get(getConf()); if (fs.exists(outputPath)) { fs.delete(outputPath, true); runUncleanImport(argv); FileStatus[] fileStatuses = listFiles(fs, outputPath, prefix); Arrays.sort(fileStatuses, new StatusPathComparator()); int previousFileCount = fileStatuses.length; String previousImage = getFileCreationTimeImage(fs, outputPath, previousFileCount); int lastPartition = getFilePartition(lastFile); runUncleanImport(argv); fileStatuses = listFiles(fs, outputPath, prefix); Arrays.sort(fileStatuses, new StatusPathComparator()); int currentFileCount = fileStatuses.length; String currentImage = getFileCreationTimeImage(fs, outputPath, previousFileCount); assertEquals("Previous files to appending operation were modified", int newPartition = getFilePartition(newFile); assertTrue("New partition file isn't correlative", lastPartition + 1 == newPartition);
String[] colNames, Configuration conf) { if (null == colNames) { colNames = getColNames(); args.add("SELECT * FROM " + getTableName() + " WHERE $CONDITIONS"); } else { args.add("--table"); args.add(getTableName()); args.add(splitByCol); args.add("--connect"); args.add(getConnectString()); args.add("--as-sequencefile"); args.add("--num-mappers"); args.add("1"); args.addAll(getExtraArgs(conf));
/** the same than ImportJobTestCase but without removing tabledir. */ protected void runUncleanImport(String[] argv) throws IOException { // run the tool through the normal entry-point. int ret; try { Configuration conf = getConf(); SqoopOptions opts = getSqoopOptions(conf); Sqoop sqoop = new Sqoop(new ImportTool(), conf, opts); ret = Sqoop.runSqoop(sqoop, argv); } catch (Exception e) { LOG.error("Got exception running Sqoop: " + e.toString()); e.printStackTrace(); ret = 1; } // expect a successful return. if (0 != ret) { throw new IOException("Failure during job; return status " + ret); } }
/** independent to target-dir. */ @Test public void testAppend() throws IOException { ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf()); args.add("--warehouse-dir"); args.add(getWarehouseDir()); Path output = new Path(getWarehouseDir(), HsqldbTestServer.getTableName()); runAppendTest(args, output); }
/** * Query based import should also work in append mode. * * @throws IOException */ @Test public void testAppendWithQuery() throws IOException { ArrayList args = getOutputlessArgv(false, true, HsqldbTestServer.getFieldNames(), getConf()); String targetDir = getWarehouseDir() + "/tempTargetDir"; args.add("--target-dir"); args.add(targetDir); Path output = new Path(targetDir); runAppendTest(args, output); }
/** * Test that when we pass in -Dmapreduce.output.basename=prefix, file should also got appended * * @throws IOException */ @Test public void testAppendWithMapreduceOutputBasename() throws IOException { String prefix = "prefix-test"; ArrayList<String> args = new ArrayList<>(); args.add("-D"); args.add(MAPREDUCE_OUTPUT_BASENAME_PROPERTY + "=" + prefix); args.addAll(getOutputlessArgv(false, true, HsqldbTestServer.getFieldNames(), getConf())); String targetDir = getWarehouseDir() + "/tempTargetDirOutputBaseNameTest"; args.add("--target-dir"); args.add(targetDir); Path output = new Path(targetDir); runAppendTest(args, output, prefix); } }