private static SqoopOptions createSqoopOptionsWithLogicalTypesEnabled() { SqoopOptions sqoopOptions = new SqoopOptions(); sqoopOptions.getConf().setBoolean(ConfigurationConstants.PROP_ENABLE_PARQUET_LOGICAL_TYPE_DECIMAL, true); return sqoopOptions; }
private SqoopOptions stubOptions(SupportedManagers supportedManagers, boolean isDirect) { SqoopOptions options = mock(SqoopOptions.class); when(options.getConnectString()).thenReturn(supportedManagers.getSchemePrefix() + "//localhost"); when(options.isDirect()).thenReturn(isDirect); when(options.getConf()).thenReturn(mock(Configuration.class)); return options; }
private SqoopOptions stubOptions(SupportedManagers supportedManagers, boolean isDirect) { SqoopOptions options = mock(SqoopOptions.class); when(options.getConnectString()).thenReturn(supportedManagers.getSchemePrefix() + "//localhost"); when(options.isDirect()).thenReturn(isDirect); when(options.getConf()).thenReturn(mock(Configuration.class)); return options; } }
@Before public void before() { connectionFactoryInitializer = new HiveServer2ConnectionFactoryInitializer(); sqoopOptions = mock(SqoopOptions.class); configuration = mock(Configuration.class); softly = new SoftAssertions(); when(sqoopOptions.getHs2User()).thenReturn(TEST_HS2_USER); when(sqoopOptions.getConf()).thenReturn(configuration); }
@Test public void valueOfOracleEscapingDisabledInHadoopConfigurationInstanceOfSqoopOptionsShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToAnyNonBooleanValue() { System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "falsetrue"); SqoopOptions opts = new SqoopOptions(); assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), is(equalTo(false))); }
@Test public void valueOfOracleEscapingDisabledInHadoopConfigurationInstanceOfSqoopOptionsShouldBeTrueIfTheValueOfTheRelatedEnvironmentVariableIsSetToTrue() { System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "true"); SqoopOptions opts = new SqoopOptions(); assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), is(equalTo(true))); }
@Test public void valueOfOracleEscapingDisabledInHadoopConfigurationInstanceOfSqoopOptionsShouldBeFalseIfTheValueOfTheRelatedEnvironmentVariableIsSetToFalse() { System.setProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED, "false"); SqoopOptions opts = new SqoopOptions(); assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), is(equalTo(false))); }
@Before public void setUp() { // set Sqoop command line arguments options = new SqoopOptions(); conf = options.getConf(); jobBase = spy(new JobBase(options)); }
@Test public void testGenerateExternalTableLoadStatement() throws IOException, SQLException { // need to set this as the other unit test functions may override it for their own test. options.setHiveExternalTableDir(hdfsTableDir); tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, options.getConf(), false); String stmt = tableDefWriter.getLoadDataStmt(); Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); LOG.debug("External table dir: "+options.getHiveExternalTableDir()); assert (isHiveExternalTableSet && stmt.contains("LOAD DATA INPATH ") && stmt.contains(testTargetDir)); } }
@Test public void hadoopConfigurationInstanceOfSqoopOptionsShouldContainFalseForOracleEscapingDisabledAsTheValueDirectlyHasBeenSetToSqoopOptions() { System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); SqoopOptions opts = new SqoopOptions(); opts.setOracleEscapingDisabled(false); assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), is(equalTo(false))); }
@Test public void hadoopConfigurationInstanceOfSqoopOptionsShouldContainTrueForOracleEscapingDisabledAsTheValueDirectlyHasBeenSetToSqoopOptions() { System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); SqoopOptions opts = new SqoopOptions(); opts.setOracleEscapingDisabled(true); assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), is(equalTo(true))); }
@Test public void testGenerateExternalTableStatement() throws IOException, SQLException { // need to set this as the other unit test functions may override it for their own test. options.setHiveExternalTableDir(hdfsTableDir); tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, options.getConf(), false); String stmt = tableDefWriter.getCreateTableStmt(); Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); LOG.debug("External table dir: "+options.getHiveExternalTableDir()); assert (isHiveExternalTableSet && stmt.contains("CREATE EXTERNAL TABLE ") && stmt.contains("LOCATION '" + hdfsTableDir)); }
@Test public void testGenerateTableIfExistsStatement() throws IOException, SQLException { // need to set this as the other unit test functions may override it for their own test. options.setHiveExternalTableDir(null); tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, options.getConf(), false); String stmt = tableDefWriter.getCreateTableStmt(); Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); LOG.debug("External table dir: "+options.getHiveExternalTableDir()); assert (!isHiveExternalTableSet && stmt.contains("CREATE TABLE IF NOT EXISTS")); }
@Test public void testGetCreateTableStmtWithAvroSchema() throws Exception { options.setFileLayout(ParquetFile); options.getConf().set(SQOOP_PARQUET_AVRO_SCHEMA_KEY, TEST_AVRO_SCHEMA); assertEquals(EXPECTED_CREATE_PARQUET_TABLE_STMNT, writer.getCreateTableStmt()); }
@Test public void hadoopConfigurationInstanceOfSqoopOptionsShouldContainTheSameValueForOracleEscapingDisabledAsSqoopOptionsProperty() { SqoopOptions opts = new SqoopOptions(); assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), is(equalTo(opts.isOracleEscapingDisabled()))); }
public void runImport(SqoopOptions options, List<String> args) { try { Sqoop importer = new Sqoop(new ImportTool(), options.getConf(), options); int ret = Sqoop.runSqoop(importer, args.toArray(new String[0])); assertEquals("Failure during job", 0, ret); } catch (Exception e) { LOG.error("Got exception running Sqoop: " + StringUtils.stringifyException(e)); throw new RuntimeException(e); } }
@Test public void valueOfOracleEscapingDisabledShouldBeEqualToNullIfASqoopOptionsInstanceWasLoadedWhichDidntContainASavedValueForIt() { System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); SqoopOptions opts = new SqoopOptions(); Properties out = opts.writeProperties(); opts = new SqoopOptions(); opts.loadProperties(out); assertThat(opts.isOracleEscapingDisabled(), is(equalTo(true))); assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), is(equalTo(true))); }
@Test public void valueOfOracleEscapingDisabledShouldBeAbleToSavedAndLoadedBackWithTheSameValue() { System.clearProperty(SqoopOptions.ORACLE_ESCAPING_DISABLED); SqoopOptions opts = new SqoopOptions(); opts.setOracleEscapingDisabled(false); Properties out = opts.writeProperties(); opts = new SqoopOptions(); opts.loadProperties(out); assertThat(opts.isOracleEscapingDisabled(), is(equalTo(false))); assertThat(OracleUtils.isOracleEscapingDisabled(opts.getConf()), is(equalTo(false))); }
@Test public void testApplyCommonOptionsSetsParquetJobConfigurationImplementationFromConfiguration() throws Exception { ParquetJobConfiguratorImplementation expectedValue = HADOOP; testSqoopOptions.getConf().set("parquetjob.configurator.implementation", expectedValue.toString()); testBaseSqoopTool.applyCommonOptions(mockCommandLine, testSqoopOptions); assertEquals(expectedValue, testSqoopOptions.getParquetConfiguratorImplementation()); }
@Test public void testLastModifiedImportWithExistingOutputDirectoryFails() throws Exception { final String TABLE_NAME = "failWithExistingOutputDirectory"; createDir(TABLE_NAME); Timestamp thePast = new Timestamp(System.currentTimeMillis() - 100); createTimestampTable(TABLE_NAME, 10, thePast); List<String> args = getArgListForTable(TABLE_NAME, true, false); SqoopOptions options = new SqoopOptions(newConf()); options.setThrowOnError(true); thrown.expectMessage("--merge-key or --append is required when using --incremental lastmodified and the output directory exists."); Sqoop sqoop = new Sqoop(new ImportTool(), options.getConf(), options); ToolRunner.run(sqoop.getConf(), sqoop, args.toArray(new String[0])); }