/** * Read user's output description from Configuration * * @return HiveOutputDescription */ public HiveOutputDescription readOutputDescription() { String value = conf.get(getOutputDescriptionKey()); HiveOutputDescription hod = null; if (value != null) { hod = new HiveOutputDescription(); Writables.readFieldsFromEncodedStr(value, hod); } return hod; }
/** * Setup the job * * @param conf Configuration * @throws IOException */ public static void setupJob(Configuration conf) throws IOException { HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.setTableDesc(getHiveTableName()); Map<String, String> partitionValues = ImmutableMap.of("ds", "2013-04-01"); outputDesc.setPartitionValues(partitionValues); LOG.info("Writing to {}", outputDesc); try { HiveApiOutputFormat.initProfile(conf, outputDesc, SampleOutputFormat.SAMPLE_PROFILE_ID); } catch (IOException e) { LOG.error("Failed to initialize profile {}", outputDesc); throw e; } }
@Test(expectedExceptions = IOException.class) public void testTableDoesntExist() throws Exception { Configuration conf = new Configuration(); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("doesnt-exist"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); outputFormat.checkOutputSpecs(jobContext); fail(); } }
@Test(expectedExceptions = IOException.class) public void testTableDoesntExist() throws Exception { Configuration conf = new Configuration(); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("doesnt-exist"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); outputFormat.checkOutputSpecs(jobContext); fail(); } }
@Test(expectedExceptions = IOException.class) public void testExceptionAfterTooManyRetriesWhenTableDoesNotExist() throws Exception { HiveMetastores.setTestClient(new FaultyThriftHiveMetastore( BackoffRetryTask.NUM_TRIES.getDefaultValue())); Configuration conf = new Configuration(); conf.setLong(BackoffRetryTask.INITIAL_RETRY_DELAY_MSEC.getKey(), 100); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("doesnt-exist"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); outputFormat.checkOutputSpecs(jobContext); fail(); }
@Test public void testOutput() throws Exception { hiveServer.createTable("CREATE TABLE " + hiveTableDesc.getTableName() + " (i1 INT, d1 DOUBLE) " + " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'"); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.setTableDesc(hiveTableDesc); HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(), null, hiveTableDesc); writeData(outputDesc, schema); HiveInputDescription inputDesc = new HiveInputDescription(); inputDesc.setTableDesc(hiveTableDesc); verifyData(inputDesc); }
@Test public void testOutput() throws Exception { hiveServer.createTable("CREATE TABLE " + hiveTableDesc.getTableName() + " (i1 INT, d1 DOUBLE) " + " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'"); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.setTableDesc(hiveTableDesc); HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(), null, hiveTableDesc); writeData(outputDesc, schema); HiveInputDescription inputDesc = new HiveInputDescription(); inputDesc.setTableDesc(hiveTableDesc); verifyData(inputDesc); }
@Test(expectedExceptions = IOException.class) public void testExceptionAfterTooManyRetriesWhenTableDoesNotExist() throws Exception { HiveMetastores.setTestClient(new FaultyThriftHiveMetastore( BackoffRetryTask.NUM_TRIES.getDefaultValue())); Configuration conf = new Configuration(); conf.setLong(BackoffRetryTask.INITIAL_RETRY_DELAY_MSEC.getKey(), 100); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("doesnt-exist"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); outputFormat.checkOutputSpecs(jobContext); fail(); }
@Test public void testRecoveredFromFailuresAfterRetries() throws Exception { FaultyThriftHiveMetastore metastore = new FaultyThriftHiveMetastore( BackoffRetryTask.NUM_TRIES.getDefaultValue() - 1); Configuration conf = new Configuration(); conf.setLong(BackoffRetryTask.INITIAL_RETRY_DELAY_MSEC.getKey(), 100); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("foo"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); HiveMetastores.setTestClient(metastore); outputFormat.checkOutputSpecs(jobContext); assertEquals( BackoffRetryTask.NUM_TRIES.getDefaultValue(), metastore.getNumCalls()); } }
@Test public void testRecoveredFromFailuresAfterRetries() throws Exception { FaultyThriftHiveMetastore metastore = new FaultyThriftHiveMetastore( BackoffRetryTask.NUM_TRIES.getDefaultValue() - 1); Configuration conf = new Configuration(); conf.setLong(BackoffRetryTask.INITIAL_RETRY_DELAY_MSEC.getKey(), 100); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("foo"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); HiveMetastores.setTestClient(metastore); outputFormat.checkOutputSpecs(jobContext); assertEquals( BackoffRetryTask.NUM_TRIES.getDefaultValue(), metastore.getNumCalls()); } }
@Test public void testOutputWithPartitions() throws Exception { hiveServer.createTable("CREATE TABLE " + hiveTableDesc.getTableName() + " (i1 INT, d1 DOUBLE) " + " PARTITIONED BY (ds STRING) " + " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'"); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.putPartitionValue("ds", "foobar"); outputDesc.setTableDesc(hiveTableDesc); HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(), null, hiveTableDesc); writeData(outputDesc, schema); HiveInputDescription inputDesc = new HiveInputDescription(); inputDesc.setPartitionFilter("ds='foobar'"); inputDesc.setTableDesc(hiveTableDesc); verifyData(inputDesc); }
@Test public void testOutputWithPartitions() throws Exception { hiveServer.createTable("CREATE TABLE " + hiveTableDesc.getTableName() + " (i1 INT, d1 DOUBLE) " + " PARTITIONED BY (ds STRING) " + " ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'"); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.putPartitionValue("ds", "foobar"); outputDesc.setTableDesc(hiveTableDesc); HiveTableSchema schema = HiveTableSchemas.lookup(hiveServer.getClient(), null, hiveTableDesc); writeData(outputDesc, schema); HiveInputDescription inputDesc = new HiveInputDescription(); inputDesc.setPartitionFilter("ds='foobar'"); inputDesc.setTableDesc(hiveTableDesc); verifyData(inputDesc); }
context.conf.setInt("mapred.task.partition", 1); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setDatabaseName(args.table.database); outputDesc.getTableDesc().setTableName(args.table.table);
createTestTable(); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.setTableDesc(hiveTableDesc);
createTestTable(); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.setTableDesc(hiveTableDesc);
createTestTable(); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.setTableDesc(hiveTableDesc);
createTestTable(); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.setTableDesc(hiveTableDesc);