@Override public RecordWriterImpl getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { HadoopUtils.setWorkOutputDir(taskAttemptContext); Configuration conf = taskAttemptContext.getConfiguration(); OutputConf outputConf = new OutputConf(conf, myProfileId); OutputInfo oti = outputConf.readOutputTableInfo(); HiveUtils.setRCileNumColumns(conf, oti.getColumnInfo().size()); HadoopUtils.setOutputKeyWritableClass(conf, NullWritable.class); Serializer serializer = oti.createSerializer(conf); HadoopUtils.setOutputValueWritableClass(conf, serializer.getSerializedClass()); org.apache.hadoop.mapred.OutputFormat baseOutputFormat = ReflectionUtils.newInstance(oti.getOutputFormatClass(), conf); // CHECKSTYLE: stop LineLength org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter = getBaseRecordWriter(taskAttemptContext, baseOutputFormat); // CHECKSTYLE: resume LineLength StructObjectInspector soi = Inspectors.createFor(oti.getColumnInfo()); if (!outputConf.shouldResetSlowWrites()) { return new RecordWriterImpl(baseWriter, serializer, soi); } else { long writeTimeout = outputConf.getWriteResetTimeout(); return new ResettableRecordWriterImpl(baseWriter, serializer, soi, taskAttemptContext, baseOutputFormat, writeTimeout); } }
@Override public Void idempotentTask() throws TException { String dbName = outputDesc.getTableDesc().getDatabaseName(); String tableName = outputDesc.getTableDesc().getTableName(); ThriftHiveMetastore.Iface client = outputDesc.metastoreClient(conf); Table hiveTable = client.get_table(dbName, tableName); Partition partition = new Partition(); partition.setDbName(dbName); partition.setTableName(tableName); partition.setParameters(outputInfo.getTableParams()); List<String> partitionValues = HiveUtils.orderedPartitionValues( hiveTable.getPartitionKeys(), outputDesc.getPartitionValues()); partition.setValues(partitionValues); StorageDescriptor sd = new StorageDescriptor(hiveTable.getSd()); sd.setParameters(outputInfo.getSerializerParams()); sd.setLocation(outputInfo.getFinalOutputPath()); sd.setCols(outputInfo.getColumnInfo()); partition.setSd(sd); LOG.info("Registering partition with values {} located at {}", outputInfo.getSerializerParams(), outputInfo.getFinalOutputPath()); try { client.add_partition(partition); } catch (AlreadyExistsException e) { LOG.info("Partition already exists; Giraph must have just created it"); } catch (InvalidObjectException e) { throw new IllegalStateException(e); } return null; } };