private void updatePaths(Path tmpPath, Path taskTmpPath) { String taskId = Utilities.getTaskId(jc); this.tmpPath = tmpPath; this.taskTmpPath = taskTmpPath; String inputFile = jc.get(MRJobConfig.MAP_INPUT_FILE); int lastSeparator = inputFile.lastIndexOf(Path.SEPARATOR) + 1; finalPath = new Path(tmpPath, inputFile.substring(lastSeparator)); outPath = new Path(taskTmpPath, Utilities.toTempPath(taskId)); }
private void testTaskIds(String [] taskIds, String expectedAttemptId, String expectedTaskId) { Configuration conf = new JobConf(TestOperators.class); for (String one: taskIds) { conf.set("mapred.task.id", one); String attemptId = Utilities.getTaskId(conf); assertEquals(expectedAttemptId, attemptId); assertEquals(Utilities.getTaskIdFromFilename(attemptId), expectedTaskId); assertEquals(Utilities.getTaskIdFromFilename(attemptId + ".gz"), expectedTaskId); assertEquals(Utilities.getTaskIdFromFilename (Utilities.toTempPath(new Path(attemptId + ".gz")).toString()), expectedTaskId); } }
private void updatePaths(Path tmpPath, Path taskTmpPath) { String taskId = Utilities.getTaskId(jc); this.tmpPath = tmpPath; this.taskTmpPath = taskTmpPath; String inputFile = jc.get(MRJobConfig.MAP_INPUT_FILE); int lastSeparator = inputFile.lastIndexOf(Path.SEPARATOR) + 1; finalPath = new Path(tmpPath, inputFile.substring(lastSeparator)); outPath = new Path(taskTmpPath, Utilities.toTempPath(taskId)); }
private void updatePaths(Path tp, Path ttp) { String taskId = Utilities.getTaskId(jc); tmpPath = tp; taskTmpPath = ttp; finalPath = new Path(tp, taskId); outPath = new Path(ttp, Utilities.toTempPath(taskId)); }
private void updatePaths(Path tp, Path ttp) { if (taskId == null) { taskId = Utilities.getTaskId(jc); } tmpPath = tp; if (isMmTable) { taskTmpPath = null; // Make sure we don't collide with the source. outPath = finalPath = new Path(tmpPath, taskId + ".merged"); } else { taskTmpPath = ttp; finalPath = new Path(tp, taskId); outPath = new Path(ttp, Utilities.toTempPath(taskId)); } if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("Paths for merge " + taskId + ": tmp " + tmpPath + ", task " + taskTmpPath + ", final " + finalPath + ", out " + outPath); } }
/** * @param configuration Job configs * * @return default consumer properties */ static Properties consumerProperties(Configuration configuration) { final Properties props = new Properties(); // we are managing the commit offset props.setProperty(CommonClientConfigs.CLIENT_ID_CONFIG, Utilities.getTaskId(configuration)); props.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); // we are seeking in the stream so no reset props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none"); String brokerEndPoint = configuration.get(KafkaTableProperties.HIVE_KAFKA_BOOTSTRAP_SERVERS.getName()); if (brokerEndPoint == null || brokerEndPoint.isEmpty()) { throw new IllegalArgumentException("Kafka Broker End Point is missing Please set Config " + KafkaTableProperties.HIVE_KAFKA_BOOTSTRAP_SERVERS.getName()); } props.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerEndPoint); props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); //case Kerberos is On if (UserGroupInformation.isSecurityEnabled()) { addKerberosJaasConf(configuration, props); } // user can always override stuff props.putAll(extractExtraProperties(configuration, CONSUMER_CONFIGURATION_PREFIX)); return props; }
Utilities.replaceTaskIdFromFilename(Utilities.getTaskId(hconf), bucketNum); this.finalPaths[writerOffset] = new Path(bDynParts ? buildTmpPath() : parent, bucketName); this.outPaths[writerOffset] = new Path(buildTaskOutputTempPath(), bucketName);
properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokers); properties.setProperty(CommonClientConfigs.CLIENT_ID_CONFIG, Utilities.getTaskId(getConf())); if (UserGroupInformation.isSecurityEnabled()) { KafkaUtils.addKerberosJaasConf(getConf(), properties);
private Path setupTempDirWithSingleOutputFile(Configuration hconf) throws IOException { Path tempDirPath = new Path("file://" + temporaryFolder.newFolder().getAbsolutePath()); Path taskOutputPath = new Path(tempDirPath, Utilities.getTaskId(hconf)); FileSystem.getLocal(hconf).create(taskOutputPath).close(); return tempDirPath; }
final String taskId = Utilities.getTaskId(jc); Path destFilePath = new Path(destDir, new Path(taskId)); for (int counter = 1; fs.exists(destFilePath); counter++) {
switch (writeSemantic) { case AT_LEAST_ONCE: recordWriter = new SimpleKafkaWriter(topic, Utilities.getTaskId(jc), producerProperties); break; case EXACTLY_ONCE:
tblSerializers = new HashMap<Byte, AbstractSerDe>(numAliases); bigKeysExistingMap = new HashMap<Byte, Boolean>(numAliases); taskId = Utilities.getTaskId(hconf);
tblSerializers = new HashMap<Byte, AbstractSerDe>(numAliases); bigKeysExistingMap = new HashMap<Byte, Boolean>(numAliases); taskId = Utilities.getTaskId(hconf);
Assert.assertEquals("false", kafkaProperties.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)); Assert.assertEquals("none", kafkaProperties.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)); Assert.assertEquals(Utilities.getTaskId(jobConf), kafkaProperties.get(CommonClientConfigs.CLIENT_ID_CONFIG));
fsp = prevFsp = null; valToPaths = new HashMap<String, FSPaths>(); taskId = originalTaskId = Utilities.getTaskId(hconf); initializeSpecPath(); fs = specPath.getFileSystem(hconf);
fsp = prevFsp = null; valToPaths = new HashMap<String, FSPaths>(); taskId = Utilities.getTaskId(hconf); initializeSpecPath(); fs = specPath.getFileSystem(hconf);
private void updatePaths(Path tp, Path ttp) { String taskId = Utilities.getTaskId(jc); tmpPath = tp; taskTmpPath = ttp; finalPath = new Path(tp, taskId); outPath = new Path(ttp, Utilities.toTempPath(taskId)); }
private void updatePaths(Path tmpPath, Path taskTmpPath) { String taskId = Utilities.getTaskId(jc); this.tmpPath = tmpPath; this.taskTmpPath = taskTmpPath; String inputFile = HiveConf.getVar(jc, HiveConf.ConfVars.HADOOPMAPFILENAME); int lastSeparator = inputFile.lastIndexOf(Path.SEPARATOR) + 1; finalPath = new Path(tmpPath, inputFile.substring(lastSeparator)); outPath = new Path(taskTmpPath, Utilities.toTempPath(taskId)); }