public static String replaceTaskIdFromFilename(String filename, String fileId) { String taskId = getTaskIdFromFilename(filename); String newTaskId = replaceTaskId(taskId, fileId); String ret = replaceTaskIdFromFilename(filename, taskId, newTaskId); return (ret); }
public static String replaceTaskIdFromFilename(String filename, String fileId) { String taskId = getTaskIdFromFilename(filename); String newTaskId = replaceTaskId(taskId, fileId); String ret = replaceTaskIdFromFilename(filename, taskId, newTaskId); return (ret); }
private void testTaskIds(String [] taskIds, String expectedAttemptId, String expectedTaskId) { Configuration conf = new JobConf(TestOperators.class); for (String one: taskIds) { conf.set("mapred.task.id", one); String attemptId = Utilities.getTaskId(conf); assertEquals(expectedAttemptId, attemptId); assertEquals(Utilities.getTaskIdFromFilename(attemptId), expectedTaskId); assertEquals(Utilities.getTaskIdFromFilename(attemptId + ".gz"), expectedTaskId); assertEquals(Utilities.getTaskIdFromFilename (Utilities.toTempPath(new Path(attemptId + ".gz")).toString()), expectedTaskId); } }
int currReducer = Integer.parseInt(Utilities.getTaskIdFromFilename(Utilities .getTaskId(hconf)));
int currReducer = Integer.parseInt(Utilities.getTaskIdFromFilename(Utilities .getTaskId(hconf)));
String taskID = Utilities.getTaskIdFromFilename(fspKey);
/** * More stuff needs to be added here. Currently it only checks some basic * file naming libraries * The old test was deactivated as part of hive-405 */ public void testFileSinkOperator() throws Throwable { try { testTaskIds (new String [] { "attempt_200707121733_0003_m_000005_0", "attempt_local_0001_m_000005_0", "task_200709221812_0001_m_000005_0", "task_local_0001_m_000005_0" }, "000005_0", "000005"); testTaskIds (new String [] { "job_local_0001_map_000005", "job_local_0001_reduce_000005", }, "000005", "000005"); testTaskIds (new String [] {"1234567"}, "1234567", "1234567"); assertEquals(Utilities.getTaskIdFromFilename ("/mnt/dev005/task_local_0001_m_000005_0"), "000005"); System.out.println("FileSink Operator ok"); } catch (Throwable e) { e.printStackTrace(); throw e; } }
int acidBucketNum = Integer.parseInt(Utilities.getTaskIdFromFilename(taskId)); fsp.updaters[filesIdx] = HiveFileFormatUtils.getAcidRecordUpdater(jc, conf.getTableInfo(), acidBucketNum, conf, fsp.outPaths[filesIdx], inspector, reporter, -1);
String taskID = Utilities.getTaskIdFromFilename(fspKey);
int acidBucketNum = Integer.parseInt(Utilities.getTaskIdFromFilename(taskId)); fsp.updaters[filesIdx] = HiveFileFormatUtils.getAcidRecordUpdater(jc, conf.getTableInfo(), acidBucketNum, conf, fsp.outPaths[filesIdx], inspector, reporter, -1);
public static String replaceTaskIdFromFilename(String filename, String fileId) { String taskId = getTaskIdFromFilename(filename); String newTaskId = replaceTaskId(taskId, fileId); String ret = replaceTaskIdFromFilename(filename, taskId, newTaskId); return (ret); }
/** * Replace the task id from the filename. It is assumed that the filename is derived from the * output of getTaskId * * @param filename * filename to replace taskid "0_0" or "0_0.gz" by 33 to "33_0" or "33_0.gz" */ public static String replaceTaskIdFromFilename(String filename, int bucketNum) { String taskId = getTaskIdFromFilename(filename); String newTaskId = replaceTaskId(taskId, bucketNum); String ret = replaceTaskIdFromFilename(filename, taskId, newTaskId); return (ret); }
String taskId = getTaskIdFromFilename(one.getPath().getName()); FileStatus otherFile = taskIdToFile.get(taskId); if (otherFile == null) {
private void publishStats() { // Initializing a stats publisher StatsPublisher statsPublisher = Utilities.getStatsPublisher(jc); if (!statsPublisher.connect(jc)) { // just return, stats gathering should not block the main query. LOG.info("StatsPublishing error: cannot connect to database."); return; } String key; String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(hconf)); if (partitionSpecs.isEmpty()) { // In case of a non-partitioned table, the key for temp storage is just // "tableName + taskID" key = conf.getStatsAggPrefix() + taskID; } else { // In case of a partition, the key for temp storage is // "tableName + partitionSpecs + taskID" key = conf.getStatsAggPrefix() + partitionSpecs + Path.SEPARATOR + taskID; } statsPublisher.publishStat(key, StatsSetupConst.ROW_COUNT, Long.toString(stat.getNumRows())); statsPublisher.closeConnection(); } }
String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(hconf)); String spSpec = conf.getStaticSpec() != null ? conf.getStaticSpec() : "";
String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(hconf)); Map<String, String> statsToPublish = new HashMap<String, String>();
int currReducer = Integer.valueOf(Utilities.getTaskIdFromFilename(Utilities .getTaskId(hconf)));
String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(jc)); key = Utilities.join(key, taskID);
String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(hconf)); String spSpec = conf.getStaticSpec(); taskID = Utilities.getTaskIdFromFilename(fspKey);
int acidBucketNum = Integer.valueOf(Utilities.getTaskIdFromFilename(taskId)); fsp.updaters[filesIdx] = HiveFileFormatUtils.getAcidRecordUpdater(jc, conf.getTableInfo(), acidBucketNum, conf, fsp.outPaths[filesIdx], inspector, reporter, -1);