public String getHDFSLocation(boolean needFilePath) throws IOException { if (hdfsLocation == null) { hdfsLocation = computeHDFSLocation(needFilePath); } return hdfsLocation; }
private String computeHDFSLocation(boolean needFilePath) throws IOException { String override = KylinConfig.getInstanceFromEnv().getOverrideHiveTableLocation(hiveTable); if (override != null) { logger.debug("Override hive table location " + hiveTable + " -- " + override); return override; } String hdfsDir = null; try { HiveClient hiveClient = new HiveClient(); hdfsDir = hiveClient.getHiveTableLocation(database, hiveTable); } catch (Exception e) { e.printStackTrace(); throw new IOException(e); } if (needFilePath) { FileSystem fs = HadoopUtil.getFileSystem(hdfsDir); FileStatus file = findOnlyFile(hdfsDir, fs); return file.getPath().toString(); } else { return hdfsDir; } }
@Override public String getColumnDelimeter() throws IOException { return getFileTable().getColumnDelimeter(); }
HiveTable htable = new HiveTable(metadataManager, lookupTableName); LookupBytesTable btable = new LookupBytesTable(metadataManager.getTableDesc(lookupTableName), join.getPrimaryKey(), htable); lookupTables.put(lookupTableName, btable);
@Test public void basicTest() throws Exception { String tableName = "EDW.TEST_SITES"; HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName); TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName); String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath(); snapshotMgr.wipeoutCache(); SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath); // compare hive & snapshot TableReader hiveReader = hiveTable.getReader(); TableReader snapshotReader = snapshot.getReader(); while (true) { boolean hiveNext = hiveReader.next(); boolean snapshotNext = snapshotReader.next(); assertEquals(hiveNext, snapshotNext); if (hiveNext == false) break; String[] hiveRow = hiveReader.getRow(); String[] snapshotRow = snapshotReader.getRow(); assertArrayEquals(hiveRow, snapshotRow); } } }
@Override public TableSignature getSignature() throws IOException { try { String path = computeHDFSLocation(); Pair<Long, Long> sizeAndLastModified = FileTable.getSizeAndLastModified(path); long size = sizeAndLastModified.getFirst(); long lastModified = sizeAndLastModified.getSecond(); // for non-native hive table, cannot rely on size & last modified on HDFS if (getHiveClient().isNativeTable(database, hiveTable) == false) { lastModified = System.currentTimeMillis(); // assume table is ever changing } return new TableSignature(path, size, lastModified); } catch (Exception e) { if (e instanceof IOException) throw (IOException) e; else throw new IOException(e); } }
private FileTable getFileTable() throws IOException { if (fileTable == null) { fileTable = new FileTable(getHDFSLocation(true), nColumns); } return fileTable; }
private String computeHDFSLocation() throws Exception { String override = KylinConfig.getInstanceFromEnv().getOverrideHiveTableLocation(hiveTable); if (override != null) { logger.debug("Override hive table location " + hiveTable + " -- " + override); return override; } return getHiveClient().getHiveTableLocation(database, hiveTable); }
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException { MetadataManager metaMgr = getMetadataManager(); SnapshotManager snapshotMgr = getSnapshotManager(); HiveTable hiveTable = new HiveTable(metaMgr, lookupTable); TableDesc tableDesc = metaMgr.getTableDesc(lookupTable); SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc); cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath()); saveResource(cubeSeg.getCubeInstance()); return snapshot; }
table = new FileTable(factColumnsPath + "/" + col.getName(), -1); } else { table = new HiveTable(metaMgr, col.getTable());
@Override public TableSignature getSignature() throws IOException { return getFileTable().getSignature(); }
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException { MetadataManager metaMgr = getMetadataManager(); SnapshotManager snapshotMgr = getSnapshotManager(); HiveTable hiveTable = new HiveTable(metaMgr, lookupTable); TableDesc tableDesc = metaMgr.getTableDesc(lookupTable); SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc); cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath()); saveResource(cubeSeg.getCubeInstance()); return snapshot; }
table = new FileTable(factColumnsPath + "/" + col.getName(), -1); } else { table = new HiveTable(metaMgr, col.getTable());