/** * Test db storage. * * @throws HiveException the hive exception */ @Test(groups = "first") public void testDBStorage() throws Exception { CubeMetastoreClient cc = CubeMetastoreClient.getInstance(conf); if (!cc.tableExists(DB_STORAGE1)) { cc.createStorage(db1); } if (!cc.tableExists(DB_STORAGE2)) { cc.createStorage(db2); } }
private String getFilter(CubeMetastoreClient client, String tableName, String values) throws LensException { List<FieldSchema> cols = client.getHiveTable(tableName).getPartCols(); String[] vals = StringUtils.split(values, ","); if (vals.length != cols.size()) { log.error("Values for all the part columns not specified, cols:" + cols + " vals:" + Arrays.toString(vals)); throw new BadRequestException("Values for all the part columns not specified"); } StringBuilder filter = new StringBuilder(); for (int i = 0; i < vals.length; i++) { filter.append(cols.get(i).getName()); filter.append("="); filter.append("\""); filter.append(vals[i]); filter.append("\""); if (i != (vals.length - 1)) { filter.append(" AND "); } } return filter.toString(); }
public <T extends Equals & HashCode & ToString> void createEntity(T entity) throws LensException { if (entity instanceof XStorage) { createStorage((XStorage) entity); } else if (entity instanceof XCube) { createCube((XCube)entity); } else if (entity instanceof XDimension) { createDimension((XDimension) entity); } else if (entity instanceof XFact) { createFactTable((XFact) entity); } else if (entity instanceof XDimensionTable) { createCubeDimensionTable((XDimensionTable) entity); } else if (entity instanceof XSegmentation) { createSegmentation((XSegmentation) entity); } else { throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass()); } }
public <T extends Equals & HashCode & ToString> void updateEntity(String name, T entity) throws LensException, HiveException { if (entity instanceof XStorage) { alterStorage((XStorage) entity); } else if (entity instanceof XCube) { alterCube((XCube)entity); } else if (entity instanceof XDimension) { alterDimension((XDimension) entity); } else if (entity instanceof XFact) { alterCubeFactTable((XFact) entity); } else if (entity instanceof XDimensionTable) { alterCubeDimensionTable((XDimensionTable) entity); } else if (entity instanceof XSegmentation) { alterSegmentation((XSegmentation) entity); } else { throw new LensException("Unable to alter entity " + entity + " as it's unrecognizable: " + entity.getClass()); } }
public List<Partition> getAllParts(String storageTableName) throws HiveException, LensException { return getClient().getPartitions(getHiveTable(storageTableName)); }
client.createCubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 100L, null, storageTables); assertTrue(client.tableExists(factName)); Table cubeTbl = client.getHiveTable(factName); assertTrue(client.isFactTable(cubeTbl)); assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME)); CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl); assertTrue(cubeFact.equals(cubeFact2)); assertTrue(client.tableExists(storageTableName)); client.addPartition(partSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); String storageTableName = getFactOrDimtableStorageTableName(cubeFact.getName(), c1); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); assertEquals(client.getAllParts(storageTableName).size(), 1); client.dropPartition(cubeFact.getName(), c1, timeParts, null, HOURLY); assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); assertEquals(client.getAllParts(storageTableName).size(), 0);
client.createCube(VIRTUAL_CUBE_NAME, cubeMeasures, cubeDimensions); String sourceFactName = "testMetastoreFact1"; List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); client.createCubeFactTable(CUBE_NAME, sourceFactName, factColumns, updatePeriods, 10.0, sourceFactPropertiesMap, storageTables); client.createVirtualFactTable(VIRTUAL_CUBE_NAME, virtualFactName, sourceFactName, null, virtualFactPropertiesMap); assertTrue(client.tableExists(virtualFactName)); Table virtualTbl = client.getHiveTable(virtualFactName); assertTrue(client.isVirtualFactTable(virtualTbl)); assertTrue(client.isVirtualFactTableForCube(virtualTbl, VIRTUAL_CUBE_NAME)); assertTrue(client.getAllFacts(client.getCube(VIRTUAL_CUBE_NAME)).get(0).getName().equals(virtualFactName.trim() .toLowerCase())); CubeVirtualFactTable actualcubeVirtualFact = (CubeVirtualFactTable) (client.getFactTable(virtualFactName)); assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); com.google.common.base.Optional.fromNullable(null), alterVirtualFactPropertiesMap, sourceFact); client.alterVirtualCubeFactTable(cubeVirtualFact); actualcubeVirtualFact = (CubeVirtualFactTable) client.getFactTable(virtualFactName); assertEquals(actualcubeVirtualFact.getProperties().get("name1"), "newvalue2"); assertEquals(actualcubeVirtualFact.getProperties().get("name3"), "value3"); sourceFact.alterColumn(newcol); sourceFact.alterWeight(100);
client.createCubeDimensionTable(zipDim.getName(), dimName, dimColumns, 0L, dumpPeriods, null, storageTables); assertTrue(client.tableExists(dimName)); Table cubeTbl = client.getHiveTable(dimName); assertTrue(client.isDimensionTable(cubeTbl)); List<CubeDimensionTable> tbls = client.getAllDimensionTables(zipDim); boolean found = false; for (CubeDimensionTable dim : tbls) { assertTrue(client.tableExists(storageTableName)); assertFalse(client.dimTableLatestPartitionExists(storageTableName)); client.addPartition(sPartSpec0, c1, CubeTableType.DIM_TABLE); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timePartsNow)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); assertEquals(client.getAllParts(storageTableName).size(), 2); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 1); Partition latestPart = parts.get(0); client.alterCubeDimensionTable(cubeDim.getName(), cubeDim, storageTables); sPartSpec1.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); sPartSpec1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); client.addPartition(sPartSpec1, c1, CubeTableType.DIM_TABLE);
client.createCubeFactTable(CUBE_NAME_WITH_PROPS, factName, factColumns, updatePeriods, 0L, null, storageTables); assertTrue(client.tableExists(factName)); Table cubeTbl = client.getHiveTable(factName); assertTrue(client.isFactTable(cubeTbl)); assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME_WITH_PROPS)); CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl); assertTrue(cubeFact.equals(cubeFact2)); assertTrue(client.tableExists(storageTableName)); client.getHiveTable(c1TableName); Table c2Table = client.getHiveTable(c2TableName); c2Table.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, getDatePartitionKey()), StoreAllPartitionTimeline.class.getCanonicalName()); c2Table.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, etPart.getName()), StoreAllPartitionTimeline.class.getCanonicalName()); client.pushHiveTable(c2Table); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1, CubeTableType.FACT); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c2, CubeTableType.FACT); PartitionTimeline timeline1Temp = client.partitionTimelineCache.get(factName, c1, HOURLY, getDatePartitionKey()); PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, HOURLY, getDatePartitionKey()); assertEquals(client.getAllParts(c1TableName).size(), 3); assertEquals(client.getAllParts(c2TableName).size(), 3); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey()));
client.createCubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, storageTables); assertTrue(client.tableExists(factName)); Table cubeTbl = client.getHiveTable(factName); assertTrue(client.isFactTable(cubeTbl)); assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME)); assertEquals(client.getAllFacts(client.getCube(CUBE_NAME)).get(0).getName(), factName.toLowerCase()); assertEquals(client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), factName.toLowerCase()); CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl); assertTrue(cubeFact.equals(cubeFact2)); assertTrue(client.tableExists(storageTableName)); client.addPartition(partSpec, c1, CubeTableType.FACT); fail("Add should fail since non_existing_part_col is non-existing"); } catch (LensException e) { client.addPartition(partSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); client.alterCubeFactTable(cubeFact.getName(), cubeFact, storageTables, new HashMap<String, String>()); String storageTableName = getFactOrDimtableStorageTableName(factName, c1); assertEquals(client.getAllParts(storageTableName).size(), 1); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); partSpec2.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); partSpec2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); client.addPartition(partSpec2, c1, CubeTableType.FACT);
client.createCubeDimensionTable(stateDim.getName(), dimName, dimColumns, 100L, dumpPeriods, null, storageTables); assertTrue(client.tableExists(dimName)); Table cubeTbl = client.getHiveTable(dimName); assertTrue(client.isDimensionTable(cubeTbl)); CubeDimensionTable cubeDim2 = new CubeDimensionTable(cubeTbl); assertTrue(cubeDim.equals(cubeDim2)); List<CubeDimensionTable> stateTbls = client.getAllDimensionTables(stateDim); boolean found = false; for (CubeDimensionTable dim : stateTbls) { assertTrue(client.tableExists(storageTableName)); client.addPartition(sPartSpec, c1, CubeTableType.DIM_TABLE); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timeParts)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); String storageTableName = getFactOrDimtableStorageTableName(dimName, c1); assertEquals(client.getAllParts(storageTableName).size(), 2); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(1, parts.size()); assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName()); assertEquals(parts.get(0).getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 0)); client.dropPartition(cubeDim.getName(), c1, timeParts, null, HOURLY); assertFalse(client.dimPartitionExists(cubeDim.getName(), c1, timeParts)); assertFalse(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); assertEquals(client.getAllParts(storageTableName).size(), 0);
client.createCubeFactTable(CUBE_NAME, factNameSkipPart, factColumns, updatePeriods, 0L, factProps, storageTables); assertTrue(client.tableExists(factNameSkipPart)); Table cubeTbl = client.getHiveTable(factNameSkipPart); assertTrue(client.isFactTable(cubeTbl)); assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME)); CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl); assertTrue(cubeFactWithParts.equals(cubeFact2)); assertTrue(client.tableExists(storageTableName)); storageDescs.add(sPartSpecTwoMonthsBack); client.addPartitions(storageDescs, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsNow, partSpec)); assertFalse(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsBeforeTwoMonths, partSpec));
client.createCubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, storageTables); client.alterCubeFactTable(factName, factTable, storageTables, new HashMap<String, String>()); Table c1Table = client.getTable(c1TableName); assertEquals(c1Table.getInputFormatClass().getCanonicalName(), TextInputFormat.class.getCanonicalName()); s1 = new StorageTableDesc(SequenceFileInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, updatePeriodStoragePrefix.put(DAILY, c4); factTable.addStorage(c4, hourlyAndDaily, updatePeriodStoragePrefix); client.alterCubeFactTable(factName, factTable, storageTables, new HashMap<String, String>()); CubeFactTable altered2 = client.getCubeFactTable(factName); assertTrue(client.tableExists(c1TableName)); Table alteredC1Table = client.getTable(c1TableName); assertEquals(alteredC1Table.getInputFormatClass(), SequenceFileInputFormat.class); assertEquals(alteredC1Table.getSerdeParam(serdeConstants.FIELD_DELIM), ":"); assertTrue(altered2.getUpdatePeriods().get("C4").equals(hourlyAndDaily)); String c4TableName = getFactOrDimtableStorageTableName(factName, c4); assertTrue(client.tableExists(c4TableName)); Map<String, StorageTableDesc> storageTableDescMap = new HashMap<>(); storageTableDescMap.put(c3, s1); client.addStorage(altered2, c3, hourlyAndDaily, storageTableDescMap, updatePeriodStoragePrefix); CubeFactTable altered3 = client.getCubeFactTable(factName); assertTrue(altered3.getStorages().contains("C3")); assertTrue(altered3.getUpdatePeriods().get("C3").equals(hourlyAndDaily)); String storageTableName = getFactOrDimtableStorageTableName(factName, c3); assertTrue(client.tableExists(storageTableName));
client.createCubeDimensionTable(zipDim.getName(), dimTblName, dimColumns, 100L, dumpPeriods, null, storageTables); CubeDimensionTable dimTable = client.getDimensionTable(dimTblName); dimTable.alterColumn(new FieldSchema("testAddDim", "int", "test add column")); List<CubeDimensionTable> tbls = client.getAllDimensionTables(zipDim); boolean found = false; for (CubeDimensionTable dim : tbls) { client.alterCubeDimensionTable(dimTblName, dimTable, storageTables); client.alterCubeDimensionTable(dimTblName, dimTable, storageTables); Table c1Table = client.getTable(c1TableName); assertEquals(c1Table.getInputFormatClass(), TextInputFormat.class); s1 = new StorageTableDesc(SequenceFileInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, storageTables.put(c4, s1); dimTable.alterSnapshotDumpPeriod(c4, null); client.alterCubeDimensionTable(dimTblName, dimTable, storageTables); CubeDimensionTable altered2 = client.getDimensionTable(dimTblName); assertTrue(client.tableExists(c1TableName)); Table alteredC1Table = client.getTable(c1TableName); assertEquals(alteredC1Table.getInputFormatClass(), SequenceFileInputFormat.class); boolean storageTblColAltered = false; assertTrue(client.tableExists(c4TableName)); Table c4Table = client.getTable(c4TableName); assertEquals(c4Table.getInputFormatClass(), SequenceFileInputFormat.class);
client.createCubeDimensionTable(zipDim.getName(), dimName, dimColumns, 0L, dumpPeriods, null, storageTables); assertTrue(client.tableExists(dimName)); Table cubeTbl = client.getHiveTable(dimName); assertTrue(client.isDimensionTable(cubeTbl)); List<CubeDimensionTable> tbls = client.getAllDimensionTables(zipDim); boolean found = false; for (CubeDimensionTable dim : tbls) { assertTrue(client.tableExists(storageTableName)); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1, CubeTableType.DIM_TABLE); String c1TableName = getFactOrDimtableStorageTableName(cubeDim.getName(), c1); assertEquals(client.getAllParts(c1TableName).size(), 8); client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1, CubeTableType.DIM_TABLE); assertEquals(client.getAllParts(c1TableName).size(), 10); assertEquals(getLatestValues(c1TableName, HOURLY, partColNames, null), toPartitionArray(HOURLY, 1, 1, 1)); Map<String, Date> timeParts6 = getTimePartitionByOffsets(getDatePartitionKey(), -2, itPart.getName(), -1, client.addPartition(partSpec6, c1, CubeTableType.DIM_TABLE); assertEquals(client.getAllParts(c1TableName).size(), 11); client.addPartition(partSpec7, c1, CubeTableType.DIM_TABLE); assertEquals(client.getAllParts(c1TableName).size(), 12);
String segmentName = "testMetastoreSegmentation"; Table cubeTbl = client.getHiveTable(CUBE_NAME); assertTrue(client.isCube(cubeTbl)); client.createSegmentation(CUBE_NAME, segmentName, cubeSegs, 0L, props); assertEquals(client.getSegmentation(segmentName).getSegments().size(), 3); segmentation.alterBaseCubeName("segCubeAltered"); segmentation.alterWeight(100.0); client.alterSegmentation(segmentName, segmentation); assertNotNull(client.getSegmentation(segmentName)); assertEquals(client.getSegmentation(segmentName).getSegments().size(), 4); assertEquals(client.getSegmentation(segmentName).getBaseCube(), "segCubeAltered"); assertEquals(client.getSegmentation(segmentName).weight(), 100.0); client.alterSegmentation(segmentName, segmentation); assertEquals(client.getSegmentation(segmentName).getSegments().size(), 3); client.dropSegmentation(segmentName); assertFalse(client.tableExists(segmentName));
Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1); CubeDimensionTable cubeDim = new CubeDimensionTable(countryDim.getName(), dimName, dimColumns, 0L, storageNames); client.createCubeDimensionTable(countryDim.getName(), dimName, dimColumns, 0L, storageNames, null, storageTables); assertTrue(client.tableExists(dimName)); Table cubeTbl = client.getHiveTable(dimName); assertTrue(client.isDimensionTable(cubeTbl)); List<CubeDimensionTable> tbls = client.getAllDimensionTables(countryDim); boolean found = false; for (CubeDimensionTable dim : tbls) { assertTrue(client.tableExists(storageTableName)); assertTrue(!client.getHiveTable(storageTableName).isPartitioned());
Cube cube = new Cube(cubeName, moreCubeMeasures, moreCubeDimensions, cubeExpressions, joinChains, emptyHashMap, 0.0); client.createCube(cubeName, moreCubeMeasures, moreCubeDimensions, cubeExpressions, joinChains, emptyHashMap); assertTrue(client.tableExists(cubeName)); Table cubeTbl = client.getHiveTable(cubeName); assertTrue(client.isCube(cubeTbl)); Cube cube2 = new Cube(cubeTbl); assertTrue(cube.equals(cube2)); client.createDerivedCube(cubeName, derivedCubeName, moreMeasures, moreDimensions, emptyHashMap, 0L); assertTrue(client.tableExists(derivedCubeName)); Table derivedTbl = client.getHiveTable(derivedCubeName); assertTrue(client.isCube(derivedTbl)); DerivedCube dcube2 = new DerivedCube(derivedTbl, cube); assertTrue(derivedCube.equals(dcube2)); assertNotNull(dcube2.getDimAttributeByName("dim1")); assertTrue(dcube2.allFieldsQueriable()); client.dropCube(derivedCubeName); client.dropCube(cubeName);
CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, storageUpdatePeriodMap, storageTablePartitionColumns); client.createCubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, storageTables, storageUpdatePeriodMap, storageTablePartitionColumns); assertTrue(client.tableExists(factName)); Table cubeTbl = client.getHiveTable(factName); assertTrue(client.isFactTable(cubeTbl)); assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME)); assertTrue(client.tableExists(storageTableName)); Table c1TableHourly = client.getHiveTable(c1TableNameHourly); c1TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, getDatePartitionKey()), StoreAllPartitionTimeline.class.getCanonicalName()); c1TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, etPart.getName()), StoreAllPartitionTimeline.class.getCanonicalName()); client.pushHiveTable(c1TableHourly); Table c2TableHourly = client.getHiveTable(c2TableNameHourly); c2TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, getDatePartitionKey()), EndsAndHolesPartitionTimeline.class.getCanonicalName()); c2TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, etPart.getName()), EndsAndHolesPartitionTimeline.class.getCanonicalName()); client.pushHiveTable(c2TableHourly); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1, CubeTableType.FACT);
String storageTableName = getStorageTableName(cubeTableName.trim(), storageName, updatePeriod); Table hiveTable = getHiveTable(storageTableName); List<FieldSchema> partCols = hiveTable.getPartCols(); List<String> partColNames = new ArrayList<>(partCols.size()); if (isDimensionTable(cubeTableName)) { String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS); Map<String, LatestInfo> latest = new HashMap<>(); Partition part = getLatestPart(storageTableName, timeCol, nonTimePartSpec); getNextLatestOfDimtable(hiveTable, timeCol, timeColIndex, updatePeriod, nonTimePartSpec); latestAvailable = (latestInfo != null && latestInfo.part != null); latest.put(timeCol, latestInfo); getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, latest, nonTimePartSpec); if (!latestAvailable) { getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, null, null); if (partitionTimelineCache.updateForDeletion(cubeTableName, storageName, updatePeriod, timePartSpec)) { this.alterTablePartitionCache((Storage.getPrefix(storageName) + cubeTableName).toLowerCase(), updatePeriod, storageTableName);