void dropStorage(String storage) { snapshotDumpPeriods.remove(storage); setSnapshotDumpPeriods(getName(), getProperties(), snapshotDumpPeriods); } }
public CubeDimensionTable(String dimName, String dimTblName, List<FieldSchema> columns, double weight, Set<String> storages, Map<String, String> properties) { this(dimName, dimTblName, columns, weight, getSnapshotDumpPeriods(storages), properties); }
private CubeDimensionTable getDimensionTable(Table tbl) { return new CubeDimensionTable(tbl); }
/** * Alter the dimension name that the table belongs to * * @param newDimName new dimension name. */ public void alterUberDim(String newDimName) { this.dimName = newDimName; setDimName(getName(), getProperties(), this.dimName); }
public CubeDimensionTable(Table tbl) { super(tbl); this.dimName = getDimName(getName(), getProperties()); Map<String, UpdatePeriod> dumpPeriods = getDumpPeriods(getName(), getProperties()); if (dumpPeriods != null) { this.snapshotDumpPeriods.putAll(dumpPeriods); } }
@Override protected void addProperties() { super.addProperties(); setDimName(getName(), getProperties(), dimName); setSnapshotDumpPeriods(getName(), getProperties(), snapshotDumpPeriods); }
dimTable.alterColumn(new FieldSchema("testAddDim", "int", "test add column")); if (dim.getName().equalsIgnoreCase(dimTblName)) { found = true; break; CubeDimensionTable altered = new CubeDimensionTable(alteredHiveTable); List<FieldSchema> columns = altered.getColumns(); boolean contains = false; for (FieldSchema column : columns) { dimTable.alterColumn(new FieldSchema("testAddDim", "float", "change type")); client.alterCubeDimensionTable(dimTblName, dimTable, storageTables); altered = new CubeDimensionTable(Hive.get(conf).getTable(dimTblName)); boolean typeChanged = false; for (FieldSchema column : altered.getColumns()) { if (column.getName().equals("testadddim") && column.getType().equals("float")) { typeChanged = true; storageTables.put(c1, s1); storageTables.put(c4, s1); dimTable.alterSnapshotDumpPeriod(c4, null); client.alterCubeDimensionTable(dimTblName, dimTable, storageTables); CubeDimensionTable altered2 = client.getDimensionTable(dimTblName); Table c4Table = client.getTable(c4TableName); assertEquals(c4Table.getInputFormatClass(), SequenceFileInputFormat.class); assertTrue(altered2.getStorages().contains("C4"));
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; }
allDimTables.put(tableName, dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { if (dimTable.hasStorageSnapshots(storageName)) { String storageTableName = getFactOrDimtableStorageTableName(dimTable.getName(), storageName); if (dimTableLatestPartitionExistsInMetastore(storageTableName, getDimension(dimTable.getDimName()).getTimedDimension())) { latestLookupCache.add(storageTableName.trim().toLowerCase());
CandidateDim candidate = i.next(); CubeDimensionTable dimtable = candidate.dimtable; if (dimtable.getStorages().isEmpty()) { cubeql .addDimPruningMsgs(dim, dimtable, new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_STORAGES)); for (String storage : dimtable.getStorages()) { if (isStorageSupportedOnDriver(storage)) { String tableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimtable.getName(), storage).toLowerCase(); if (validDimTables != null && !validDimTables.contains(tableName)) { if (dimtable.hasStorageSnapshots(storage)) {
storageNames.add(c1); Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1); CubeDimensionTable cubeDim = new CubeDimensionTable(countryDim.getName(), dimName, dimColumns, 0L, storageNames); client.createCubeDimensionTable(countryDim.getName(), dimName, dimColumns, 0L, storageNames, null, storageTables); boolean found = false; for (CubeDimensionTable dim : tbls) { if (dim.getName().equalsIgnoreCase(dimName)) { found = true; break; CubeDimensionTable cubeDim2 = new CubeDimensionTable(cubeTbl); assertTrue(cubeDim.equals(cubeDim2));
public static XDimensionTable dimTableFromCubeDimTable(CubeDimensionTable cubeDimTable) { if (cubeDimTable == null) { return null; } XDimensionTable dimTab = XCF.createXDimensionTable(); dimTab.setDimensionName(cubeDimTable.getDimName()); dimTab.setTableName(cubeDimTable.getName()); dimTab.setWeight(cubeDimTable.weight()); dimTab.setColumns(new XColumns()); dimTab.setProperties(new XProperties()); dimTab.setStorageTables(new XStorageTables()); for (FieldSchema column : cubeDimTable.getColumns()) { dimTab.getColumns().getColumn().add(columnFromFieldSchema(column)); } dimTab.getProperties().getProperty().addAll(xPropertiesFromMap(cubeDimTable.getProperties())); return dimTab; }
XDimensionTable dimTable = dt.getValue(); CubeDimensionTable cdim = JAXBUtils.cubeDimTableFromDimTable(dimTable); assertTrue(cdim.getStorages().contains("test")); assertTrue(cdim.getStorages().contains("test2")); assertEquals(cdim.getSnapshotDumpPeriods().get("test2"), DAILY); assertEquals(cdim.getSnapshotDumpPeriods().get("test"), HOURLY); dimTable = dt.getValue(); cdim = JAXBUtils.cubeDimTableFromDimTable(dimTable); assertFalse(cdim.getStorages().contains("test")); assertTrue(cdim.getStorages().contains("test2")); assertEquals(cdim.getSnapshotDumpPeriods().get("test2"), DAILY); dimTable = dt.getValue(); cdim = JAXBUtils.cubeDimTableFromDimTable(dimTable); assertTrue(cdim.getStorages().contains("test2")); assertTrue(cdim.getStorages().contains("test3")); assertNull(cdim.getSnapshotDumpPeriods().get("test3")); } finally { setCurrentDatabase(prevDb, mediaType);
CubeDimensionTable cubeDim = new CubeDimensionTable(zipDim.getName(), dimName, dimColumns, 0L, dumpPeriods); client.createCubeDimensionTable(zipDim.getName(), dimName, dimColumns, 0L, dumpPeriods, null, storageTables); boolean found = false; for (CubeDimensionTable dim : tbls) { if (dim.getName().equalsIgnoreCase(dimName)) { found = true; break; CubeDimensionTable cubeDim2 = new CubeDimensionTable(cubeTbl); assertTrue(cubeDim.equals(cubeDim2)); StoragePartitionDesc sPartSpec0 = new StoragePartitionDesc(cubeDim.getName(), timePartsNow, null, HOURLY); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timePartsNow)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); assertEquals(client.getAllParts(storageTableName).size(), 2); cubeDim.alterColumn(newcol); client.alterCubeDimensionTable(cubeDim.getName(), cubeDim, storageTables); StoragePartitionDesc sPartSpec1 = new StoragePartitionDesc(cubeDim.getName(), timeParts1, null, HOURLY); sPartSpec1.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); sPartSpec1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timePartsNow)); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timeParts1)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey()));
@Override public void dropAllStoragesOfDimTable(LensSessionHandle sessionid, String dimTblName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable tab = msClient.getDimensionTable(dimTblName); int total = tab.getStorages().size(); int i = 0; List<String> storageNames = new ArrayList<>(tab.getStorages()); for (String s : storageNames) { msClient.dropStorageFromDim(dimTblName, s); log.info("Dropped storage " + s + " from dimension table " + dimTblName + " [" + ++i + "/" + total + "]"); } log.info("Dropped " + total + " storages from dimension table " + dimTblName); } catch (HiveException exc) { throw new LensException(exc); } }
@Override public String getName() { return dimtable.getName(); }
/** * Adds storage to dimension and creates corresponding storage table * * @param dim The CubeDimensionTable * @param storage The storage * @param dumpPeriod The dumpPeriod if any, null otherwise * @param storageTableDesc The storage table description * @throws LensException */ public void addStorage(CubeDimensionTable dim, String storage, UpdatePeriod dumpPeriod, StorageTableDesc storageTableDesc) throws LensException { dim.alterSnapshotDumpPeriod(storage, dumpPeriod); createOrAlterStorageHiveTable(getTableWithTypeFailFast(dim.getName(), CubeTableType.DIM_TABLE), storage, storageTableDesc); alterCubeTable(dim.getName(), getTableWithTypeFailFast(dim.getName(), CubeTableType.DIM_TABLE), dim); updateDimCache(dim.getName()); }
public Set<String> getPartCols() { Set<String> partCols = Sets.newHashSet(); String partColsStr = getProperties().get(MetastoreUtil.getDimTablePartsKey(getName())); if (partColsStr != null) { for (String s : StringUtils.split(partColsStr, ",")) { if (!StringUtils.isBlank(s)) { partCols.add(s); } } } return partCols; }
@Override public boolean equals(Object obj) { if (!super.equals(obj)) { return false; } CubeDimensionTable other = (CubeDimensionTable) obj; if (this.getDimName() == null) { if (other.getDimName() != null) { return false; } } else { if (!this.getDimName().equals(other.getDimName())) { return false; } } if (this.getSnapshotDumpPeriods() == null) { if (other.getSnapshotDumpPeriods() != null) { return false; } } else { if (!this.getSnapshotDumpPeriods().equals(other.getSnapshotDumpPeriods())) { return false; } } return true; }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; } /**