/** * Add a partition specified by the storage partition desc on the storage passed. * <p></p> * TODO: separate methods for fact and dim partition addition. * * @param partSpec The storage partition description * @param storageName The storage object * @param type * @throws HiveException */ public List<Partition> addPartition(StoragePartitionDesc partSpec, String storageName, CubeTableType type) throws HiveException, LensException { return addPartitions(Collections.singletonList(partSpec), storageName, type); }
/** * Add a partition specified by the storage partition desc on the storage passed. * <p></p> * TODO: separate methods for fact and dim partition addition. * * @param partSpec The storage partition description * @param storageName The storage object * @param type * @throws HiveException */ public List<Partition> addPartition(StoragePartitionDesc partSpec, String storageName, CubeTableType type) throws HiveException, LensException { return addPartitions(Collections.singletonList(partSpec), storageName, type); }
/** batch addition */ public List<Partition> addPartitions(List<StoragePartitionDesc> storagePartitionDescs, String storageName, CubeTableType type) throws HiveException, LensException { List<Partition> partsAdded = Lists.newArrayList(); for (Map.Entry<String, Map<UpdatePeriod, List<StoragePartitionDesc>>> group : groupPartitionDescs( storagePartitionDescs).entrySet()) { String factOrDimtable = group.getKey(); for (Map.Entry<UpdatePeriod, List<StoragePartitionDesc>> entry : group.getValue().entrySet()) { partsAdded.addAll(addPartitions(factOrDimtable, storageName, entry.getKey(), entry.getValue(), type)); } } return partsAdded; }
/** batch addition */ public List<Partition> addPartitions(List<StoragePartitionDesc> storagePartitionDescs, String storageName, CubeTableType type) throws HiveException, LensException { List<Partition> partsAdded = Lists.newArrayList(); for (Map.Entry<String, Map<UpdatePeriod, List<StoragePartitionDesc>>> group : groupPartitionDescs( storagePartitionDescs).entrySet()) { String factOrDimtable = group.getKey(); for (Map.Entry<UpdatePeriod, List<StoragePartitionDesc>> entry : group.getValue().entrySet()) { partsAdded.addAll(addPartitions(factOrDimtable, storageName, entry.getKey(), entry.getValue(), type)); } } return partsAdded; }
@Override public int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName, XPartitionList partitions) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ checkDimTableStorage(sessionid, dimTblName, storageName); return getClient(sessionid).addPartitions(storagePartSpecListFromXPartitionList(partitions), storageName, CubeTableType.DIM_TABLE).size(); } catch (HiveException exc) { throw new LensException(exc); } }
@Override public int addPartitionsToFactStorage(LensSessionHandle sessionid, String fact, String storageName, XPartitionList partitions) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { checkFactStorage(sessionid, fact, storageName); return getClient(sessionid) .addPartitions(storagePartSpecListFromXPartitionList(partitions), storageName, CubeTableType.FACT).size(); } catch (HiveException exc) { throw new LensException(exc); } }
storageDescs.add(sPartSpecTwoMonthsBack); client.addPartitions(storageDescs, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsNow, partSpec)); assertFalse(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY,
client.addPartitions(storageDescs, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timePartsNow, partSpec)); assertFalse(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY,
temp = cal.getTime(); client.addPartitions(storagePartitionDescs, c4, CubeTableType.FACT); client.clearHiveTableCache(); table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c4)));
final StoragePartitionDesc partSpec3 = new StoragePartitionDesc(cubeDim.getName(), timeParts3, nonTimeSpec, HOURLY); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1, CubeTableType.DIM_TABLE); String c1TableName = getFactOrDimtableStorageTableName(cubeDim.getName(), c1); assertEquals(client.getAllParts(c1TableName).size(), 8); final StoragePartitionDesc partSpec5 = new StoragePartitionDesc(cubeDim.getName(), timeParts5, null, HOURLY); client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1, CubeTableType.DIM_TABLE);
HOURLY); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1, CubeTableType.FACT); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c2, CubeTableType.FACT); PartitionTimeline timeline1Temp = client.partitionTimelineCache.get(factName, c1, HOURLY, getDatePartitionKey()); PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, HOURLY, getDatePartitionKey());
HOURLY); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1, CubeTableType.FACT); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c2, CubeTableType.FACT); PartitionTimeline timeline1Temp = client.partitionTimelineCache.get(factName, c1, HOURLY, getDatePartitionKey()); PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, HOURLY, getDatePartitionKey()); final StoragePartitionDesc partSpec5 = new StoragePartitionDesc(cubeFact.getName(), timeParts5, null, HOURLY); client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1, CubeTableType.FACT); client.addPartitions(Arrays.asList(partSpec4, partSpec5), c2, CubeTableType.FACT);