@Override public Iterator<TimePartition> iterator() { return new Iterator<TimePartition>() { TimePartition cur = getFirst(); @Override public boolean hasNext() { return cur != null && getLatest() != null && !cur.after(getLatest()); } @Override public TimePartition next() { while (holes.contains(cur)) { cur = cur.next(); } TimePartition toReturn = cur; cur = cur.next(); return toReturn; } @Override public void remove() { throw new UnsupportedOperationException(); } }; } }
@Override public boolean add(@NonNull TimePartition partition) throws LensException { if (isEmpty()) { // First partition being added first = partition; latest = partition; return true; } if (partition.before(first)) { addHolesBetween(partition, first, partition.getUpdatePeriod()); first = partition; return true; } else if (partition.after(latest)) { addHolesBetween(latest, partition, partition.getUpdatePeriod()); latest = partition; return true; } else { return holes.remove(partition); } }
@Override public boolean drop(@NonNull TimePartition toDrop) throws LensException { if (first.equals(latest) && first.equals(toDrop)) { this.first = null; this.latest = null; this.holes.clear(); return true; } else if (first.equals(toDrop)) { this.first = this.getNextPartition(first, latest, 1); return true; } else if (latest.equals(toDrop)) { this.latest = this.getNextPartition(latest, first, -1); return true; } else { return addHole(toDrop); } }
@Override public boolean initFromProperties(Map<String, String> properties) throws LensException { first = null; latest = null; holes.clear(); String firstStr = properties.get("first"); String latestStr = properties.get("latest"); String holesStr = MetastoreUtil.getNamedStringValue(properties, "holes"); if (!Strings.isNullOrEmpty(firstStr)) { first = TimePartition.of(getUpdatePeriod(), firstStr); } if (!Strings.isNullOrEmpty(latestStr)) { latest = TimePartition.of(getUpdatePeriod(), latestStr); } holes = Sets.newTreeSet(); if (!Strings.isNullOrEmpty(holesStr)) { for (String hole : holesStr.split("\\s*,\\s*")) { holes.add(TimePartition.of(getUpdatePeriod(), hole)); } } return isConsistent(); }
private void assertTimeline(EndsAndHolesPartitionTimeline endsAndHolesPartitionTimeline, StoreAllPartitionTimeline storeAllPartitionTimeline, UpdatePeriod updatePeriod, Date first, Date latest, Date... holes) throws LensException { TimePartition firstPart = TimePartition.of(updatePeriod, first); TimePartition latestPart = TimePartition.of(updatePeriod, latest); assertEquals(endsAndHolesPartitionTimeline.getFirst(), firstPart); assertEquals(endsAndHolesPartitionTimeline.getLatest(), TimePartition.of(updatePeriod, latest)); assertEquals(endsAndHolesPartitionTimeline.getHoles().size(), holes.length); for (Date date : holes) { assertTrue(endsAndHolesPartitionTimeline.getHoles().contains(TimePartition.of(updatePeriod, date))); } TreeSet<TimePartition> partitions = new TreeSet<>(); for (Date dt : TimeRange.iterable(firstPart.getDate(), latestPart.next().getDate(), updatePeriod, 1)) { partitions.add(TimePartition.of(updatePeriod, dt)); } for (Date holeDate : holes) { partitions.remove(TimePartition.of(updatePeriod, holeDate)); } assertEquals(storeAllPartitionTimeline.getAllPartitions(), partitions); }
assertTimeline(client, fact.getName(), storageName, MINUTELY, p, EndsAndHolesPartitionTimeline.class); for (UpdatePeriod up : Arrays.asList(DAILY, HOURLY)) { EndsAndHolesPartitionTimeline timeline = new EndsAndHolesPartitionTimeline(storageTableName, up, p); timeline.setFirst(TimePartition.of(up, times.get(p).get(up).first())); timeline.setLatest(TimePartition.of(up, times.get(p).get(up).last())); assertTimeline(client, fact.getName(), storageName, up, p, timeline);
@Override public boolean exists(TimePartition toCheck) { return !isEmpty() && !toCheck.before(first) && !toCheck.after(latest) && !holes.contains(toCheck); }
/** * Checks in table params if desired implementing class is given. Otherwise by default returns instance of {@link * org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline}. * * @param client * @param storageTable * @param updatePeriod * @param partitionColumn * @return */ public static PartitionTimeline get(CubeMetastoreClient client, String storageTable, UpdatePeriod updatePeriod, String partitionColumn) { try { String storageClassName = client.getTable(storageTable).getParameters().get( MetastoreUtil.getPartitionTimelineStorageClassKey( updatePeriod, partitionColumn)); Class<? extends PartitionTimeline> clz = (Class<? extends PartitionTimeline>) Class.forName(storageClassName); Constructor<? extends PartitionTimeline> constructor = clz.getConstructor( String.class, UpdatePeriod.class, String.class); return constructor.newInstance( storageTable, updatePeriod, partitionColumn); } catch (Exception e) { return new EndsAndHolesPartitionTimeline(storageTable, updatePeriod, partitionColumn); } } }
private void addHolesBetween(TimePartition begin, TimePartition end, UpdatePeriod updatePeriod) throws LensException { for (Date date : TimeRange.iterable(begin.next().getDate(), end.getDate(), updatePeriod, 1)) { addHole(TimePartition.of(updatePeriod, date)); } }
@Override public Map<String, String> toProperties() { HashMap<String, String> ret = Maps.newHashMap(); ret.put("first", ""); ret.put("latest", ""); MetastoreUtil.addNameStrings(ret, "holes", holes); if (isEmpty()) { return ret; } ret.put("first", first.getDateString()); ret.put("latest", latest.getDateString()); return ret; }
@Override public boolean drop(@NonNull TimePartition toDrop) throws LensException { if (first.equals(latest) && first.equals(toDrop)) { this.first = null; this.latest = null; this.holes.clear(); return true; } else if (first.equals(toDrop)) { this.first = this.getNextPartition(first, latest, 1); return true; } else if (latest.equals(toDrop)) { this.latest = this.getNextPartition(latest, first, -1); return true; } else { return addHole(toDrop); } }
@Override public boolean initFromProperties(Map<String, String> properties) throws LensException { first = null; latest = null; holes.clear(); String firstStr = properties.get("first"); String latestStr = properties.get("latest"); String holesStr = MetastoreUtil.getNamedStringValue(properties, "holes"); if (!Strings.isNullOrEmpty(firstStr)) { first = TimePartition.of(getUpdatePeriod(), firstStr); } if (!Strings.isNullOrEmpty(latestStr)) { latest = TimePartition.of(getUpdatePeriod(), latestStr); } holes = Sets.newTreeSet(); if (!Strings.isNullOrEmpty(holesStr)) { for (String hole : holesStr.split("\\s*,\\s*")) { holes.add(TimePartition.of(getUpdatePeriod(), hole)); } } return isConsistent(); }
/** * Checks in table params if desired implementing class is given. Otherwise by default returns instance of {@link * org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline}. * * @param client * @param storageTable * @param updatePeriod * @param partitionColumn * @return */ public static PartitionTimeline get(CubeMetastoreClient client, String storageTable, UpdatePeriod updatePeriod, String partitionColumn) { try { String storageClassName = client.getTable(storageTable).getParameters().get( MetastoreUtil.getPartitionTimelineStorageClassKey( updatePeriod, partitionColumn)); Class<? extends PartitionTimeline> clz = (Class<? extends PartitionTimeline>) Class.forName(storageClassName); Constructor<? extends PartitionTimeline> constructor = clz.getConstructor( String.class, UpdatePeriod.class, String.class); return constructor.newInstance( storageTable, updatePeriod, partitionColumn); } catch (Exception e) { return new EndsAndHolesPartitionTimeline(storageTable, updatePeriod, partitionColumn); } } }
private void addHolesBetween(TimePartition begin, TimePartition end, UpdatePeriod updatePeriod) throws LensException { for (Date date : TimeRange.iterable(begin.next().getDate(), end.getDate(), updatePeriod, 1)) { addHole(TimePartition.of(updatePeriod, date)); } }
@Override public Map<String, String> toProperties() { HashMap<String, String> ret = Maps.newHashMap(); ret.put("first", ""); ret.put("latest", ""); MetastoreUtil.addNameStrings(ret, "holes", holes); if (isEmpty()) { return ret; } ret.put("first", first.getDateString()); ret.put("latest", latest.getDateString()); return ret; }
@Override public boolean add(@NonNull TimePartition partition) throws LensException { if (isEmpty()) { // First partition being added first = partition; latest = partition; return true; } if (partition.before(first)) { addHolesBetween(partition, first, partition.getUpdatePeriod()); first = partition; return true; } else if (partition.after(latest)) { addHolesBetween(latest, partition, partition.getUpdatePeriod()); latest = partition; return true; } else { return holes.remove(partition); } }
@Override public Iterator<TimePartition> iterator() { return new Iterator<TimePartition>() { TimePartition cur = getFirst(); @Override public boolean hasNext() { return cur != null && getLatest() != null && !cur.after(getLatest()); } @Override public TimePartition next() { while (holes.contains(cur)) { cur = cur.next(); } TimePartition toReturn = cur; cur = cur.next(); return toReturn; } @Override public void remove() { throw new UnsupportedOperationException(); } }; } }
@Override public boolean exists(TimePartition toCheck) { return !isEmpty() && !toCheck.before(first) && !toCheck.after(latest) && !holes.contains(toCheck); }