if (v.getIGNORE()) { seenIgnore = true; if (log.isDebugEnabled())
public static Tuple3<Long,Boolean,List<String>> parseUids(Key k, Value v) throws IOException { final String dataType = parseDataType(k); Uid.List docIds = Uid.List.parseFrom(v.get()); final boolean ignore = docIds.getIGNORE(); List<String> uids = ignore || docIds.getUIDList() == null ? Collections.emptyList() : Lists.transform(docIds.getUIDList(), s -> dataType + "\u0000" + s.trim()); return Tuples.tuple(docIds.getCOUNT(), ignore, uids); }
/** * Determine if we have events. For this to be true * * @param v * @return */ private boolean hasEvents(final Value v) { try { Uid.List protobuf = Uid.List.parseFrom(v.get()); // the protobuf list should be aggregated already return protobuf.getIGNORE() || !protobuf.getUIDList().isEmpty(); } catch (InvalidProtocolBufferException e) { // if we cannot parse the protocol buffer, then we // won't be able to use it for evaluation anyway log.error(e); return false; } }
if (uidList.getIGNORE()) { if (log.isDebugEnabled()) { log.debug("Adding range for the shard & datatype");
return this; if (other.hasIGNORE()) { setIGNORE(other.getIGNORE());
@Test public void testEqualsMax() throws Exception { agg.reset(); List<String> savedUUIDs = new ArrayList<>(); Collection<Value> values = Lists.newArrayList(); for (int i = 0; i < GlobalIndexUidAggregator.MAX; i++) { Builder b = createNewUidList(); b.setIGNORE(false); String uuid = UUID.randomUUID().toString(); savedUUIDs.add(uuid); b.setCOUNT(1); b.addUID(uuid); Uid.List uidList = b.build(); Value val = new Value(uidList.toByteArray()); values.add(val); } Value result = agg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); assertNotNull(resultList); assertEquals(false, resultList.getIGNORE()); assertEquals(resultList.getUIDCount(), (GlobalIndexUidAggregator.MAX)); List<String> resultListUUIDs = resultList.getUIDList(); for (String s : savedUUIDs) assertTrue(resultListUUIDs.contains(s)); }
@Test public void testLessThanMax() throws Exception { agg.reset(); List<String> savedUUIDs = new ArrayList<>(); Collection<Value> values = Lists.newArrayList(); for (int i = 0; i < GlobalIndexUidAggregator.MAX - 1; i++) { Builder b = createNewUidList(); b.setIGNORE(false); String uuid = UUID.randomUUID().toString(); savedUUIDs.add(uuid); b.setCOUNT(1); b.addUID(uuid); Uid.List uidList = b.build(); Value val = new Value(uidList.toByteArray()); values.add(val); } Value result = agg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); assertEquals(false, resultList.getIGNORE()); assertEquals(resultList.getUIDCount(), (GlobalIndexUidAggregator.MAX - 1)); List<String> resultListUUIDs = resultList.getUIDList(); for (String s : savedUUIDs) assertTrue(resultListUUIDs.contains(s)); }
@Test public void testSeenIgnore() throws Exception { Logger.getRootLogger().setLevel(Level.ALL); agg.reset(); Builder b = createNewUidList(); b.setIGNORE(true); b.setCOUNT(0); Uid.List uidList = b.build(); Collection<Value> values = Lists.newArrayList(); Value val = new Value(uidList.toByteArray()); values.add(val); b = createNewUidList(); b.setIGNORE(false); b.setCOUNT(1); b.addUID(UUID.randomUUID().toString()); uidList = b.build(); val = new Value(uidList.toByteArray()); values.add(val); Value result = agg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); assertEquals(true, resultList.getIGNORE()); assertEquals(0, resultList.getUIDCount()); assertEquals(1, resultList.getCOUNT()); }
@Test public void testCount() throws Exception { agg.reset(); UUID uuid = UUID.randomUUID(); // Collect the same UUID five times. Collection<Value> values = Lists.newArrayList(); for (int i = 0; i < 5; i++) { Builder b = createNewUidList(); b.setCOUNT(1); b.setIGNORE(false); b.addUID(uuid.toString()); Uid.List uidList = b.build(); Value val = new Value(uidList.toByteArray()); values.add(val); } Value result = agg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); assertEquals(5, resultList.getCOUNT()); assertEquals(false, resultList.getIGNORE()); assertEquals(1, resultList.getUIDCount()); } }
@Test public void testMoreThanMax() throws Exception { agg.reset(); List<String> savedUUIDs = new ArrayList<>(); Collection<Value> values = Lists.newArrayList(); for (int i = 0; i < GlobalIndexUidAggregator.MAX + 10; i++) { Builder b = createNewUidList(); b.setIGNORE(false); String uuid = UUID.randomUUID().toString(); savedUUIDs.add(uuid); b.setCOUNT(1); b.addUID(uuid); Uid.List uidList = b.build(); Value val = new Value(uidList.toByteArray()); values.add(val); } Value result = agg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); assertEquals(true, resultList.getIGNORE()); assertEquals(0, resultList.getUIDCount()); assertEquals(resultList.getCOUNT(), (GlobalIndexUidAggregator.MAX + 10)); }
@Test public void testInvalidValueType() throws Exception { Logger log = Logger.getLogger(GlobalIndexUidAggregator.class); Level origLevel = log.getLevel(); log.setLevel(Level.FATAL); Collection<Value> values = Lists.newArrayList(); agg.reset(); Value val = new Value(UUID.randomUUID().toString().getBytes()); values.add(val); Value result = agg.reduce(new Key("key"), values.iterator()); Uid.List resultList = Uid.List.parseFrom(result.get()); assertEquals(false, resultList.getIGNORE()); assertEquals(0, resultList.getUIDCount()); assertEquals(0, resultList.getCOUNT()); log.setLevel(origLevel); }
public static Tuple3<Long,Boolean,List<String>> parseUids(Key k, Value v) throws IOException { final String dataType = parseDataType(k); Uid.List docIds = Uid.List.parseFrom(v.get()); final boolean ignore = docIds.getIGNORE(); List<String> uids = ignore || docIds.getUIDList() == null ? Collections.emptyList() : Lists.transform(docIds.getUIDList(), s -> dataType + "\u0000" + s.trim()); return Tuples.tuple(docIds.getCOUNT(), ignore, uids); }