@Override public Coder<Iterable<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) { return IterableCoder.of(inputCoder); }
@Override public IterableCoder<?> fromComponents(List<Coder<?>> components) { return IterableCoder.of(components.get(0)); } };
@Override public IterableCoder fromCloudObject(CloudObject object) { List<Coder<?>> components = getComponents(object); checkArgument(components.size() == 1, "Expecting 1 component, got %s", components.size()); return IterableCoder.of(components.get(0)); }
private KeyedWorkItemCoder( Coder<K> keyCoder, Coder<ElemT> elemCoder, Coder<? extends BoundedWindow> windowCoder) { this.keyCoder = keyCoder; this.elemCoder = elemCoder; this.windowCoder = windowCoder; this.timersCoder = IterableCoder.of(TimerDataCoder.of(windowCoder)); this.elemsCoder = IterableCoder.of(FullWindowedValueCoder.of(elemCoder, windowCoder)); }
public ValuesSource(Iterable<T> values, Coder<T> coder) { this.iterableCoder = IterableCoder.of(coder); ByteArrayOutputStream bos = new ByteArrayOutputStream(); try { iterableCoder.encode(values, bos, Context.OUTER); } catch (IOException ex) { throw new RuntimeException(ex); } this.codedValues = bos.toByteArray(); }
private static <K, InputT> FullWindowedValueCoder<KV<K, Iterable<InputT>>> windowedValueKeyValueCoderOf( final Coder<K> keyCoder, final Coder<InputT> iCoder, final Coder<? extends BoundedWindow> wCoder) { return FullWindowedValueCoder.of(KvCoder.of(keyCoder, IterableCoder.of(iCoder)), wCoder); }
private PCollectionSingletonIterableAssert<T> satisfies( AssertRelation<Iterable<T>, Iterable<T>> relation, Iterable<T> expectedElements) { return satisfies( new CheckRelationAgainstExpected<>( relation, expectedElements, IterableCoder.of(elementCoder))); } }
/** * Applies a {@link SerializableFunction} to check the elements of the {@code Iterable}. * * <p>Returns this {@code IterableAssert}. */ private PCollectionContentsAssert<T> satisfies( AssertRelation<Iterable<T>, Iterable<T>> relation, Iterable<T> expectedElements) { return satisfies( new CheckRelationAgainstExpected<>( relation, expectedElements, IterableCoder.of(actual.getCoder()))); }
@Override public PCollection<KV<K, Iterable<V>>> expand(PCollection<KeyedWorkItem<K, V>> input) { KeyedWorkItemCoder<K, V> inputCoder = getKeyedWorkItemCoder(input.getCoder()); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), outputWindowingStrategy, input.isBounded(), KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getElementCoder()))); } }
@Test public void testSerializableCoder() throws Exception { IterableCoder<MyRecord> coder = IterableCoder.of(SerializableCoder.of(MyRecord.class)); List<MyRecord> records = new ArrayList<>(); for (String l : LINES) { records.add(new MyRecord(l)); } byte[] encoded = CoderUtils.encodeToByteArray(coder, records); Iterable<MyRecord> decoded = CoderUtils.decodeFromByteArray(coder, encoded); assertEquals(records, decoded); }
/** * Tests that {@link MutationDetectors#forValueWithCoder} does not false positive on a {@link * LinkedList} coded as an {@link Iterable}. */ @Test public void testImmutableList() throws Exception { List<Integer> value = Lists.newLinkedList(Arrays.asList(1, 2, 3, 4)); MutationDetector detector = MutationDetectors.forValueWithCoder(value, IterableCoder.of(VarIntCoder.of())); detector.verifyUnmodified(); }
/** * Tests that {@link MutationDetectors#forValueWithCoder} does not false positive on a {@link Set} * coded as an {@link Iterable}. */ @Test public void testStructuralValue() throws Exception { Set<Integer> value = Sets.newHashSet(Arrays.asList(1, 2, 3, 4)); MutationDetector detector = MutationDetectors.forValueWithCoder(value, IterableCoder.of(VarIntCoder.of())); detector.verifyUnmodified(); }
@Test public void noInputUnkeyedOutput() { PCollection<KV<Integer, Iterable<Void>>> unkeyed = p.apply( Create.of(KV.<Integer, Iterable<Void>>of(-1, Collections.emptyList())) .withCoder(KvCoder.of(VarIntCoder.of(), IterableCoder.of(VoidCoder.of())))); p.traverseTopologically(visitor); assertThat(visitor.getKeyedPValues(), not(hasItem(unkeyed))); }
/** * Tests that {@link MutationDetectors#forValueWithCoder} does not false positive on an {@link * Iterable} that is not known to be bounded; after coder-based cloning the bound will be known * and it will be a {@link List} so it will encode more compactly the second time around. */ @Test public void testImmutableIterable() throws Exception { Iterable<Integer> value = FluentIterable.from(Arrays.asList(1, 2, 3, 4)).cycle().limit(50); MutationDetector detector = MutationDetectors.forValueWithCoder(value, IterableCoder.of(VarIntCoder.of())); detector.verifyUnmodified(); }
@Override public PDone expand(PCollection<Iterable<T>> input) { input .apply("GroupGlobally", new GroupGlobally<>(rewindowingStrategy)) .apply("GetPane", MapElements.via(paneExtractor)) .setCoder(IterableCoder.of(input.getCoder())) .apply("RunChecks", ParDo.of(new SingletonCheckerDoFn<>(checkerFn, site))) .apply("VerifyAssertions", new DefaultConcludeTransform()); return PDone.in(input.getPipeline()); } }
@Override public PDone expand(PCollection<T> input) { input .apply("GroupGlobally", new GroupGlobally<>(rewindowingStrategy)) .apply("GetPane", MapElements.via(paneExtractor)) .setCoder(IterableCoder.of(input.getCoder())) .apply("RunChecks", ParDo.of(new GroupedValuesCheckerDoFn<>(checkerFn, site))) .apply("VerifyAssertions", new DefaultConcludeTransform()); return PDone.in(input.getPipeline()); } }
/** Tests that when writing to a non-existent table, the write fails. */ @Test public void testTableCheckIgnoredWhenCanNotAccessConfig() throws Exception { PCollection<KV<ByteString, Iterable<Mutation>>> emptyInput = p.apply( Create.empty( KvCoder.of(ByteStringCoder.of(), IterableCoder.of(ProtoCoder.of(Mutation.class))))); emptyInput.apply("write", defaultWrite.withTableId(NOT_ACCESSIBLE_VALUE)); p.run(); }
@Test public void kvCoderComponentsToConstructor() throws IOException { KvCoder<byte[], Iterable<Long>> javaCoder = KvCoder.of(ByteArrayCoder.of(), IterableCoder.of(LengthPrefixCoder.of(VarLongCoder.of()))); MessageWithComponents coderAndComponents = CoderTranslation.toProto(javaCoder); KvCoderComponents kvCoderComponents = ModelCoders.getKvCoderComponents(coderAndComponents.getCoder()); Coder kvCoder = ModelCoders.kvCoder(kvCoderComponents.keyCoderId(), kvCoderComponents.valueCoderId()); assertThat(kvCoder, equalTo(coderAndComponents.getCoder())); }
@Test @Category(ValidatesRunner.class) public void testFlattenIterables() { PCollection<Iterable<String>> input = p.apply( Create.<Iterable<String>>of(LINES).withCoder(IterableCoder.of(StringUtf8Coder.of()))); PCollection<String> output = input.apply(Flatten.iterables()); PAssert.that(output).containsInAnyOrder(LINES_ARRAY); p.run(); }
@Test @Category(ValidatesRunner.class) public void testFlattenIterablesEmpty() { PCollection<Iterable<String>> input = p.apply( Create.<Iterable<String>>of(NO_LINES) .withCoder(IterableCoder.of(StringUtf8Coder.of()))); PCollection<String> output = input.apply(Flatten.iterables()); PAssert.that(output).containsInAnyOrder(NO_LINES_ARRAY); p.run(); }