/** Estimates a size of the mutation group in bytes. */ public static long sizeOf(MutationGroup group) { long result = 0; for (Mutation m : group) { result += sizeOf(m); } return result; }
/** Estimates a size of mutation in bytes. */ static long sizeOf(Mutation m) { if (m.getOperation() == Mutation.Op.DELETE) { return sizeOf(m.getKeySet()); } long result = 0; for (Value v : m.getValues()) { switch (v.getType().getCode()) { case ARRAY: result += estimateArrayValue(v); break; case STRUCT: throw new IllegalArgumentException("Structs are not supported in mutation."); default: result += estimatePrimitiveValue(v); } } return result; }
MutationGroup mg = decode(kv.getValue()); long groupSize = MutationSizeEstimator.sizeOf(mg); long groupCells = MutationCellCounter.countOf(spannerSchema, mg);
@Test public void primitives() throws Exception { Mutation int64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(1).build(); Mutation float64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(2.9).build(); Mutation bool = Mutation.newInsertOrUpdateBuilder("test").set("one").to(false).build(); assertThat(MutationSizeEstimator.sizeOf(int64), is(8L)); assertThat(MutationSizeEstimator.sizeOf(float64), is(8L)); assertThat(MutationSizeEstimator.sizeOf(bool), is(1L)); }
@Test public void nullPrimitiveArrays() throws Exception { Mutation int64 = Mutation.newInsertOrUpdateBuilder("test").set("one").toInt64Array((long[]) null).build(); Mutation float64 = Mutation.newInsertOrUpdateBuilder("test") .set("one") .toFloat64Array((double[]) null) .build(); Mutation bool = Mutation.newInsertOrUpdateBuilder("test").set("one").toBoolArray((boolean[]) null).build(); assertThat(MutationSizeEstimator.sizeOf(int64), is(0L)); assertThat(MutationSizeEstimator.sizeOf(float64), is(0L)); assertThat(MutationSizeEstimator.sizeOf(bool), is(0L)); }
@ProcessElement public void processElement(ProcessContext c) throws Exception { SpannerSchema spannerSchema = c.sideInput(schemaView); MutationKeyEncoder encoder = new MutationKeyEncoder(spannerSchema); MutationGroup mg = c.element(); long groupSize = MutationSizeEstimator.sizeOf(mg); long groupCells = MutationCellCounter.countOf(spannerSchema, mg); synchronized (this) { if (((batchCells + groupCells) > maxNumMutations) || (batchSizeBytes + groupSize) > maxBatchSizeBytes) { c.output(sortAndGetList()); initSorter(); } mutationsToSort.add(KV.of(encoder.encodeTableNameAndKey(mg.primary()), encode(mg))); batchSizeBytes += groupSize; batchCells += groupCells; } } }
@DoFn.ProcessElement public void processElement(ProcessContext c) { MutationGroup mg = c.element(); if (mg.primary().getOperation() == Op.DELETE && !isPointDelete(mg.primary())) { // Ranged deletes are not batchable. c.output(unbatchableMutationsTag, Arrays.asList(mg)); unBatchableMutationGroupsCounter.inc(); return; } SpannerSchema spannerSchema = c.sideInput(schemaView); long groupSize = MutationSizeEstimator.sizeOf(mg); long groupCells = MutationCellCounter.countOf(spannerSchema, mg); if (groupSize >= batchSizeBytes || groupCells >= maxNumMutations) { c.output(unbatchableMutationsTag, Arrays.asList(mg)); unBatchableMutationGroupsCounter.inc(); } else { c.output(mg); batchableMutationGroupsCounter.inc(); } } }
@Test public void strings() throws Exception { Mutation emptyString = Mutation.newInsertOrUpdateBuilder("test").set("one").to("").build(); Mutation nullString = Mutation.newInsertOrUpdateBuilder("test").set("one").to((String) null).build(); Mutation sampleString = Mutation.newInsertOrUpdateBuilder("test").set("one").to("abc").build(); Mutation sampleArray = Mutation.newInsertOrUpdateBuilder("test") .set("one") .toStringArray(Arrays.asList("one", "two", null)) .build(); Mutation nullArray = Mutation.newInsertOrUpdateBuilder("test").set("one").toStringArray(null).build(); assertThat(MutationSizeEstimator.sizeOf(emptyString), is(0L)); assertThat(MutationSizeEstimator.sizeOf(nullString), is(0L)); assertThat(MutationSizeEstimator.sizeOf(sampleString), is(3L)); assertThat(MutationSizeEstimator.sizeOf(sampleArray), is(6L)); assertThat(MutationSizeEstimator.sizeOf(nullArray), is(0L)); }
@Test public void bytes() throws Exception { Mutation empty = Mutation.newInsertOrUpdateBuilder("test").set("one").to(ByteArray.fromBase64("")).build(); Mutation nullValue = Mutation.newInsertOrUpdateBuilder("test").set("one").to((ByteArray) null).build(); Mutation sample = Mutation.newInsertOrUpdateBuilder("test") .set("one") .to(ByteArray.fromBase64("abcdabcd")) .build(); Mutation nullArray = Mutation.newInsertOrUpdateBuilder("test").set("one").toBytesArray(null).build(); assertThat(MutationSizeEstimator.sizeOf(empty), is(0L)); assertThat(MutationSizeEstimator.sizeOf(nullValue), is(0L)); assertThat(MutationSizeEstimator.sizeOf(sample), is(6L)); assertThat(MutationSizeEstimator.sizeOf(nullArray), is(0L)); }
@Test public void primitiveArrays() throws Exception { Mutation int64 = Mutation.newInsertOrUpdateBuilder("test") .set("one") .toInt64Array(new long[] {1L, 2L, 3L}) .build(); Mutation float64 = Mutation.newInsertOrUpdateBuilder("test") .set("one") .toFloat64Array(new double[] {1., 2.}) .build(); Mutation bool = Mutation.newInsertOrUpdateBuilder("test") .set("one") .toBoolArray(new boolean[] {true, true, false, true}) .build(); assertThat(MutationSizeEstimator.sizeOf(int64), is(24L)); assertThat(MutationSizeEstimator.sizeOf(float64), is(16L)); assertThat(MutationSizeEstimator.sizeOf(bool), is(4L)); }
Mutation.newInsertOrUpdateBuilder("test").set("one").toDateArray(null).build(); assertThat(MutationSizeEstimator.sizeOf(timestamp), is(12L)); assertThat(MutationSizeEstimator.sizeOf(date), is(12L)); assertThat(MutationSizeEstimator.sizeOf(nullTimestamp), is(12L)); assertThat(MutationSizeEstimator.sizeOf(nullDate), is(12L)); assertThat(MutationSizeEstimator.sizeOf(timestampArray), is(24L)); assertThat(MutationSizeEstimator.sizeOf(dateArray), is(48L)); assertThat(MutationSizeEstimator.sizeOf(nullTimestampArray), is(0L)); assertThat(MutationSizeEstimator.sizeOf(nullDateArray), is(0L));
@Test public void testBatchFn_size() throws Exception { long mutationSize = MutationSizeEstimator.sizeOf(m(1L));
@Test public void group() throws Exception { Mutation int64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(1).build(); Mutation float64 = Mutation.newInsertOrUpdateBuilder("test").set("one").to(2.9).build(); Mutation bool = Mutation.newInsertOrUpdateBuilder("test").set("one").to(false).build(); MutationGroup group = MutationGroup.create(int64, float64, bool); assertThat(MutationSizeEstimator.sizeOf(group), is(17L)); } }
}; long mutationSize = MutationSizeEstimator.sizeOf(m(1L)); BatchableMutationFilterFn testFn = new BatchableMutationFilterFn(null, null, mutationSize * 3, 1000);