public static void updateVarianceState(VarianceState state, double value) { state.setCount(state.getCount() + 1); double delta = value - state.getMean(); state.setMean(state.getMean() + delta / state.getCount()); state.setM2(state.getM2() + delta * (value - state.getMean())); }
public static void mergeVarianceState(VarianceState state, VarianceState otherState) { long count = otherState.getCount(); double mean = otherState.getMean(); double m2 = otherState.getM2(); checkArgument(count >= 0, "count is negative"); if (count == 0) { return; } long newCount = count + state.getCount(); double newMean = ((count * mean) + (state.getCount() * state.getMean())) / (double) newCount; double delta = mean - state.getMean(); state.setM2(state.getM2() + m2 + delta * delta * count * state.getCount() / (double) newCount); state.setCount(newCount); state.setMean(newMean); }
@Test public void testVarianceStateSerialization() { AccumulatorStateFactory<VarianceState> factory = StateCompiler.generateStateFactory(VarianceState.class); AccumulatorStateSerializer<VarianceState> serializer = StateCompiler.generateStateSerializer(VarianceState.class); VarianceState singleState = factory.createSingleState(); VarianceState deserializedState = factory.createSingleState(); singleState.setMean(1); singleState.setCount(2); singleState.setM2(3); BlockBuilder builder = RowType.anonymous(ImmutableList.of(BIGINT, DOUBLE, DOUBLE)).createBlockBuilder(null, 1); serializer.serialize(singleState, builder); Block block = builder.build(); serializer.deserialize(block, 0, deserializedState); assertEquals(deserializedState.getCount(), singleState.getCount()); assertEquals(deserializedState.getMean(), singleState.getMean()); assertEquals(deserializedState.getM2(), singleState.getM2()); }
public static void updateVarianceState(VarianceState state, double value) { state.setCount(state.getCount() + 1); double delta = value - state.getMean(); state.setMean(state.getMean() + delta / state.getCount()); state.setM2(state.getM2() + delta * (value - state.getMean())); }
public static void mergeVarianceState(VarianceState state, VarianceState otherState) { long count = otherState.getCount(); double mean = otherState.getMean(); double m2 = otherState.getM2(); checkArgument(count >= 0, "count is negative"); if (count == 0) { return; } long newCount = count + state.getCount(); double newMean = ((count * mean) + (state.getCount() * state.getMean())) / (double) newCount; double delta = mean - state.getMean(); double m2Delta = m2 + delta * delta * count * state.getCount() / (double) newCount; state.setM2(state.getM2() + m2Delta); state.setCount(newCount); state.setMean(newMean); }
@Test public void testVarianceStateSerialization() { StateCompiler compiler = new StateCompiler(); AccumulatorStateFactory<VarianceState> factory = compiler.generateStateFactory(VarianceState.class); AccumulatorStateSerializer<VarianceState> serializer = compiler.generateStateSerializer(VarianceState.class); VarianceState singleState = factory.createSingleState(); VarianceState deserializedState = factory.createSingleState(); singleState.setMean(1); singleState.setCount(2); singleState.setM2(3); BlockBuilder builder = VarcharType.VARCHAR.createBlockBuilder(new BlockBuilderStatus(), 1); serializer.serialize(singleState, builder); Block block = builder.build(); serializer.deserialize(block, 0, deserializedState); assertEquals(deserializedState.getCount(), singleState.getCount()); assertEquals(deserializedState.getMean(), singleState.getMean()); assertEquals(deserializedState.getM2(), singleState.getM2()); }