public DataSize getMaxInitialSplitSize() { if (maxInitialSplitSize == null) { return new DataSize(maxSplitSize.getValue() / 2, maxSplitSize.getUnit()); } return maxInitialSplitSize; }
protected Map<String, Long> execute(TaskContext taskContext) { List<Driver> drivers = createDrivers(taskContext); long peakMemory = 0; boolean done = false; while (!done) { boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); long lastPeakMemory = peakMemory; peakMemory = (long) taskContext.getTaskStats().getUserMemoryReservation().getValue(BYTE); if (peakMemory <= lastPeakMemory) { peakMemory = lastPeakMemory; } processed = true; } } done = !processed; } return ImmutableMap.of("peak_memory", peakMemory); }
if (alwaysRevokeMemory) { driver.getDriverContext().getOperatorContexts().stream() .filter(operatorContext -> operatorContext.getOperatorStats().getRevocableMemoryReservation().getValue() > 0) .forEach(OperatorContext::requestMemoryRevoking);
long outputBytes = taskStats.getOutputDataSize().toBytes(); double inputMegaBytes = new DataSize(inputBytes, BYTE).getValue(MEGABYTE);
@Override public int hashCode() { double value = getValue(Unit.BYTE); return Double.hashCode(value); }
public long roundTo(Unit unit) { double rounded = Math.floor(getValue(unit) + 0.5d); checkArgument(rounded <= Long.MAX_VALUE, "size is too large to be represented in requested unit as a long"); return (long) rounded; }
public long toBytes() { double bytes = getValue(Unit.BYTE); checkState(bytes <= Long.MAX_VALUE, "size is too large to be represented in bytes as a long"); return (long) bytes; }
public DataSize convertTo(Unit unit) { requireNonNull(unit, "unit is null"); return new DataSize(getValue(unit), unit); }
public DataSize convertToMostSuccinctDataSize() { Unit unitToUse = Unit.BYTE; for (Unit unitToTest : DATASIZE_UNITS) { if (getValue(unitToTest) >= 1.0) { unitToUse = unitToTest; } else { break; } } return convertTo(unitToUse); }
public DataSize getMaxInitialSplitSize() { if (maxInitialSplitSize == null) { return new DataSize(maxSplitSize.getValue() / 2, maxSplitSize.getUnit()); } return maxInitialSplitSize; }
public DataSize getMaxInitialSplitSize() { if (maxInitialSplitSize == null) { return new DataSize(maxSplitSize.getValue() / 2, maxSplitSize.getUnit()); } return maxInitialSplitSize; }
@Test(dataProvider = "conversions") public void testConversions(DataSize.Unit unit, DataSize.Unit toUnit, double factor) { DataSize size = new DataSize(1, unit).convertTo(toUnit); assertEquals(size.getUnit(), toUnit); assertEquals(size.getValue(), factor); assertEquals(size.getValue(toUnit), factor); }
@Test(dataProvider = "parseableValues") public void testValueOf(String string, double expectedValue, DataSize.Unit expectedUnit) { DataSize size = DataSize.valueOf(string); assertEquals(size.getUnit(), expectedUnit); assertEquals(size.getValue(), expectedValue); }
protected Map<String, Long> execute(TaskContext taskContext) { List<Driver> drivers = createDrivers(taskContext); long peakMemory = 0; boolean done = false; while (!done) { boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); long lastPeakMemory = peakMemory; peakMemory = (long) taskContext.getTaskStats().getUserMemoryReservation().getValue(BYTE); if (peakMemory <= lastPeakMemory) { peakMemory = lastPeakMemory; } processed = true; } } done = !processed; } return ImmutableMap.of("peak_memory", peakMemory); }
protected Map<String, Long> execute(TaskContext taskContext) { List<Driver> drivers = createDrivers(taskContext); long peakMemory = 0; boolean done = false; while (!done) { boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); long lastPeakMemory = peakMemory; peakMemory = (long) taskContext.getTaskStats().getUserMemoryReservation().getValue(BYTE); if (peakMemory <= lastPeakMemory) { peakMemory = lastPeakMemory; } processed = true; } } done = !processed; } return ImmutableMap.of("peak_memory", peakMemory); }
protected Map<String, Long> execute(TaskContext taskContext) { List<Driver> drivers = createDrivers(taskContext); long peakMemory = 0; boolean done = false; while (!done) { boolean processed = false; for (Driver driver : drivers) { if (!driver.isFinished()) { driver.process(); long lastPeakMemory = peakMemory; peakMemory = (long) taskContext.getTaskStats().getUserMemoryReservation().getValue(BYTE); if (peakMemory <= lastPeakMemory) { peakMemory = lastPeakMemory; } processed = true; } } done = !processed; } return ImmutableMap.of("peak_memory", peakMemory); }
@Test(dataProvider = "conversions") public void testConvertToMostSuccinctDataSize(DataSize.Unit unit, DataSize.Unit toUnit, double factor) { DataSize size = new DataSize(factor, toUnit); DataSize actual = size.convertToMostSuccinctDataSize(); assertThat(actual).isEqualTo(new DataSize(1, unit)); assertThat(actual.getValue(unit)).isCloseTo(1.0, offset(0.001)); assertThat(actual.getUnit()).isEqualTo(unit); }
private static void assertJsonRoundTrip(DataSize dataSize) { JsonCodec<DataSize> dataSizeCodec = JsonCodec.jsonCodec(DataSize.class); String json = dataSizeCodec.toJson(dataSize); DataSize dataSizeCopy = dataSizeCodec.fromJson(json); assertThat(dataSizeCopy.getValue(BYTE)) .isCloseTo(dataSize.getValue(BYTE), withPercentage(1)); }
@Description("Converts a string representing data size in airlift's DataSize format to a double representing size in the specified size unit") @ScalarFunction("from_datasize") @SqlType(StandardTypes.DOUBLE) public static double fromDataSize(@SqlType(StandardTypes.VARCHAR) Slice size, @SqlType(StandardTypes.VARCHAR) Slice sizeUnit) { String sizeUnitString = sizeUnit.toStringUtf8(); DataSize dataSize = DataSize.valueOf(size.toStringUtf8()); for (DataSize.Unit unit : DataSize.Unit.values()) { if (unit.getUnitString().equals(sizeUnitString)) { return dataSize.getValue(unit); } } throw new IllegalArgumentException("Unknown unit: " + sizeUnitString); } }