/** * Validates a Flink {@link MemorySize}. The boundaries are inclusive and in bytes. * * <p>The precision defines the allowed minimum unit in bytes (e.g. 1024 would only allow KB). */ public void validateMemorySize(String key, boolean isOptional, int precision, long min, long max) { Preconditions.checkArgument(precision > 0); validateComparable( key, isOptional, min, max, "memory size (in bytes)", (value) -> { final long bytes = MemorySize.parse(value, MemorySize.MemoryUnit.BYTES).getBytes(); if (bytes % precision != 0) { throw new ValidationException( "Memory size for key '" + key + "' must be a multiple of " + precision + " bytes but was: " + value); } return bytes; } ); }
@Test public void testParseMebiBytes() { assertEquals(7657623, MemorySize.parse("7657623m").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623 m").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623mb").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623 mb").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623mebibytes").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623 mebibytes").getMebiBytes()); }
/** * Parses the given string as as MemorySize. * * @param text The string to parse * @return The parsed MemorySize * * @throws IllegalArgumentException Thrown, if the expression cannot be parsed. */ public static MemorySize parse(String text) throws IllegalArgumentException { return new MemorySize(parseBytes(text)); }
@Test public void testParseTebiBytes() { assertEquals(1234567, MemorySize.parse("1234567t").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567 t").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567tb").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567 tb").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567tebibytes").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567 tebibytes").getTebiBytes()); }
/** * Configures how to buffer elements before sending them in bulk to the cluster for efficiency. * * <p>Sets the maximum size of buffered actions per bulk request (using the syntax of {@link MemorySize}). */ public Elasticsearch bulkFlushMaxSize(String maxSize) { internalProperties.putMemorySize(CONNECTOR_BULK_FLUSH_MAX_SIZE, MemorySize.parse(maxSize, MemorySize.MemoryUnit.BYTES)); return this; }
@Test public void testUpperCase() { assertEquals(1L, MemorySize.parse("1 B").getBytes()); assertEquals(1L, MemorySize.parse("1 K").getKibiBytes()); assertEquals(1L, MemorySize.parse("1 M").getMebiBytes()); assertEquals(1L, MemorySize.parse("1 G").getGibiBytes()); assertEquals(1L, MemorySize.parse("1 T").getTebiBytes()); }
@Test public void testParseWithDefaultUnit() { assertEquals(7, MemorySize.parse("7", MEGA_BYTES).getMebiBytes()); assertNotEquals(7, MemorySize.parse("7340032", MEGA_BYTES)); assertEquals(7, MemorySize.parse("7m", MEGA_BYTES).getMebiBytes()); assertEquals(7168, MemorySize.parse("7", MEGA_BYTES).getKibiBytes()); assertEquals(7168, MemorySize.parse("7m", MEGA_BYTES).getKibiBytes()); assertEquals(7, MemorySize.parse("7 m", MEGA_BYTES).getMebiBytes()); assertEquals(7, MemorySize.parse("7mb", MEGA_BYTES).getMebiBytes()); assertEquals(7, MemorySize.parse("7 mb", MEGA_BYTES).getMebiBytes()); assertEquals(7, MemorySize.parse("7mebibytes", MEGA_BYTES).getMebiBytes()); assertEquals(7, MemorySize.parse("7 mebibytes", MEGA_BYTES).getMebiBytes()); }
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) { if (cmd.hasOption(container.getOpt())) { // number of containers is required option! LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt()); } // TODO: The number of task manager should be deprecated soon final int numberTaskManagers; if (cmd.hasOption(container.getOpt())) { numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); } else { numberTaskManagers = 1; } // JobManager Memory final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); // Task Managers memory final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); return new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMB) .setTaskManagerMemoryMB(taskManagerMemoryMB) .setNumberTaskManagers(numberTaskManagers) .setSlotsPerTaskManager(slotsPerTaskManager) .createClusterSpecification(); }
@Test public void testParseKibiBytes() { assertEquals(667766, MemorySize.parse("667766k").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766 k").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766kb").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766 kb").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766kibibytes").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766 kibibytes").getKibiBytes()); }
@Test public void testParseGibiBytes() { assertEquals(987654, MemorySize.parse("987654g").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654 g").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654gb").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654 gb").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654gibibytes").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654 gibibytes").getGibiBytes()); }
@Test public void testStandardUtils() throws IOException { final MemorySize size = new MemorySize(1234567890L); final MemorySize cloned = CommonTestUtils.createCopySerializable(size); assertEquals(size, cloned); assertEquals(size.hashCode(), cloned.hashCode()); assertEquals(size.toString(), cloned.toString()); }
@Test(expected = IllegalArgumentException.class) public void testParseNumberTimeUnitOverflow() { MemorySize.parseBytes("100000000000000 tb"); }
/** * Adds a Flink {@link MemorySize} under the given key. */ public void putMemorySize(String key, MemorySize size) { checkNotNull(key); checkNotNull(size); put(key, size.toString()); }
@Test(expected = IllegalArgumentException.class) public void testInvalid() { new MemorySize(-1); }
/** * Parses the given string with a default unit. * * @param text The string to parse. * @param defaultUnit specify the default unit. * @return The parsed MemorySize. * * @throws IllegalArgumentException Thrown, if the expression cannot be parsed. */ public static MemorySize parse(String text, MemoryUnit defaultUnit) throws IllegalArgumentException { if (!hasUnit(text)) { return parse(text + defaultUnit.getUnits()[0]); } return parse(text); }
public static ClusterSpecification fromConfiguration(Configuration configuration) { int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); return new ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(1) .setSlotsPerTaskManager(slots) .createClusterSpecification(); }
@Test(expected = IllegalArgumentException.class) public void testParseNumberOverflow() { MemorySize.parseBytes("100000000000000000000000000000000 bytes"); }
/** * Adds a Flink {@link MemorySize} under the given key. */ public void putMemorySize(String key, MemorySize size) { checkNotNull(key); checkNotNull(size); put(key, size.toString()); }
.ifPresent(v -> builder.setBulkFlushMaxSizeMb(MemorySize.parse(v).getMebiBytes()));