/** * Configures how to buffer elements before sending them in bulk to the cluster for efficiency. * * <p>Sets the maximum size of buffered actions per bulk request (using the syntax of {@link MemorySize}). */ public Elasticsearch bulkFlushMaxSize(String maxSize) { internalProperties.putMemorySize(CONNECTOR_BULK_FLUSH_MAX_SIZE, MemorySize.parse(maxSize, MemorySize.MemoryUnit.BYTES)); return this; }
/** * Parses the given string with a default unit. * * @param text The string to parse. * @param defaultUnit specify the default unit. * @return The parsed MemorySize. * * @throws IllegalArgumentException Thrown, if the expression cannot be parsed. */ public static MemorySize parse(String text, MemoryUnit defaultUnit) throws IllegalArgumentException { if (!hasUnit(text)) { return parse(text + defaultUnit.getUnits()[0]); } return parse(text); }
/** * Returns a Flink {@link MemorySize} under the given key if it exists. */ public Optional<MemorySize> getOptionalMemorySize(String key) { return optionalGet(key).map((value) -> { try { return MemorySize.parse(value, MemorySize.MemoryUnit.BYTES); } catch (Exception e) { throw new ValidationException("Invalid memory size value for key '" + key + "'.", e); } }); }
/** * Get job manager's heap memory. This method will check the new key * {@link JobManagerOptions#JOB_MANAGER_HEAP_MEMORY} and * the old key {@link JobManagerOptions#JOB_MANAGER_HEAP_MEMORY_MB} for backwards compatibility. * * @param configuration the configuration object * @return the memory size of job manager's heap memory. */ public static MemorySize getJobManagerHeapMemory(Configuration configuration) { if (configuration.containsKey(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY.key())) { return MemorySize.parse(configuration.getString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY)); } else if (configuration.containsKey(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB.key())) { return MemorySize.parse(configuration.getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB) + "m"); } else { //use default value return MemorySize.parse(configuration.getString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY)); } }
/** * Get task manager's heap memory. This method will check the new key * {@link TaskManagerOptions#TASK_MANAGER_HEAP_MEMORY} and * the old key {@link TaskManagerOptions#TASK_MANAGER_HEAP_MEMORY_MB} for backwards compatibility. * * @param configuration the configuration object * @return the memory size of task manager's heap memory. */ public static MemorySize getTaskManagerHeapMemory(Configuration configuration) { if (configuration.containsKey(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY.key())) { return MemorySize.parse(configuration.getString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY)); } else if (configuration.containsKey(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB.key())) { return MemorySize.parse(configuration.getInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB) + "m"); } else { //use default value return MemorySize.parse(configuration.getString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY)); } }
.ifPresent(v -> builder.setBulkFlushMaxSizeMb(MemorySize.parse(v).getMebiBytes()));
@Test public void testParseTebiBytes() { assertEquals(1234567, MemorySize.parse("1234567t").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567 t").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567tb").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567 tb").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567tebibytes").getTebiBytes()); assertEquals(1234567, MemorySize.parse("1234567 tebibytes").getTebiBytes()); }
@Test public void testParseKibiBytes() { assertEquals(667766, MemorySize.parse("667766k").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766 k").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766kb").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766 kb").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766kibibytes").getKibiBytes()); assertEquals(667766, MemorySize.parse("667766 kibibytes").getKibiBytes()); }
@Test public void testParseMebiBytes() { assertEquals(7657623, MemorySize.parse("7657623m").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623 m").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623mb").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623 mb").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623mebibytes").getMebiBytes()); assertEquals(7657623, MemorySize.parse("7657623 mebibytes").getMebiBytes()); }
@Test public void testParseWithDefaultUnit() { assertEquals(7, MemorySize.parse("7", MEGA_BYTES).getMebiBytes()); assertNotEquals(7, MemorySize.parse("7340032", MEGA_BYTES)); assertEquals(7, MemorySize.parse("7m", MEGA_BYTES).getMebiBytes()); assertEquals(7168, MemorySize.parse("7", MEGA_BYTES).getKibiBytes()); assertEquals(7168, MemorySize.parse("7m", MEGA_BYTES).getKibiBytes()); assertEquals(7, MemorySize.parse("7 m", MEGA_BYTES).getMebiBytes()); assertEquals(7, MemorySize.parse("7mb", MEGA_BYTES).getMebiBytes()); assertEquals(7, MemorySize.parse("7 mb", MEGA_BYTES).getMebiBytes()); assertEquals(7, MemorySize.parse("7mebibytes", MEGA_BYTES).getMebiBytes()); assertEquals(7, MemorySize.parse("7 mebibytes", MEGA_BYTES).getMebiBytes()); }
/** * Validates a Flink {@link MemorySize}. The boundaries are inclusive and in bytes. * * <p>The precision defines the allowed minimum unit in bytes (e.g. 1024 would only allow KB). */ public void validateMemorySize(String key, boolean isOptional, int precision, long min, long max) { Preconditions.checkArgument(precision > 0); validateComparable( key, isOptional, min, max, "memory size (in bytes)", (value) -> { final long bytes = MemorySize.parse(value, MemorySize.MemoryUnit.BYTES).getBytes(); if (bytes % precision != 0) { throw new ValidationException( "Memory size for key '" + key + "' must be a multiple of " + precision + " bytes but was: " + value); } return bytes; } ); }
@Test public void testParseGibiBytes() { assertEquals(987654, MemorySize.parse("987654g").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654 g").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654gb").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654 gb").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654gibibytes").getGibiBytes()); assertEquals(987654, MemorySize.parse("987654 gibibytes").getGibiBytes()); }
@Test public void testUpperCase() { assertEquals(1L, MemorySize.parse("1 B").getBytes()); assertEquals(1L, MemorySize.parse("1 K").getKibiBytes()); assertEquals(1L, MemorySize.parse("1 M").getMebiBytes()); assertEquals(1L, MemorySize.parse("1 G").getGibiBytes()); assertEquals(1L, MemorySize.parse("1 T").getTebiBytes()); }
MemorySize.parse(config.getString(TaskManagerOptions.MEMORY_SEGMENT_SIZE)) .getBytes());
/** * Configures how to buffer elements before sending them in bulk to the cluster for efficiency. * * <p>Sets the maximum size of buffered actions per bulk request (using the syntax of {@link MemorySize}). */ public Elasticsearch bulkFlushMaxSize(String maxSize) { internalProperties.putMemorySize(CONNECTOR_BULK_FLUSH_MAX_SIZE, MemorySize.parse(maxSize, MemorySize.MemoryUnit.BYTES)); return this; }
/** * Configures how to buffer elements before sending them in bulk to the cluster for efficiency. * * <p>Sets the maximum size of buffered actions per bulk request (using the syntax of {@link MemorySize}). */ public Elasticsearch bulkFlushMaxSize(String maxSize) { internalProperties.putMemorySize(CONNECTOR_BULK_FLUSH_MAX_SIZE, MemorySize.parse(maxSize, MemorySize.MemoryUnit.BYTES)); return this; }
/** * Parses the given string with a default unit. * * @param text The string to parse. * @param defaultUnit specify the default unit. * @return The parsed MemorySize. * * @throws IllegalArgumentException Thrown, if the expression cannot be parsed. */ public static MemorySize parse(String text, MemoryUnit defaultUnit) throws IllegalArgumentException { if (!hasUnit(text)) { return parse(text + defaultUnit.getUnits()[0]); } return parse(text); }
/** * Parses the given string with a default unit. * * @param text The string to parse. * @param defaultUnit specify the default unit. * @return The parsed MemorySize. * * @throws IllegalArgumentException Thrown, if the expression cannot be parsed. */ public static MemorySize parse(String text, MemoryUnit defaultUnit) throws IllegalArgumentException { if (!hasUnit(text)) { return parse(text + defaultUnit.getUnits()[0]); } return parse(text); }
/** * Returns a Flink {@link MemorySize} under the given key if it exists. */ public Optional<MemorySize> getOptionalMemorySize(String key) { return optionalGet(key).map((value) -> { try { return MemorySize.parse(value, MemorySize.MemoryUnit.BYTES); } catch (Exception e) { throw new ValidationException("Invalid memory size value for key '" + key + "'.", e); } }); }
/** * Returns a Flink {@link MemorySize} under the given key if it exists. */ public Optional<MemorySize> getOptionalMemorySize(String key) { return optionalGet(key).map((value) -> { try { return MemorySize.parse(value, MemorySize.MemoryUnit.BYTES); } catch (Exception e) { throw new ValidationException("Invalid memory size value for key '" + key + "'.", e); } }); }