@Override public Long getBytes(String path) { return c.getBytes(path); }
protected void sbytes(final String path, final Config config, final String name, final Consumer<Long> setter) { if (config.hasPath(name)) { long value = config.getBytes(name); log.debug("setting {}.{} = {}", path, name, value); setter.accept(value); } }
public NettyPipeline(final EventExecutorGroup executor, final HttpHandler handler, final Config conf, final SslContext sslCtx) { this.executor = executor; this.handler = handler; this.config = conf; maxInitialLineLength = conf.getBytes("netty.http.MaxInitialLineLength").intValue(); maxHeaderSize = conf.getBytes("netty.http.MaxHeaderSize").intValue(); maxChunkSize = conf.getBytes("netty.http.MaxChunkSize").intValue(); maxContentLength = conf.getBytes("netty.http.MaxContentLength").intValue(); idleTimeOut = conf.getDuration("netty.http.IdleTimeout", TimeUnit.MILLISECONDS); supportH2 = conf.getBoolean("server.http2.enabled"); this.tmpdir = config.getString("application.tmpdir"); this.bufferSize = config.getBytes("server.http.ResponseBufferSize").intValue(); this.wsMaxMessageSize = Math .max( config.getBytes("server.ws.MaxTextMessageSize").intValue(), config.getBytes("server.ws.MaxBinaryMessageSize").intValue()); this.sslCtx = sslCtx; }
private void tryOption(final Object source, final Config config, final Method option) { Try.run(() -> { String optionName = option.getName().replace("set", ""); Object optionValue = config.getAnyRef(optionName); Class<?> optionType = Primitives.wrap(option.getParameterTypes()[0]); if (Number.class.isAssignableFrom(optionType) && optionValue instanceof String) { // either a byte or time unit try { optionValue = config.getBytes(optionName); } catch (ConfigException.BadValue ex) { optionValue = config.getDuration(optionName, TimeUnit.MILLISECONDS); } if (optionType == Integer.class) { // to int optionValue = ((Number) optionValue).intValue(); } } log.debug("{}.{}({})", source.getClass().getSimpleName(), option.getName(), optionValue); option.invoke(source, optionValue); }).unwrap(InvocationTargetException.class) .throwException(); }
@SuppressWarnings("rawtypes") private static Consumer<Map.Entry<String, ConfigValue>> setOption(final Config config, final String level, final BiConsumer<Option, Object> setter) { return entry -> { String name = entry.getKey(); Object value = entry.getValue().unwrapped(); Option option = findOption(name, UndertowOptions.class, Options.class); if (option != null) { // parse option to adjust correct type Object parsedValue = value.toString(); try { parsedValue = option.parseValue(value.toString(), null); } catch (NumberFormatException ex) { // try bytes try { parsedValue = option.parseValue(config.getBytes(level + "." + name).toString(), null); } catch (ConfigException.BadValue badvalue) { // try duration parsedValue = option.parseValue( config.getDuration(level + "." + name, TimeUnit.MILLISECONDS) + "", null); } } log.debug("{}.{}({})", level, option.getName(), parsedValue); setter.accept(option, parsedValue); } else { log.error("Unknown option: 'undertow.{}.{} = {}'", level, name, value); } }; }
private LogCopier buildLogCopier(Config config, Path sinkLogDir, Path appWorkDir) throws IOException { FileSystem rawLocalFs = this.closer.register(new RawLocalFileSystem()); rawLocalFs.initialize(URI.create(ConfigurationKeys.LOCAL_FS_URI), new Configuration()); LogCopier.Builder builder = LogCopier.newBuilder() .useSrcFileSystem(this.fs) .useDestFileSystem(rawLocalFs) .readFrom(getHdfsLogDir(appWorkDir)) .writeTo(sinkLogDir) .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) { builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)); } if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) { builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)); } return builder.build(); }
@Override public Mutant body() throws Exception { long length = length(); if (length > 0) { MediaType type = type(); Config conf = require(Config.class); // TODO: sanitization of arguments File fbody = new File(conf.getString("application.tmpdir"), Integer.toHexString(System.identityHashCode(this))); files.add(fbody); int bufferSize = conf.getBytes("server.http.RequestBufferSize").intValue(); Parser.BodyReference body = new BodyReferenceImpl(length, charset(), fbody, req.in(), bufferSize); return new MutantImpl(require(ParserExecutor.class), type, body); } return new MutantImpl(require(ParserExecutor.class), type, new EmptyBodyReference()); }
int value = $undertow.getBytes(name).intValue(); log.debug("undertow.bufferSize({})", value); builder.setBufferSize(value);
/** * Build a {@link LogCopier} instance used to copy the logs out from this {@link GobblinYarnLogSource}. * * @param config the {@link Config} use to create the {@link LogCopier} * @param containerId the {@link ContainerId} of the container the {@link LogCopier} runs in * @param destFs the destination {@link FileSystem} * @param appWorkDir the Gobblin Yarn application working directory on HDFS * @return a {@link LogCopier} instance * @throws IOException if it fails on any IO operation */ protected LogCopier buildLogCopier(Config config, ContainerId containerId, FileSystem destFs, Path appWorkDir) throws IOException { LogCopier.Builder builder = LogCopier.newBuilder() .useSrcFileSystem(FileSystem.getLocal(new Configuration())) .useDestFileSystem(destFs) .readFrom(getLocalLogDir()) .writeTo(getHdfsLogDir(containerId, destFs, appWorkDir)) .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR)) .useLogFileNamePrefix(containerId.toString()); if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) { builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)); } if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) { builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)); } return builder.build(); }
@Override public Long getBytes(String path) { return config.getBytes(path); }
@Override public Long call() throws Exception { return configuration.getBytes(key); } }, null);
@Override public Long call() throws Exception { return configuration.getBytes(key); } }, defaultValue);
@Override public Long getBytes(String path) { return c.getBytes(path); }
protected void sbytes(final String path, final Config config, final String name, final Consumer<Long> setter) { if (config.hasPath(name)) { long value = config.getBytes(name); log.debug("setting {}.{} = {}", path, name, value); setter.accept(value); } }
public void configure(Config conf) { this.extractImageFeatures = conf .getBoolean("warc.index.extract.content.images.enabled"); log.info("Image feature extraction = " + this.extractImageFeatures); this.max_size_bytes = conf.getBytes("warc.index.extract.content.images.maxSizeInBytes"); log.info("Image - max size in bytes " + this.max_size_bytes); this.sampleRate = 1.0 / conf .getInt("warc.index.extract.content.images.analysisSamplingRate"); log.info("Image sample rate " + this.sampleRate); }
public ImageAnalyser(Config conf) { this.extractFaces = conf .getBoolean("warc.index.extract.content.images.detectFaces"); this.extractDominantColours = conf .getBoolean("warc.index.extract.content.images.dominantColours"); log.info("Image - detect faces = " + this.extractFaces); this.max_size_bytes = conf.getBytes("warc.index.extract.content.images.maxSizeInBytes"); log.info("Image - max size in bytes " + this.max_size_bytes); this.sampleRate = 1.0 / conf .getInt("warc.index.extract.content.images.analysisSamplingRate"); log.info("Image sample rate " + this.sampleRate); // Set up the parser: fdp = new FaceDetectionParser(conf); }
@Override public Mutant body() throws Exception { long length = length(); if (length > 0) { MediaType type = type(); Config conf = require(Config.class); // TODO: sanitization of arguments File fbody = new File(conf.getString("application.tmpdir"), Integer.toHexString(System.identityHashCode(this))); files.add(fbody); int bufferSize = conf.getBytes("server.http.RequestBufferSize").intValue(); Parser.BodyReference body = new BodyReferenceImpl(length, charset(), fbody, req.in(), bufferSize); return new MutantImpl(require(ParserExecutor.class), type, body); } return new MutantImpl(require(ParserExecutor.class), type, new EmptyBodyReference()); }