public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) { if (localJar == null) { throw new RuntimeException("Must submit topologies using the 'jstorm' client script so that " + "StormSubmitter knows which jar to upload."); } try { LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation); int bufferSize = 512 * 1024; Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE); if (maxBufSizeObject != null) { bufferSize = Utils.getInt(maxBufSizeObject) / 2; } BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize); while (true) { byte[] toSubmit = is.read(); if (toSubmit.length == 0) break; client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit)); } client.getClient().finishFileUpload(uploadLocation); LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation); return uploadLocation; } catch (Exception e) { throw new RuntimeException(e); } }
public byte[] read() throws IOException { int length = stream.read(buffer); if (length == -1) { close(); return new byte[0]; } else if (length == buffer.length) { return buffer; } else { return Arrays.copyOf(buffer, length); } }
@Override public String beginFileDownload(String file) throws TException { BufferFileInputStream is = null; String id = null; try { is = new BufferFileInputStream(file); id = UUID.randomUUID().toString(); data.getDownloaders().put(id, is); } catch (FileNotFoundException e) { LOG.error(e + "file:" + file + " not found"); } return id; }
@Override public ByteBuffer downloadChunk(String id) throws TException { TimeCacheMap<Object,Object> downloaders = data.getDownloaders(); Object obj = downloaders.get(id); if (obj == null) { throw new RuntimeException( "Could not find input stream for that id"); } byte[] ret = null; try { if(obj instanceof BufferFileInputStream){ BufferFileInputStream is=(BufferFileInputStream) obj; ret=is.read(); if (ret != null) { downloaders.put(id, (BufferFileInputStream) is); } } } catch (IOException e) { LOG.error(e + "BufferFileInputStream read failed when downloadChunk "); } return ByteBuffer.wrap(ret); }
@Override public String beginFileDownload(String file) throws TException { BufferFileInputStream is; String id; try { int bufferSize = JStormUtils.parseInt(conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE), 1024 * 1024) / 2; is = new BufferFileInputStream(file, bufferSize); id = UUID.randomUUID().toString(); data.getDownloaders().put(id, is); } catch (FileNotFoundException e) { LOG.error(e + "file:" + file + " not found"); throw new TException(e); } return id; }
@Override public ByteBuffer downloadChunk(String id) throws TException { TimeCacheMap<Object, Object> downloaders = data.getDownloaders(); Object obj = downloaders.get(id); if (obj == null) { throw new TException("Could not find input stream for that id"); } try { if (obj instanceof BufferFileInputStream) { BufferFileInputStream is = (BufferFileInputStream) obj; byte[] ret = is.read(); if (ret != null) { downloaders.put(id, is); return ByteBuffer.wrap(ret); } } else { throw new TException("Object isn't BufferFileInputStream for " + id); } } catch (IOException e) { LOG.error("BufferFileInputStream read failed when downloadChunk ", e); throw new TException(e); } byte[] empty = {}; return ByteBuffer.wrap(empty); }
public static String submitJar(Map conf, String localJar) { NimbusClient client = NimbusClient.getConfiguredClient(conf); try { String uploadLocation = client.getClient().beginFileUpload(); LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation); BufferFileInputStream is = new BufferFileInputStream(localJar); while(true) { byte[] toSubmit = is.read(); if(toSubmit.length==0) break; client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit)); } client.getClient().finishFileUpload(uploadLocation); LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation); return uploadLocation; } catch(Exception e) { throw new RuntimeException(e); } finally { client.close(); } } }
@Override public String beginFileDownload(String file) throws TException { BufferFileInputStream is; String id; try { int bufferSize = JStormUtils.parseInt(conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE), 1024 * 1024) / 2; is = new BufferFileInputStream(file, bufferSize); id = UUID.randomUUID().toString(); data.getDownloaders().put(id, is); } catch (FileNotFoundException e) { LOG.error(e + "file:" + file + " not found"); throw new TException(e); } return id; }
@Override public ByteBuffer downloadChunk(String id) throws TException { TimeCacheMap<Object, Object> downloaders = data.getDownloaders(); Object obj = downloaders.get(id); if (obj == null) { throw new TException("Could not find input stream for that id"); } try { if (obj instanceof BufferFileInputStream) { BufferFileInputStream is = (BufferFileInputStream) obj; byte[] ret = is.read(); if (ret != null) { downloaders.put(id, is); return ByteBuffer.wrap(ret); } } else { throw new TException("Object isn't BufferFileInputStream for " + id); } } catch (IOException e) { LOG.error("BufferFileInputStream read failed when downloadChunk ", e); throw new TException(e); } byte[] empty = {}; return ByteBuffer.wrap(empty); }
public byte[] read() throws IOException { int length = stream.read(buffer); if(length==-1) { close(); return new byte[0]; } else if(length==buffer.length) { return buffer; } else { return Arrays.copyOf(buffer, length); } }
public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) { if (localJar == null) { throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload."); } try { LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation); int bufferSize = 512 * 1024; Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE); if (maxBufSizeObject != null) { bufferSize = Utils.getInt(maxBufSizeObject) / 2; } BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize); while (true) { byte[] toSubmit = is.read(); if (toSubmit.length == 0) break; client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit)); } client.getClient().finishFileUpload(uploadLocation); LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation); return uploadLocation; } catch (Exception e) { throw new RuntimeException(e); } finally { } }
@Override public void expire(Object key, Object val) { try { LOG.info("Close file " + String.valueOf(key)); if (val != null) { if (val instanceof Channel) { Channel channel = (Channel) val; channel.close(); } else if (val instanceof BufferFileInputStream) { BufferFileInputStream is = (BufferFileInputStream) val; is.close(); } } } catch (IOException e) { LOG.error(e.getMessage(), e); } } };
String uploadLocation = client.getClient().beginFileUpload(); LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation); BufferFileInputStream is = new BufferFileInputStream(localJar, THRIFT_CHUNK_SIZE_BYTES); byte[] toSubmit = is.read(); bytesUploaded += toSubmit.length; if (listener != null) {
public byte[] read() throws IOException { int length = stream.read(buffer); if (length == -1) { close(); return new byte[0]; } else if (length == buffer.length) { return buffer; } else { return Arrays.copyOf(buffer, length); } }
String uploadLocation = client.getClient().beginFileUpload(); LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation); BufferFileInputStream is = new BufferFileInputStream(localJar, THRIFT_CHUNK_SIZE_BYTES); byte[] toSubmit = is.read(); bytesUploaded += toSubmit.length; if (listener != null) {
public byte[] read() throws IOException { int length = stream.read(buffer); if(length==-1) { close(); return new byte[0]; } else if(length==buffer.length) { return buffer; } else { return Arrays.copyOf(buffer, length); } }
@Override public void expire(Object key, Object val) { try { LOG.info("Close file " + String.valueOf(key)); if (val != null) { if (val instanceof Channel) { Channel channel = (Channel) val; channel.close(); } else if (val instanceof BufferFileInputStream) { BufferFileInputStream is = (BufferFileInputStream) val; is.close(); } } } catch (IOException e) { LOG.error(e.getMessage(), e); } } };