Refine search
public static byte[] toBytes(SubmitWorkInfo submitWorkInfo) throws IOException { DataOutputBuffer dob = new DataOutputBuffer(); submitWorkInfo.write(dob); return dob.getData(); }
/** * Serialize the shuffle port into a ByteBuffer for use later on. * @param port the port to be sent to the ApplciationMaster * @return the serialized form of the port. */ public static ByteBuffer serializeMetaData(int port) throws IOException { //TODO these bytes should be versioned DataOutputBuffer port_dob = new DataOutputBuffer(); port_dob.writeInt(port); return ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength()); }
/** * A helper function to serialize the JobTokenIdentifier to be sent to the * ShuffleHandler as ServiceData. * @param jobToken the job token to be used for authentication of * shuffle data requests. * @return the serialized version of the jobToken. */ public static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken) throws IOException { //TODO these bytes should be versioned DataOutputBuffer jobToken_dob = new DataOutputBuffer(); jobToken.write(jobToken_dob); return ByteBuffer.wrap(jobToken_dob.getData(), 0, jobToken_dob.getLength()); }
/** * Creates a ByteBuffer with serialized {@link Credentials}. * * @param creds The credentials. * @return The ByteBuffer with the credentials. * @throws IOException */ public static ByteBuffer createTokenBuffer(Credentials creds) throws IOException { DataOutputBuffer dob = new DataOutputBuffer(); creds.writeTokenStorageToStream(dob); return ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } }
/** * Get the bytes for the token identifier * @return the bytes of the identifier */ public byte[] getBytes() { DataOutputBuffer buf = new DataOutputBuffer(4096); try { this.write(buf); } catch (IOException ie) { throw new RuntimeException("i/o error in getBytes", ie); } return Arrays.copyOf(buf.getData(), buf.getLength()); }
/** Convert writables to a byte array */ public static byte[] toByteArray(Writable... writables) { final DataOutputBuffer out = new DataOutputBuffer(); try { for(Writable w : writables) { w.write(out); } out.close(); } catch (IOException e) { throw new RuntimeException("Fail to convert writables to a byte array",e); } return out.getData(); }
UserPayload getBytePayload(Multimap<Integer, Integer> routingTable) throws IOException { CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(numBuckets, routingTable); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] serialized = dob.getData(); return UserPayload.create(ByteBuffer.wrap(serialized)); }
UserPayload getBytePayload(Multimap<Integer, Integer> routingTable) throws IOException { CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(numBuckets, routingTable); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] serialized = dob.getData(); return UserPayload.create(ByteBuffer.wrap(serialized)); }
private ByteBuffer serializeCredentials(Credentials credentials) throws IOException { Credentials containerCredentials = new Credentials(); containerCredentials.addAll(credentials); DataOutputBuffer containerTokens_dob = new DataOutputBuffer(); containerCredentials.writeTokenStorageToStream(containerTokens_dob); return ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength()); }
public static ByteBuffer serializeCredentials(Credentials credentials) throws IOException { Credentials containerCredentials = new Credentials(); containerCredentials.addAll(credentials); DataOutputBuffer containerTokensDob = new DataOutputBuffer(); containerCredentials.writeTokenStorageToStream(containerTokensDob); return ByteBuffer.wrap(containerTokensDob.getData(), 0, containerTokensDob.getLength()); } }
private ByteBuffer getSecurityTokens() throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); Closer closer = Closer.create(); try { DataOutputBuffer dataOutputBuffer = closer.register(new DataOutputBuffer()); credentials.writeTokenStorageToStream(dataOutputBuffer); // Remove the AM->RM token so that containers cannot access it Iterator<Token<?>> tokenIterator = credentials.getAllTokens().iterator(); while (tokenIterator.hasNext()) { Token<?> token = tokenIterator.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { tokenIterator.remove(); } } return ByteBuffer.wrap(dataOutputBuffer.getData(), 0, dataOutputBuffer.getLength()); } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
/** * Generate a string with the url-quoted base64 encoded serialized form * of the Writable. * @param obj the object to serialize * @return the encoded string * @throws IOException */ private static String encodeWritable(Writable obj) throws IOException { DataOutputBuffer buf = new DataOutputBuffer(); obj.write(buf); Base64 encoder = new Base64(0, null, true); byte[] raw = new byte[buf.getLength()]; System.arraycopy(buf.getData(), 0, raw, 0, buf.getLength()); return encoder.encodeToString(raw); }
private void setupSecurityTokens(ContainerLaunchContext containerLaunchContext) throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); String tokenRenewer = this.yarnConfiguration.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Failed to get master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. Token<?> tokens[] = this.fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOGGER.info("Got delegation token for " + this.fs.getUri() + "; " + token); } } Closer closer = Closer.create(); try { DataOutputBuffer dataOutputBuffer = closer.register(new DataOutputBuffer()); credentials.writeTokenStorageToStream(dataOutputBuffer); ByteBuffer fsTokens = ByteBuffer.wrap(dataOutputBuffer.getData(), 0, dataOutputBuffer.getLength()); containerLaunchContext.setTokens(fsTokens); } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
public static void setTokensFor(ContainerLaunchContext amContainer, List<Path> paths, Configuration conf) throws IOException { Credentials credentials = new Credentials(); // for HDFS TokenCache.obtainTokensForNamenodes(credentials, paths.toArray(new Path[0]), conf); // for HBase obtainTokenForHBase(credentials, conf); // for user UserGroupInformation currUsr = UserGroupInformation.getCurrentUser(); Collection<Token<? extends TokenIdentifier>> usrTok = currUsr.getTokens(); for (Token<? extends TokenIdentifier> token : usrTok) { final Text id = new Text(token.getIdentifier()); LOG.info("Adding user token " + id + " with " + token); credentials.addToken(id, token); } try (DataOutputBuffer dob = new DataOutputBuffer()) { credentials.writeTokenStorageToStream(dob); if (LOG.isDebugEnabled()) { LOG.debug("Wrote tokens. Credentials buffer length: " + dob.getLength()); } ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(securityTokens); } }
/** Used by child copy constructors. */ protected synchronized void copy(Writable other) { if (other != null) { try { DataOutputBuffer out = new DataOutputBuffer(); other.write(out); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), out.getLength()); readFields(in); } catch (IOException e) { throw new IllegalArgumentException("map cannot be copied: " + e.getMessage()); } } else { throw new IllegalArgumentException("source map cannot be null"); } }
log.debug("Adding security tokens to TaskExecutor's container launch context."); try (DataOutputBuffer dob = new DataOutputBuffer()) { Method readTokenStorageFileMethod = Credentials.class.getMethod( "readTokenStorageFile", File.class, org.apache.hadoop.conf.Configuration.class); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); ctx.setTokens(securityTokens); } catch (Throwable t) {
/** Read a compressed buffer */ private synchronized void readBuffer(DataInputBuffer buffer, CompressionInputStream filter) throws IOException { // Read data into a temporary buffer DataOutputBuffer dataBuffer = new DataOutputBuffer(); try { int dataBufferLength = WritableUtils.readVInt(in); dataBuffer.write(in, dataBufferLength); // Set up 'buffer' connected to the input-stream buffer.reset(dataBuffer.getData(), 0, dataBuffer.getLength()); } finally { dataBuffer.close(); } // Reset the codec filter.resetState(); }
@Override public byte[] serialize() throws IOException { DataOutputBuffer dob = new DataOutputBuffer(5); dob.writeInt(masterKeyId); dob.write(index); byte[] b = dob.getData(); dob.close(); return b; }
DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens);
public void write(DataOutput out) throws IOException { if (firstKey == null) { Utils.writeVInt(out, 0); return; } DataOutputBuffer dob = new DataOutputBuffer(); Utils.writeVInt(dob, firstKey.size()); dob.write(firstKey.buffer()); Utils.writeVInt(out, dob.size()); out.write(dob.getData(), 0, dob.getLength()); for (TFileIndexEntry entry : index) { dob.reset(); entry.write(dob); Utils.writeVInt(out, dob.getLength()); out.write(dob.getData(), 0, dob.getLength()); } } }