/** * Marks that the clusters with the given clusterIds have consumed the mutation * @param clusterIds of the clusters that have consumed the mutation */ public Mutation setClusterIds(List<UUID> clusterIds) { ByteArrayDataOutput out = ByteStreams.newDataOutput(); out.writeInt(clusterIds.size()); for (UUID clusterId : clusterIds) { out.writeLong(clusterId.getMostSignificantBits()); out.writeLong(clusterId.getLeastSignificantBits()); } setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray()); return this; }
/** * @return the set of clusterIds that have consumed the mutation */ public List<UUID> getClusterIds() { List<UUID> clusterIds = new ArrayList<>(); byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS); if(bytes != null) { ByteArrayDataInput in = ByteStreams.newDataInput(bytes); int numClusters = in.readInt(); for(int i=0; i<numClusters; i++){ clusterIds.add(new UUID(in.readLong(), in.readLong())); } } return clusterIds; }
@Test public void testEmptyWorks() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); MessageCodec cmc = new MessageCodec(); Codec.Encoder encoder = cmc.getEncoder(dos); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(0, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = cmc.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(0, cis.getCount()); }
private void corrupt(FileStatus file) throws IOException { LOG.info("Corrupt " + file); Path tmpFile = file.getPath().suffix(".tmp"); // remove the last byte to make the trailer corrupted try (FSDataInputStream in = fs.open(file.getPath()); FSDataOutputStream out = fs.create(tmpFile)) { ByteStreams.copy(ByteStreams.limit(in, file.getLen() - 1), out); } fs.delete(file.getPath(), false); fs.rename(tmpFile, file.getPath()); }
/** * This version of protobuf's mergeDelimitedFrom avoid the hard-coded 64MB limit for decoding * buffers * @param builder current message builder * @param in Inputsream with delimited protobuf data * @throws IOException */ public static void mergeDelimitedFrom(Message.Builder builder, InputStream in) throws IOException { // This used to be builder.mergeDelimitedFrom(in); // but is replaced to allow us to bump the protobuf size limit. final int firstByte = in.read(); if (firstByte != -1) { final int size = CodedInputStream.readRawVarint32(firstByte, in); final InputStream limitedInput = ByteStreams.limit(in, size); final CodedInputStream codedInput = CodedInputStream.newInstance(limitedInput); codedInput.setSizeLimit(size); builder.mergeFrom(codedInput); codedInput.checkLastTagWas(0); } }
private void writeAndVerify(BufferChain chain, String string, int chunkSize) throws IOException { FileOutputStream fos = new FileOutputStream(tmpFile); FileChannel ch = fos.getChannel(); try { long remaining = string.length(); while (chain.hasRemaining()) { long n = chain.write(ch, chunkSize); assertTrue(n == chunkSize || n == remaining); remaining -= n; } assertEquals(0, remaining); } finally { fos.close(); } assertFalse(chain.hasRemaining()); assertEquals(string, Files.toString(tmpFile, Charsets.UTF_8)); } }
@Test public void testEmptyWorks() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); KeyValueCodec kvc = new KeyValueCodec(); Codec.Encoder encoder = kvc.getEncoder(dos); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(0, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = kvc.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(0, cis.getCount()); }
@AfterClass public static void tearDown() throws Exception { Closeables.close(CONN, true); UTIL.shutdownMiniCluster(); }
@Test public void testWithSpy() throws IOException { ByteBuffer[] bufs = new ByteBuffer[] { stringBuf("XXXhelloYYY", 3, 5), stringBuf(" ", 0, 1), stringBuf("XXXXworldY", 4, 5) }; BufferChain chain = new BufferChain(bufs); FileOutputStream fos = new FileOutputStream(tmpFile); FileChannel ch = Mockito.spy(fos.getChannel()); try { chain.write(ch, 2); assertEquals("he", Files.toString(tmpFile, Charsets.UTF_8)); chain.write(ch, 2); assertEquals("hell", Files.toString(tmpFile, Charsets.UTF_8)); chain.write(ch, 3); assertEquals("hello w", Files.toString(tmpFile, Charsets.UTF_8)); chain.write(ch, 8); assertEquals("hello world", Files.toString(tmpFile, Charsets.UTF_8)); } finally { ch.close(); fos.close(); } }
@Test public void testEmptyWorks() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(0, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(0, cis.getCount()); }
@AfterClass public static void tearDown() throws Exception { Closeables.close(CONN, true); TEST_UTIL.shutdownMiniCluster(); }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); MessageCodec cmc = new MessageCodec(); Codec.Encoder encoder = cmc.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = cmc.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV assertFalse(decoder.advance()); // Second read should trip over the end-of-stream marker and return false dis.close(); assertEquals(offset, cis.getCount()); }
@AfterClass public static void tearDownAfterClass() throws Exception { Closeables.close(ASYNC_CONN, true); TEST_UTIL.shutdownMiniCluster(); }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); kv.setSequenceId(Long.MAX_VALUE); encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV // Second read should trip over the end-of-stream marker and return false assertFalse(decoder.advance()); dis.close(); assertEquals(offset, cis.getCount()); }
@AfterClass public static void tearDown() throws IOException { Closeables.close(ZKW, true); UTIL.shutdownMiniZKCluster(); UTIL.cleanupTestDir(); }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); KeyValueCodec kvc = new KeyValueCodec(); Codec.Encoder encoder = kvc.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); final int length = kv.getLength() + Bytes.SIZEOF_INT; encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(length, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = kvc.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV // Second read should trip over the end-of-stream marker and return false assertFalse(decoder.advance()); dis.close(); assertEquals(length, cis.getCount()); }
Closeables.close(scanner, true); scanners.forEach(KeyValueScanner::close);
@Test public void testThree() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); KeyValueCodec kvc = new KeyValueCodec(); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(length * 3, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = kvc.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals((length * 3), cis.getCount());
@Test public void testThree() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); MessageCodec cmc = new MessageCodec(); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = cmc.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(offset, cis.getCount());