private void writePut(PutWritable put) throws IOException { ImmutableBytesWritable row = new ImmutableBytesWritable(put.getPut().getRow()); SortedMap<byte[], List<Cell>> cells = put.getPut().getFamilyCellMap(); for (Map.Entry<byte[], List<Cell>> entry : cells.entrySet()) { Collections.sort(entry.getValue(), new CellComparatorImpl()); for (Cell c : entry.getValue()) { try { fileWriter.write(row, KeyValueUtil.copyToNewKeyValue(c)); } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException().initCause(e); } } } }
public void write(ImmutableBytesWritable key, Object value) throws IOException { Put put; if (value instanceof Put){ put = (Put)value; } else if (value instanceof PutWritable) { put = new Put(((PutWritable)value).getPut()); } else { throw new IllegalArgumentException("Illegal Argument " + (value == null ? "null" : value.getClass().getName())); } if(m_walEnabled) { put.setDurability(Durability.SYNC_WAL); } else { put.setDurability(Durability.SKIP_WAL); } m_table.mutate(put); }
@Test public void testPut() throws Exception { byte[] row = Bytes.toBytes("test-row"); // Initialize a result KeyValue[] kvs = new KeyValue[] { new KeyValue(row, Bytes.toBytes("cfa"), Bytes.toBytes("col1"), Bytes.toBytes("cfacol1")), new KeyValue(row, Bytes.toBytes("cfa"), Bytes.toBytes("col2"), Bytes.toBytes("cfacol2")) }; Put expected = new Put(row); for (int i = 0; i < kvs.length; i++) { expected.add(kvs[i]); } PutWritable actual = copy(new PutWritable(expected), new PutWritable()); Assert.assertArrayEquals(expected.getRow(), actual.getPut().getRow()); Assert.assertEquals(expected.getFamilyCellMap().keySet(), actual.getPut().getFamilyCellMap().keySet()); }
private void deserializeAndSerializeHiveAvro(HBaseSerDe serDe, Result r, Put p, Object[] expectedFieldsData, String expectedDeserializedAvroString) throws SerDeException, IOException { StructObjectInspector soi = (StructObjectInspector) serDe.getObjectInspector(); List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs(); Object row = serDe.deserialize(new ResultWritable(r)); for (int j = 0; j < fieldRefs.size(); j++) { Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j)); assertNotNull(fieldData); assertEquals(expectedFieldsData[j], fieldData.toString().trim()); } assertEquals(expectedDeserializedAvroString, SerDeUtils.getJSONString(row, soi)); // Now serialize Put put = ((PutWritable) serDe.serialize(row, soi)).getPut(); assertNotNull(put); assertEquals(p.getFamilyCellMap(), put.getFamilyCellMap()); }
private void deserializeAndSerialize( HBaseSerDe serDe, Result r, Put p, Object[] expectedFieldsData) throws SerDeException { // Get the row structure StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector(); List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs(); assertEquals(9, fieldRefs.size()); // Deserialize Object row = serDe.deserialize(new ResultWritable(r)); for (int i = 0; i < fieldRefs.size(); i++) { Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); if (fieldData != null) { fieldData = ((LazyPrimitive<?, ?>)fieldData).getWritableObject(); } assertEquals("Field " + i, expectedFieldsData[i], fieldData); } // Serialize assertEquals(PutWritable.class, serDe.getSerializedClass()); PutWritable serializedPut = (PutWritable) serDe.serialize(row, oi); assertEquals("Serialized data", p.toString(),String.valueOf(serializedPut.getPut())); }
private void deserializeAndSerializeHBaseValueStruct(HBaseSerDe serDe, Result r, Put p) throws SerDeException, IOException { StructObjectInspector soi = (StructObjectInspector) serDe.getObjectInspector(); List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs(); Object row = serDe.deserialize(new ResultWritable(r)); Object fieldData = null; for (int j = 0; j < fieldRefs.size(); j++) { fieldData = soi.getStructFieldData(row, fieldRefs.get(j)); assertNotNull(fieldData); if (fieldData instanceof LazyStruct) { assertEquals(((LazyStruct) fieldData).getField(0).toString(), "A"); assertEquals(((LazyStruct) fieldData).getField(1).toString(), "B"); assertEquals(((LazyStruct) fieldData).getField(2).toString(), "C"); } else { Assert.fail("fieldData should be an instance of LazyStruct"); } } assertEquals( "{\"key\":{\"col1\":\"A\",\"col2\":\"B\",\"col3\":\"C\"},\"astring\":{\"col1\":\"A\",\"col2\":\"B\",\"col3\":\"C\"}}", SerDeUtils.getJSONString(row, soi)); // Now serialize Put put = ((PutWritable) serDe.serialize(row, soi)).getPut(); assertEquals("Serialized put:", p.toString(), put.toString()); }
private void deserializeAndSerializeHBaseCompositeKey(HBaseSerDe serDe, Result r, Put p) throws SerDeException, IOException { StructObjectInspector soi = (StructObjectInspector) serDe.getObjectInspector(); List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs(); Object row = serDe.deserialize(new ResultWritable(r)); for (int j = 0; j < fieldRefs.size(); j++) { Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j)); assertNotNull(fieldData); } assertEquals( "{\"key\":{\"col1\":\"A\",\"col2\":\"B\",\"col3\":\"C\"},\"astring\":\"This is a test data\"}", SerDeUtils.getJSONString(row, soi)); // Now serialize Put put = ((PutWritable) serDe.serialize(row, soi)).getPut(); assertEquals("Serialized put:", p.toString(), put.toString()); }
Put put = ((PutWritable) serDe.serialize(row, soi)).getPut();
Put put = ((PutWritable) serDe.serialize(row, soi)).getPut();
private void deserializeAndSerializeHiveMapHBaseColumnFamilyII( HBaseSerDe hbaseSerDe, Result r, Put p, Object [] expectedData, byte [][] columnFamilies, byte [][] columnQualifiersAndValues) throws SerDeException { StructObjectInspector soi = (StructObjectInspector) hbaseSerDe.getObjectInspector(); List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs(); assertEquals(9, fieldRefs.size()); // Deserialize Object row = hbaseSerDe.deserialize(new ResultWritable(r)); for (int j = 0; j < fieldRefs.size(); j++) { Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j)); assertNotNull(fieldData); if (fieldData instanceof LazyPrimitive<?, ?>) { assertEquals(expectedData[j], ((LazyPrimitive<?, ?>) fieldData).getWritableObject()); } else if (fieldData instanceof LazyHBaseCellMap) { LazyPrimitive<?, ?> lazyPrimitive = (LazyPrimitive<?, ?>) ((LazyHBaseCellMap) fieldData).getMapValueElement(expectedData[j]); assertEquals(expectedData[j], lazyPrimitive.getWritableObject()); } else { fail("Error: field data not an instance of LazyPrimitive<?, ?> or LazyHBaseCellMap"); } } // Serialize Put serializedPut = ((PutWritable) hbaseSerDe.serialize(row, soi)).getPut(); assertEquals("Serialized data: ", p.toString(), serializedPut.toString()); }
Put serializedPut = ((PutWritable) hbaseSerDe.serialize(row, soi)).getPut(); byte [] rowKey = serializedPut.getRow();
protected static Put toPut(Object o) { if(o != null) { if(o instanceof Put) { return (Put)o; } else if(o instanceof PutWritable) { return ((PutWritable)o).getPut(); } } throw new IllegalArgumentException("Illegal Argument " + (o == null ? "null" : o.getClass().getName())); }
private void writePut(PutWritable put) throws IOException { ImmutableBytesWritable row = new ImmutableBytesWritable(put.getPut().getRow()); SortedMap<byte[], List<Cell>> cells = put.getPut().getFamilyCellMap(); for (Map.Entry<byte[], List<Cell>> entry : cells.entrySet()) { Collections.sort(entry.getValue(), new CellComparator()); for (Cell c : entry.getValue()) { try { fileWriter.write(row, KeyValueUtil.copyToNewKeyValue(c)); } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException().initCause(e); } } } }
public void write(ImmutableBytesWritable key, Object value) throws IOException { Put put; if (value instanceof Put){ put = (Put)value; } else if (value instanceof PutWritable) { put = new Put(((PutWritable)value).getPut()); } else { throw new IllegalArgumentException("Illegal Argument " + (value == null ? "null" : value.getClass().getName())); } if(m_walEnabled) { put.setDurability(Durability.SYNC_WAL); } else { put.setDurability(Durability.SKIP_WAL); } m_table.put(put); } }