Refine search
private void readPartitionFile(FileSystem fs, Configuration conf, Path path) throws IOException { @SuppressWarnings("deprecation") SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf); ImmutableBytesWritable key = new ImmutableBytesWritable(); partitions = new ArrayList<>(); while (reader.next(key)) { partitions.add(new ImmutableBytesWritable(key.copyBytes())); } reader.close(); if (!Ordering.natural().isOrdered(partitions)) { throw new IOException("Partitions are not ordered!"); } }
ImmutableBytesWritable first = sorted.first(); if (writeMultipleTables) { first = new ImmutableBytesWritable(MultiTableHFileOutputFormat.getSuffix(sorted.first ().get())); + Bytes.toStringBinary(first.get()));
/** * Test map method of Importer */ @SuppressWarnings({ "unchecked", "rawtypes" }) @Test public void testKeyValueImporter() throws Throwable { CellImporter importer = new CellImporter(); Configuration configuration = new Configuration(); Context ctx = mock(Context.class); when(ctx.getConfiguration()).thenReturn(configuration); doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArgument(0); MapReduceExtendedCell key = (MapReduceExtendedCell) invocation.getArgument(1); assertEquals("Key", Bytes.toString(writer.get())); assertEquals("row", Bytes.toString(CellUtil.cloneRow(key))); return null; } }).when(ctx).write(any(), any()); importer.setup(ctx); Result value = mock(Result.class); KeyValue[] keys = { new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"), Bytes.toBytes("value")), new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"), Bytes.toBytes("value1")) }; when(value.rawCells()).thenReturn(keys); importer.map(new ImmutableBytesWritable(Bytes.toBytes("Key")), value, ctx); }
@Test public void testForCorrectSeparatorBytes1() throws Exception { Object[] o = new Object[]{"a", "b", "c"}; Object element = "d"; PDataType baseType = PVarchar.INSTANCE; PhoenixArray arr = new PhoenixArray(baseType, o); LiteralExpression arrayLiteral, elementLiteral; arrayLiteral = LiteralExpression.newConstant(arr, PVarcharArray.INSTANCE, null, null, SortOrder.ASC, Determinism.ALWAYS); elementLiteral = LiteralExpression.newConstant(element, baseType, null, null, SortOrder.ASC, Determinism.ALWAYS); List<Expression> expressions = Lists.newArrayList((Expression) elementLiteral); expressions.add(arrayLiteral); Expression arrayPrependFunction = new ArrayPrependFunction(expressions); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); arrayPrependFunction.evaluate(null, ptr); byte[] expected = new byte[]{100, 0, 97, 0, 98, 0, 99, 0, 0, 0, -128, 1, -128, 3, -128, 5, -128, 7, 0, 0, 0, 10, 0, 0, 0, 4, 1}; assertArrayEquals(expected, ptr.get()); }
lineBytes, value.getLength()); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength()); tags.add(new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(ttl)));
@Test public void TestConfigurePartitioner() throws IOException { Configuration conf = util.getConfiguration(); ImmutableBytesWritable writable = new ImmutableBytesWritable(); List<ImmutableBytesWritable> splitPoints = new LinkedList<ImmutableBytesWritable>(); splitPoints.add(writable);
.put(-1, new ImmutableBytesWritable(Bytes.fromHex("714cb10a9e3b5569852980edd8c6ca2f"))) .put(5, new ImmutableBytesWritable(Bytes.fromHex("28d961d9252ce8f8d44a07b38d3e1d96"))) .put(10, new ImmutableBytesWritable(Bytes.fromHex("f6bbc4a224d8fd929b783a92599eaffa"))) .put(15, new ImmutableBytesWritable(Bytes.fromHex("522deb5d97f73a414ecc11457be46881"))) .put(20, new ImmutableBytesWritable(Bytes.fromHex("b026f2611aaa46f7110116d807545352"))) .put(25, new ImmutableBytesWritable(Bytes.fromHex("39ffc1a3094aa12a2e90ffd9cef2ce93"))) .put(30, new ImmutableBytesWritable(Bytes.fromHex("f6b4d75727ce9a30ac29e4f08f601666"))) .put(35, new ImmutableBytesWritable(Bytes.fromHex("422e2d2f1eb79a8f02171a705a42c090"))) .put(40, new ImmutableBytesWritable(Bytes.fromHex("559ad61c900fffefea0a15abf8a97bc3"))) .put(45, new ImmutableBytesWritable(Bytes.fromHex("23019084513eca41cee436b2a29611cb"))) .put(50, new ImmutableBytesWritable(Bytes.fromHex("b40467d222ddb4949b142fe145ee9edc"))) .put(55, new ImmutableBytesWritable(Bytes.fromHex("372bf89fcd8ca4b7ab3c1add9d07f7e4"))) .put(60, new ImmutableBytesWritable(Bytes.fromHex("69ae0585e6255de27dce974e332b8f8b"))) .put(65, new ImmutableBytesWritable(Bytes.fromHex("8029610044297aad0abdbecd485d8e59"))) .put(70, new ImmutableBytesWritable(Bytes.fromHex("de5f784f7f78987b6e57ecfd81c8646f"))) .put(75, new ImmutableBytesWritable(Bytes.fromHex("1cd757cc4e1715c8c3b1c24447a1ec56"))) .put(80, new ImmutableBytesWritable(Bytes.fromHex("f9a53aacfeb6142b08066615e7038095"))) .put(85, new ImmutableBytesWritable(Bytes.fromHex("89b872b7e639df32d3276b33928c0c91"))) .put(90, new ImmutableBytesWritable(Bytes.fromHex("45eeac0646d46a474ea0484175faed38"))) .put(95, new ImmutableBytesWritable(Bytes.fromHex("f57c447e32a08f4bf1abb2892839ac56"))) .build(); ImmutableBytesWritable key = new ImmutableBytesWritable(); ImmutableBytesWritable hash = new ImmutableBytesWritable(); while(reader.next(key, hash)) { String keyString = Bytes.toHex(key.get(), key.getOffset(), key.getLength()); LOG.debug("Key: " + (keyString.isEmpty() ? "-1" : Integer.parseInt(keyString, 16)) + " Hash: " + Bytes.toHex(hash.get(), hash.getOffset(), hash.getLength()));
private boolean isTenantRowCell(Cell cell) { ImmutableBytesWritable key = new ImmutableBytesWritable(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); //rows in system.catalog that aren't tenant-owned will have a leading separator byte return key.get()[key.getOffset()] != QueryConstants.SEPARATOR_BYTE; } }
&& (isTableEndRow(stopRow) || isTableStartRow(regionStartKey) || Bytes.compareTo(stopRow, regionStartKey) > 0)) { startKeys.add(regionStartKey); partitions.add(new ImmutableBytesWritable(startKeys.get(splitIndex)));
@Test public void testForCorrectSeparatorBytes2() throws Exception { Object[] o = new Object[]{"a", "b", "c"}; Object element = "d"; PDataType baseType = PVarchar.INSTANCE; PhoenixArray arr = new PhoenixArray(baseType, o); LiteralExpression arrayLiteral, elementLiteral; arrayLiteral = LiteralExpression.newConstant(arr, PVarcharArray.INSTANCE, null, null, SortOrder.DESC, Determinism.ALWAYS); elementLiteral = LiteralExpression.newConstant(element, baseType, null, null, SortOrder.ASC, Determinism.ALWAYS); List<Expression> expressions = Lists.newArrayList((Expression) elementLiteral); expressions.add(arrayLiteral); Expression arrayPrependFunction = new ArrayPrependFunction(expressions); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); arrayPrependFunction.evaluate(null, ptr); byte[] expected = new byte[]{-101, -1, -98, -1, -99, -1, -100, -1, -1, -1, -128, 1, -128, 3, -128, 5, -128, 7, 0, 0, 0, 10, 0, 0, 0, 4, 1}; assertArrayEquals(expected, ptr.get()); }
@Test public void test_WritingTagData() throws Exception { tags.add(new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(978670))); KeyValue kv = new KeyValue(b, b, b, HConstants.LATEST_TIMESTAMP, b, tags); writer.write(new ImmutableBytesWritable(), kv); writer.close(context); writer = null;
@Test public void testIndexBuilder() throws Exception { Configuration configuration = new Configuration(); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(Bytes.toBytes("test")); Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>.Context ctx = mock(Context.class);
@Override public synchronized boolean getIndexMaintainers(ImmutableBytesWritable ptr, PhoenixConnection connection) { if (indexMaintainersPtr == null || indexMaintainersPtr.getLength()==0) { indexMaintainersPtr = new ImmutableBytesWritable(); if (indexes.isEmpty()) { indexMaintainersPtr.set(ByteUtil.EMPTY_BYTE_ARRAY); } else { IndexMaintainer.serialize(this, indexMaintainersPtr, connection); } } ptr.set(indexMaintainersPtr.get(), indexMaintainersPtr.getOffset(), indexMaintainersPtr.getLength()); return indexMaintainersPtr.getLength() > 0; }
if (key == null) key = new ImmutableBytesWritable(); if (value == null) value = new Result(); try { lastSuccessfulRow = key.get(); return true; lastSuccessfulRow = key.get(); return true;
SecureWriter out = new SecureWriter(conf, userProvider, userToken, opts)) { scanner = cp.checkScannerOpen(scan); ImmutableBytesWritable key = new ImmutableBytesWritable(); long rowCount = 0; long cellCount = 0; results.add(Result.create(cells)); cells.clear(); cp.postScannerNext(scanner, results, scan.getBatch(), hasMore);
@Test public void testForCorrectSeparatorBytes1() throws Exception { Object[] o = new Object[]{"a", "b", "c"}; Object element = "d"; PDataType baseType = PVarchar.INSTANCE; PhoenixArray arr = new PhoenixArray(baseType, o); LiteralExpression arrayLiteral, elementLiteral; arrayLiteral = LiteralExpression.newConstant(arr, PVarcharArray.INSTANCE, null, null, SortOrder.ASC, Determinism.ALWAYS); elementLiteral = LiteralExpression.newConstant(element, baseType, null, null, SortOrder.ASC, Determinism.ALWAYS); List<Expression> expressions = Lists.newArrayList((Expression) arrayLiteral); expressions.add(elementLiteral); Expression arrayAppendFunction = new ArrayAppendFunction(expressions); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); arrayAppendFunction.evaluate(null, ptr); byte[] expected = new byte[]{97, 0, 98, 0, 99, 0, 100, 0, 0, 0, -128, 1, -128, 3, -128, 5, -128, 7, 0, 0, 0, 10, 0, 0, 0, 4, 1}; assertArrayEquals(expected, ptr.get()); }
/** * Test HRegionPartitioner */ @Test public void testHRegionPartitioner() throws Exception { byte[][] families = { Bytes.toBytes("familyA"), Bytes.toBytes("familyB") }; UTIL.createTable(TableName.valueOf(name.getMethodName()), families, 1, Bytes.toBytes("aa"), Bytes.toBytes("cc"), 3); HRegionPartitioner<Long, Long> partitioner = new HRegionPartitioner<>(); Configuration configuration = UTIL.getConfiguration(); configuration.set(TableOutputFormat.OUTPUT_TABLE, name.getMethodName()); partitioner.setConf(configuration); ImmutableBytesWritable writable = new ImmutableBytesWritable(Bytes.toBytes("bb")); assertEquals(1, partitioner.getPartition(writable, 10L, 3)); assertEquals(0, partitioner.getPartition(writable, 10L, 1)); } }
@Override public synchronized void getIndexMaintainers(ImmutableBytesWritable ptr) { if (indexMaintainersPtr == null) { indexMaintainersPtr = new ImmutableBytesWritable(); if (indexes.isEmpty()) { indexMaintainersPtr.set(ByteUtil.EMPTY_BYTE_ARRAY); } else { IndexMaintainer.serialize(this, indexMaintainersPtr); } } ptr.set(indexMaintainersPtr.get(), indexMaintainersPtr.getOffset(), indexMaintainersPtr.getLength()); }
private static void assertEqualValue(Expression e, PDataType type, Object value) { ImmutableBytesWritable ptr = new ImmutableBytesWritable(); boolean evaluated = e.evaluate(null, ptr); assertTrue(evaluated); assertEquals(value, type.toObject(ptr.get())); CloneExpressionVisitor visitor = new CloneExpressionVisitor(); Expression clone = e.accept(visitor); evaluated = clone.evaluate(null, ptr); assertTrue(evaluated); assertEquals(value, type.toObject(ptr.get())); } }
public JoinKey(List<Expression> expressions) { this.expressions = expressions; this.keys = Lists.newArrayListWithExpectedSize(expressions.size()); for (int i = 0; i < expressions.size(); i++) { this.keys.add(new ImmutableBytesWritable(EMPTY_PTR)); } }