private KuduSplit toKuduSplit(KuduTableHandle tableHandle, KuduScanToken token, int primaryKeyColumnCount) { try { byte[] serializedScanToken = token.serialize(); return new KuduSplit(tableHandle, primaryKeyColumnCount, serializedScanToken); } catch (IOException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } } }
public KuduScanner createScanner(KuduSplit kuduSplit) { try { return KuduScanToken.deserializeIntoScanner(kuduSplit.getSerializedScanToken(), client); } catch (IOException e) { throw new RuntimeException(e); } }
@Test(timeout = 100000) public void testScanTokenReadMode() throws Exception { ScanTokenPB.Builder pbBuilder = ScanTokenPB.newBuilder(); pbBuilder.setTableName(table.getName()); pbBuilder.setReadMode(Common.ReadMode.READ_YOUR_WRITES); Client.ScanTokenPB scanTokenPB = pbBuilder.build(); final byte[] serializedToken = KuduScanToken.serialize(scanTokenPB); // Deserialize scan tokens and make sure the read mode is updated accordingly. KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, client); assertEquals(AsyncKuduScanner.ReadMode.READ_YOUR_WRITES, scanner.getReadMode()); }
@Override public KuduInputSplit[] createInputSplits(int minNumSplits) throws IOException { startTableContext(); Preconditions.checkNotNull(tableContext,"tableContext should not be null"); List<KuduScanToken> tokens = tableContext.scanTokens(tableFilters, tableProjections, rowsLimit); KuduInputSplit[] splits = new KuduInputSplit[tokens.size()]; for (int i = 0; i < tokens.size(); i++) { KuduScanToken token = tokens.get(i); List<String> locations = new ArrayList<>(token.getTablet().getReplicas().size()); for (LocatedTablet.Replica replica : token.getTablet().getReplicas()) { locations.add(getLocation(replica.getRpcHost(), replica.getRpcPort())); } KuduInputSplit split = new KuduInputSplit( token.serialize(), i, locations.toArray(new String[locations.size()]) ); splits[i] = split; } if (splits.length < minNumSplits) { LOG.warn(" The minimum desired number of splits with your configured parallelism level " + "is {}. Current kudu splits = {}. {} instances will remain idle.", minNumSplits, splits.length, (minNumSplits - splits.length) ); } return splits; }
/** * Collects the rows from a set of scan tokens. * * @param scanTokens the scan token builder * @return the rows */ private Set<Row> collectRows(KuduScanToken.KuduScanTokenBuilder scanTokens) throws Exception { Set<Row> rows = new HashSet<>(); for (KuduScanToken token : scanTokens.build()) { LOG.debug("Scanning token: {}", KuduScanToken.stringifySerializedToken(token.serialize(), client)); int existingCount = rows.size(); Set<Row> newRows = collectRows(token.intoScanner(client)); rows.addAll(newRows); assertEquals(existingCount + newRows.size(), rows.size()); } return rows; }
KuduScanner aPartitionSpecificScanner = KuduScanToken.deserializeIntoScanner( kuduPartitionScanAssignmentMeta.getSerializedKuduScanToken(), kuduClientHandle); LOG.info("Scanning the following tablet " + KuduScanToken.stringifySerializedToken(kuduPartitionScanAssignmentMeta .getSerializedKuduScanToken(), kuduClientHandle)); KuduRecordWithMeta<T> beginScanRecord = new KuduRecordWithMeta<>(); KuduScanToken.stringifySerializedToken(kuduPartitionScanAssignmentMeta.getSerializedKuduScanToken(), kuduClientHandle)); return numRowsScanned;
LOG.debug(KuduScanToken.stringifySerializedToken(token.serialize(), client));
@Override public int compareTo(KuduScanToken other) { if (!message.getTableName().equals(other.message.getTableName())) { throw new IllegalArgumentException("Scan tokens from different tables may not be compared"); } return tablet.getPartition().compareTo(other.getTablet().getPartition()); }
builder.setUpperBoundPartitionKey( UnsafeByteOperations.unsafeWrap(tablet.getPartition().getPartitionKeyEnd())); tokens.add(new KuduScanToken(tablet, builder.build()));
/** Test that scanRequestTimeout makes it from the scan token to the underlying Scanner class. */ @Test public void testScanRequestTimeout() throws IOException { final int NUM_ROWS_DESIRED = 100; final int SCAN_REQUEST_TIMEOUT_MS = 20; KuduTable table = createDefaultTable(client, testTableName); loadDefaultTable(client, testTableName, NUM_ROWS_DESIRED); KuduScanToken.KuduScanTokenBuilder builder = new KuduScanToken.KuduScanTokenBuilder(asyncClient, table); builder.scanRequestTimeout(SCAN_REQUEST_TIMEOUT_MS); List<KuduScanToken> tokens = builder.build(); for (KuduScanToken token : tokens) { byte[] serialized = token.serialize(); KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serialized, client); assertEquals(SCAN_REQUEST_TIMEOUT_MS, scanner.getScanRequestTimeout()); } } }
LOG.debug(" Predicates scheduled for this query are " + predicateList.size()); for ( int i = 0; i < allPossibleScanTokens.size(); i++) { LOG.debug("A tablet scheduled for all operators scanning is " + allPossibleScanTokens.get(i).getTablet()); aMetaForThisQuery.setSerializedKuduScanToken(aTokenForThisOperator.serialize()); returnOfAssignments.add(aMetaForThisQuery); LOG.debug("Added query scan for this operator " + aMetaForThisQuery + " with scan tablet as " + allPossibleScanTokens.get(aPartofThePie.getOrdinal()).getTablet());
KuduScanner scanner = tokens.get(0).intoScanner(client); assertEquals(0, scanner.nextRows().getNumRows()); LOG.debug(KuduScanToken.stringifySerializedToken(token.serialize(), client));
/** * Retrieves the partitions of a table. * * @param table the table * @return the partitions of the table */ private List<Partition> getTablePartitions(KuduTable table) { List<Partition> partitions = new ArrayList<>(); for (KuduScanToken token : client.newScanTokenBuilder(table).build()) { partitions.add(token.getTablet().getPartition()); } return partitions; }
@Test(timeout = 100000) public void testScanTokenReplicaSelections() throws Exception { ScanTokenPB.Builder pbBuilder = ScanTokenPB.newBuilder(); pbBuilder.setTableName(table.getName()); pbBuilder.setReplicaSelection(Common.ReplicaSelection.CLOSEST_REPLICA); Client.ScanTokenPB scanTokenPB = pbBuilder.build(); final byte[] serializedToken = KuduScanToken.serialize(scanTokenPB); // Deserialize the scan token into a scanner, and make sure it is using // 'CLOSEST_REPLICA' selection policy. KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, client); assertEquals(ReplicaSelection.CLOSEST_REPLICA, scanner.getReplicaSelection()); assertEquals(9, countRowsInScan(scanner)); }
/** * Serializes this {@code KuduScanToken} into a byte array. * @return the serialized scan token * @throws IOException */ public byte[] serialize() throws IOException { return serialize(message); }
public KuduScanner createScanner(KuduSplit kuduSplit) { try { return KuduScanToken.deserializeIntoScanner(kuduSplit.getSerializedScanToken(), client); } catch (IOException e) { throw new RuntimeException(e); } }
/** * Retrieves this table's range partitions. The range partitions will be returned * in sorted order by value, and will contain no duplicates. * * @param timeout the timeout of the operation * @return a list of the formatted range partitions */ @InterfaceAudience.Private @InterfaceStability.Unstable public List<Partition> getRangePartitions(long timeout) throws Exception { // TODO: This could be moved into the RangeSchemaPB returned from server // to avoid an extra call to get the range partitions. List<Partition> rangePartitions = new ArrayList<>(); List<KuduScanToken> scanTokens = new KuduScanToken.KuduScanTokenBuilder(client, this) .setTimeout(timeout) .build(); for (KuduScanToken token : scanTokens) { Partition partition = token.getTablet().getPartition(); // Filter duplicate range partitions by taking only the tablets whose hash // partitions are all 0s. if (!Iterators.all(partition.getHashBuckets().iterator(), Predicates.equalTo(0))) { continue; } rangePartitions.add(partition); } return rangePartitions; } }
@Test(timeout = 100000) public void testScanTokenPropagatesTimestamp() throws Exception { // Initially, the client does not have the timestamp set. assertEquals(AsyncKuduClient.NO_TIMESTAMP, asyncClient.getLastPropagatedTimestamp()); assertEquals(KuduClient.NO_TIMESTAMP, client.getLastPropagatedTimestamp()); AsyncKuduScanner scanner = asyncClient.newScannerBuilder(table).build(); KuduScanner syncScanner = new KuduScanner(scanner); // Let the client receive the propagated timestamp in the scanner response. syncScanner.nextRows().getNumRows(); final long tsPrev = asyncClient.getLastPropagatedTimestamp(); final long tsPropagated = tsPrev + 1000000; ScanTokenPB.Builder pbBuilder = ScanTokenPB.newBuilder(); pbBuilder.setTableName(table.getName()); pbBuilder.setPropagatedTimestamp(tsPropagated); Client.ScanTokenPB scanTokenPB = pbBuilder.build(); final byte[] serializedToken = KuduScanToken.serialize(scanTokenPB); // Deserialize scan tokens and make sure the client's last propagated // timestamp is updated accordingly. assertEquals(tsPrev, asyncClient.getLastPropagatedTimestamp()); KuduScanToken.deserializeIntoScanner(serializedToken, client); assertEquals(tsPropagated, asyncClient.getLastPropagatedTimestamp()); }
private KuduSplit toKuduSplit(KuduTableHandle tableHandle, KuduScanToken token, int primaryKeyColumnCount) { try { byte[] pb = token.serialize(); return new KuduSplit(tableHandle, primaryKeyColumnCount, pb); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public KuduScanner createScanner(KuduSplit kuduSplit) { try { KuduScanner scanner = KuduScanToken.deserializeIntoScanner(kuduSplit.getPb(), client); return scanner; } catch (IOException e) { throw new RuntimeException(e); } }