public KuduRecordWriterImpl(OperatorContext context, KuduClient client, String name) { this.client = client; this.name = name; this.context = context; session = client.newSession(); session.setFlushMode(FlushMode.MANUAL_FLUSH); }
private KuduPageSink( ConnectorSession connectorSession, KuduClientSession clientSession, KuduTableHandle tableHandle, KuduTableMapping mapping) { requireNonNull(clientSession, "clientSession is null"); this.connectorSession = connectorSession; this.columnTypes = mapping.getColumnTypes(); this.originalColumnTypes = mapping.getOriginalColumnTypes(); this.generateUUID = mapping.isGenerateUUID(); this.table = tableHandle.getTable(clientSession); this.session = clientSession.newSession(); this.session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); uuid = UUID.randomUUID().toString(); }
protected KuduSession getKuduSession(final KuduClient client) { final KuduSession kuduSession = client.newSession(); kuduSession.setMutationBufferSpace(batchSize); kuduSession.setFlushMode(flushMode); if (operationType == OperationType.INSERT_IGNORE) { kuduSession.setIgnoreAllDuplicateRows(true); } return kuduSession; }
@Override public void init() throws DBException { this.tableName = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); initClient(); this.session = client.newSession(); if (getProperties().getProperty(SYNC_OPS_OPT) != null && getProperties().getProperty(SYNC_OPS_OPT).equals("false")) { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND); this.session.setMutationBufferSpace( getIntFromProp(getProperties(), BUFFER_NUM_OPS_OPT, BUFFER_NUM_OPS_DEFAULT)); } else { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_SYNC); } try { this.kuduTable = client.openTable(tableName); this.schema = kuduTable.getSchema(); } catch (Exception e) { throw new DBException("Could not open a table because of:", e); } }
private void createAndFillSchemasTable(KuduClient client) throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(client); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); try { session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); insert.getRow().addString(0, schemaName); session.apply(insert); } } finally { session.close(); } }
@Override public void deleteRows(Block rowIds) { Schema schema = table.getSchema(); KuduSession session = clientSession.newSession(); session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND); try { try { for (int i = 0; i < rowIds.getPositionCount(); i++) { int len = rowIds.getSliceLength(i); Slice slice = rowIds.getSlice(i, 0, len); PartialRow row = KeyEncoderAccessor.decodePrimaryKey(schema, slice.getBytes()); Delete delete = table.newDelete(); RowHelper.copyPrimaryKey(schema, row, delete.getRow()); session.apply(delete); } } finally { session.close(); } } catch (KuduException e) { throw new RuntimeException(e); } }
session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC); for (int i = 0; i < rows; i++) { Insert insert = table.newInsert();
public KuduPageSink(ConnectorSession connectorSession, KuduClientSession clientSession, KuduExtendedTableHandle extendedTableHandle, boolean generateUUID) { requireNonNull(clientSession, "clientSession is null"); this.connectorSession = connectorSession; this.columnTypes = extendedTableHandle.getColumnTypes(); this.originalColumnTypes = extendedTableHandle.getOriginalColumnTypes(); this.generateUUID = generateUUID; this.table = extendedTableHandle.getTable(clientSession); this.session = clientSession.newSession(); this.session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); uuid = UUID.randomUUID().toString(); }
private KuduPageSink( ConnectorSession connectorSession, KuduClientSession clientSession, KuduTableHandle tableHandle, KuduTableMapping mapping) { requireNonNull(clientSession, "clientSession is null"); this.connectorSession = connectorSession; this.columnTypes = mapping.getColumnTypes(); this.originalColumnTypes = mapping.getOriginalColumnTypes(); this.generateUUID = mapping.isGenerateUUID(); this.table = tableHandle.getTable(clientSession); this.session = clientSession.newSession(); this.session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); uuid = UUID.randomUUID().toString(); }
KuduConnection(Config config, byte[] token) { client = new KuduClient.KuduClientBuilder(config.getString(CONNECTION_CONFIG_NAME)).build(); if (token != null) { client.importAuthenticationCredentials(token); } session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); session.setMutationBufferSpace(10000); session.setIgnoreAllDuplicateRows(KuduUtils.doesInsertIgnoreDuplicates(config)); }
/** * Regression test for KUDU-1402. Calls to session.flush() should return an empty list * instead of null. * @throws Exception */ @Test(timeout = 100000) public void testEmptyFlush() throws Exception { KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); List<OperationResponse> result = session.flush(); assertNotNull(result); assertTrue(result.isEmpty()); }
@Override public void run() { try { KuduTable table = localClient.openTable(TABLE_NAME); KuduSession session = localClient.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC); for (int i = 0; i < 100; i++) { Insert insert = createBasicSchemaInsert(table, id * 100 + i); session.apply(insert); } session.close(); } catch (Exception e) { fail("insert thread should not throw exception: " + e); } } });
/** * Insert rows into the provided table. The table's columns must be ints, and * must have a primary key in the first column. * @param table the table * @param start the inclusive start key * @param end the exclusive end key */ private void insertRows(KuduTable table, int start, int end) throws KuduException { KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (int i = start; i < end; i++) { Insert insert = table.newInsert(); for (int idx = 0; idx < table.getSchema().getColumnCount(); idx++) { insert.getRow().addInt(idx, i); } session.apply(insert); } session.flush(); RowError[] rowErrors = session.getPendingErrors().getRowErrors(); assertEquals(String.format("row errors: %s", Arrays.toString(rowErrors)), 0, rowErrors.length); }
@Test(timeout = 10000) public void testInsertAutoFlushSyncNonCoveredRange() throws Exception { CreateTableOptions createOptions = getBasicTableOptionsWithNonCoveredRange(); createOptions.setNumReplicas(1); client.createTable(tableName, basicSchema, createOptions); KuduTable table = client.openTable(tableName); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC); List<Integer> nonCoveredKeys = ImmutableList.of(350, 300, 199, 150, 100, -1, -50); for (int key : nonCoveredKeys) { OperationResponse response = session.apply(createBasicSchemaInsert(table, key)); assertTrue(response.hasRowError()); assertTrue(response.getRowError().getErrorStatus().isNotFound()); } }
/** * Test collecting errors from multiple tablets. * @throws Exception */ @Test(timeout = 100000) public void multiTabletTest() throws Exception { String tableName = TestRowErrors.class.getName() + "-" + System.currentTimeMillis(); createFourTabletsTableWithNineRows(harness.getAsyncClient(), tableName, DEFAULT_SLEEP); table = harness.getClient().openTable(tableName); KuduSession session = harness.getClient().newSession(); session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND); int dupRows = 3; session.apply(createInsert(12)); session.apply(createInsert(22)); session.apply(createInsert(32)); session.flush(); RowErrorsAndOverflowStatus reos = session.getPendingErrors(); assertEquals(dupRows, reos.getRowErrors().length); assertEquals(0, session.countPendingErrors()); }
/** * Creates a local aclient that we auto-close while buffering one row, then makes sure that after * closing that we can read the row. */ @Test(timeout = 100000) public void testAutoClose() throws Exception { try (KuduClient localClient = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()).build()) { localClient.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); KuduTable table = localClient.openTable(TABLE_NAME); KuduSession session = localClient.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); Insert insert = createBasicSchemaInsert(table, 0); session.apply(insert); } KuduTable table = client.openTable(TABLE_NAME); AsyncKuduScanner scanner = new AsyncKuduScanner.AsyncKuduScannerBuilder(asyncClient, table).build(); assertEquals(1, countRowsInScan(scanner)); }
@Test(timeout = 100000) public void testBasicOps() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } assertEquals(10, countRowsInScan(client.newScannerBuilder(table).build())); OperationResponse resp = session.apply(createInsert(table, 0)); assertTrue(resp.hasRowError()); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); for (int i = 10; i < 20; i++) { session.apply(createInsert(table, i)); } session.flush(); assertEquals(20, countRowsInScan(client.newScannerBuilder(table).build())); }
@Test public void testBytePredicates() throws Exception { Schema schema = createTableSchema(Type.INT8); client.createTable("byte-table", schema, createTableOptions()); KuduTable table = client.openTable("byte-table"); NavigableSet<Long> values = createIntegerValues(Type.INT8); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); long i = 0; for (long value : values) { Insert insert = table.newInsert(); insert.getRow().addLong("key", i++); insert.getRow().addByte("value", (byte) value); session.apply(insert); } Insert nullInsert = table.newInsert(); nullInsert.getRow().addLong("key", i++); nullInsert.getRow().setNull("value"); session.apply(nullInsert); session.flush(); checkIntPredicates(table, values, createIntegerTestValues(Type.INT8)); }
@Test public void testIntPredicates() throws Exception { Schema schema = createTableSchema(Type.INT32); client.createTable("int-table", schema, createTableOptions()); KuduTable table = client.openTable("int-table"); NavigableSet<Long> values = createIntegerValues(Type.INT32); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); long i = 0; for (long value : values) { Insert insert = table.newInsert(); insert.getRow().addLong("key", i++); insert.getRow().addInt("value", (int) value); session.apply(insert); } Insert nullInsert = table.newInsert(); nullInsert.getRow().addLong("key", i++); nullInsert.getRow().setNull("value"); session.apply(nullInsert); session.flush(); checkIntPredicates(table, values, createIntegerTestValues(Type.INT32)); }
@Test public void testTimestampPredicate() throws Exception { Schema schema = createTableSchema(Type.INT64); client.createTable("timestamp-table", schema, createTableOptions()); KuduTable table = client.openTable("timestamp-table"); NavigableSet<Long> values = createIntegerValues(Type.INT64); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); long i = 0; for (long value : values) { Insert insert = table.newInsert(); insert.getRow().addLong("key", i++); insert.getRow().addLong("value", value); session.apply(insert); } Insert nullInsert = table.newInsert(); nullInsert.getRow().addLong("key", i++); nullInsert.getRow().setNull("value"); session.apply(nullInsert); session.flush(); checkIntPredicates(table, values, createIntegerTestValues(Type.INT64)); }