return bigquery.insertAll(InsertAllRequest.of(getTableId(), rows));
rowContent.put("recordField", recordsContent); InsertAllResponse response = bigquery.insertAll( InsertAllRequest.newBuilder(tableId) .addRow("rowId", rowContent)
@Test public void testInsertComplete() throws Exception { initializeExpectedTable(1); expect(bigquery.getOptions()).andReturn(mockOptions); expect(bigquery.insertAll(INSERT_ALL_REQUEST_COMPLETE)).andReturn(EMPTY_INSERT_ALL_RESPONSE); replay(bigquery); initializeTable(); InsertAllResponse response = table.insert(ROWS_TO_INSERT, true, true); assertSame(EMPTY_INSERT_ALL_RESPONSE, response); }
@Test public void testInsert() throws Exception { initializeExpectedTable(1); expect(bigquery.getOptions()).andReturn(mockOptions); expect(bigquery.insertAll(INSERT_ALL_REQUEST)).andReturn(EMPTY_INSERT_ALL_RESPONSE); replay(bigquery); initializeTable(); InsertAllResponse response = table.insert(ROWS_TO_INSERT); assertSame(EMPTY_INSERT_ALL_RESPONSE, response); }
InsertAllRequest.newBuilder(tableId).addRow(firstRow).addRow(secondRow).build(); InsertAllResponse insertResponse = bigquery.insertAll(insertRequest);
.setIgnoreUnknownValues(ignoreUnknownValues) .build(); return bigquery.insertAll(request);
.addRow(builder2.build()) .build(); InsertAllResponse response = bigquery.insertAll(request); assertFalse(response.hasErrors()); assertEquals(0, response.getInsertErrors().size());
.setSkipInvalidRows(true) .build(); InsertAllResponse response = bigquery.insertAll(request); assertTrue(response.hasErrors()); assertEquals(2, response.getInsertErrors().size());
.setTemplateSuffix("_suffix") .build(); InsertAllResponse response = bigquery.insertAll(request); assertFalse(response.hasErrors()); assertEquals(0, response.getInsertErrors().size());
createBigQueryOptionsForProject(OTHER_PROJECT, rpcFactoryMock); bigquery = bigQueryOptions.getService(); InsertAllResponse response = bigquery.insertAll(request); assertNotNull(response.getErrorsFor(0L)); assertNull(response.getErrorsFor(1L));
createBigQueryOptionsForProject(OTHER_PROJECT, rpcFactoryMock); bigquery = bigQueryOptions.getService(); InsertAllResponse response = bigquery.insertAll(request); assertNotNull(response.getErrorsFor(0L)); assertNull(response.getErrorsFor(1L));
.getService(); thrown.expect(BigQueryException.class); bigquery.insertAll(request);
.build() .getService(); InsertAllResponse response = bigquery.insertAll(request); assertNotNull(response.getErrorsFor(0L)); assertNull(response.getErrorsFor(1L));
/** * Insert rows into the table. * * @param rows rows to be inserted * @throws BigQueryException upon failure */ public InsertAllResponse insert(Iterable<InsertAllRequest.RowToInsert> rows) throws BigQueryException { return bigquery.insertAll(InsertAllRequest.of(tableId(), rows)); }
/** Inserts multiple rows of the same schema to a BigQuery table. */ public void insertAll(Collection<Map<String, ?>> rows, String table) { TableId tableId = TableId.of(projectId, dataset, table); InsertAllRequest.Builder builder = InsertAllRequest.newBuilder(tableId); for (Map<String, ?> row : rows) { builder.addRow(row); } InsertAllResponse response = client.insertAll(builder.build()); handleBigQueryResponseExceptions(response); }
@Test public void testSimplePut() { final String topic = "test-topic"; Map<String, String> properties = propertiesFactory.getProperties(); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, ".*=scratch"); BigQuery bigQuery = mock(BigQuery.class); Storage storage = mock(Storage.class); SinkTaskContext sinkTaskContext = mock(SinkTaskContext.class); InsertAllResponse insertAllResponse = mock(InsertAllResponse.class); when(bigQuery.insertAll(anyObject())).thenReturn(insertAllResponse); when(insertAllResponse.hasErrors()).thenReturn(false); BigQuerySinkTask testTask = new BigQuerySinkTask(bigQuery, null, storage); testTask.initialize(sinkTaskContext); testTask.start(properties); testTask.put(Collections.singletonList(spoofSinkRecord(topic))); testTask.flush(Collections.emptyMap()); verify(bigQuery, times(1)).insertAll(any(InsertAllRequest.class)); }
@Test public void testSimplePutWhenSchemaRetrieverIsNotNull() { final String topic = "test-topic"; Map<String, String> properties = propertiesFactory.getProperties(); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, ".*=scratch"); BigQuery bigQuery = mock(BigQuery.class); Storage storage = mock(Storage.class); SinkTaskContext sinkTaskContext = mock(SinkTaskContext.class); InsertAllResponse insertAllResponse = mock(InsertAllResponse.class); when(bigQuery.insertAll(anyObject())).thenReturn(insertAllResponse); when(insertAllResponse.hasErrors()).thenReturn(false); SchemaRetriever schemaRetriever = mock(SchemaRetriever.class); BigQuerySinkTask testTask = new BigQuerySinkTask(bigQuery, schemaRetriever, storage); testTask.initialize(sinkTaskContext); testTask.start(properties); testTask.put(Collections.singletonList(spoofSinkRecord(topic))); testTask.flush(Collections.emptyMap()); verify(bigQuery, times(1)).insertAll(any(InsertAllRequest.class)); verify(schemaRetriever, times(1)).setLastSeenSchema(any(TableId.class), any(String.class), any(Schema.class)); }
/** * Insert rows into the table. * * @param rows rows to be inserted * @param skipInvalidRows whether to insert all valid rows, even if invalid rows exist. If not set * the entire insert operation will fail if rows to be inserted contain an invalid row * @param ignoreUnknownValues whether to accept rows that contain values that do not match the * schema. The unknown values are ignored. If not set, rows with unknown values are considered * to be invalid * @throws BigQueryException upon failure */ public InsertAllResponse insert(Iterable<InsertAllRequest.RowToInsert> rows, boolean skipInvalidRows, boolean ignoreUnknownValues) throws BigQueryException { InsertAllRequest request = InsertAllRequest.builder(tableId(), rows) .skipInvalidRows(skipInvalidRows) .ignoreUnknownValues(ignoreUnknownValues) .build(); return bigquery.insertAll(request); }
@Test(expected = ConnectException.class) public void testPutWhenPartitioningOnMessageTimeWhenNoTimestampType() { final String topic = "test-topic"; Map<String, String> properties = propertiesFactory.getProperties(); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, ".*=scratch"); properties.put(BigQuerySinkTaskConfig.BIGQUERY_MESSAGE_TIME_PARTITIONING_CONFIG, "true"); BigQuery bigQuery = mock(BigQuery.class); Storage storage = mock(Storage.class); SinkTaskContext sinkTaskContext = mock(SinkTaskContext.class); InsertAllResponse insertAllResponse = mock(InsertAllResponse.class); when(bigQuery.insertAll(anyObject())).thenReturn(insertAllResponse); when(insertAllResponse.hasErrors()).thenReturn(false); BigQuerySinkTask testTask = new BigQuerySinkTask(bigQuery, null, storage); testTask.initialize(sinkTaskContext); testTask.start(properties); testTask.put(Collections.singletonList(spoofSinkRecord(topic, "value", "message text", TimestampType.NO_TIMESTAMP_TYPE, null))); }
@Test(expected = RejectedExecutionException.class) public void testStop() { final String dataset = "scratch"; final String topic = "test_topic"; Map<String, String> properties = propertiesFactory.getProperties(); properties.put(BigQuerySinkConfig.TOPICS_CONFIG, topic); properties.put(BigQuerySinkConfig.DATASETS_CONFIG, String.format(".*=%s", dataset)); BigQuery bigQuery = mock(BigQuery.class); SinkTaskContext sinkTaskContext = mock(SinkTaskContext.class); InsertAllResponse insertAllResponse = mock(InsertAllResponse.class); when(bigQuery.insertAll(anyObject())).thenReturn(insertAllResponse); when(insertAllResponse.hasErrors()).thenReturn(false); Storage storage = mock(Storage.class); BigQuerySinkTask testTask = new BigQuerySinkTask(bigQuery, null, storage); testTask.initialize(sinkTaskContext); testTask.start(properties); testTask.put(Collections.singletonList(spoofSinkRecord(topic))); assertEquals(1, testTask.getTaskThreadsActiveCount()); testTask.stop(); assertEquals(0, testTask.getTaskThreadsActiveCount()); verify(bigQuery, times(1)).insertAll(any(InsertAllRequest.class)); testTask.put(Collections.singletonList(spoofSinkRecord(topic))); }