@ProcessElement public void processElement(ProcessContext c) { TableRow row = c.element(); String countryCode = (String) row.get("ActionGeo_CountryCode"); String sqlDate = (String) row.get("SQLDATE"); String actor1Name = (String) row.get("Actor1Name"); String sourceUrl = (String) row.get("SOURCEURL"); String eventInfo = "Date: " + sqlDate + ", Actor1: " + actor1Name + ", url: " + sourceUrl; c.output(KV.of(countryCode, eventInfo)); } }
@ProcessElement public void processElement(ProcessContext c) { TableRow row = c.element(); int timestamp; // TODO(BEAM-5390): Avoid this workaround. try { timestamp = ((BigDecimal) row.get("timestamp")).intValue(); } catch (ClassCastException e) { timestamp = ((Integer) row.get("timestamp")).intValue(); } String userName = (String) row.get("contributor_username"); if (userName != null) { // Sets the implicit timestamp field to be used in windowing. c.outputWithTimestamp(userName, new Instant(timestamp * 1000L)); } } }
@ProcessElement public void processElement(ProcessContext c) { TableRow row = c.element(); Double meanTemp = Double.parseDouble(row.get("mean_temp").toString()); c.output(meanTemp); } }
@ProcessElement public void processElement(ProcessContext c) { TableRow row = c.element(); Integer month; month = (Integer) row.get("month"); if (month.equals(this.monthFilter)) { c.output(row); } } }
@ProcessElement public void processElement(ProcessContext c) { TableRow row = c.element(); // Grab year, month, day, mean_temp from the row Integer year = Integer.parseInt((String) row.get("year")); Integer month = Integer.parseInt((String) row.get("month")); Integer day = Integer.parseInt((String) row.get("day")); Double meanTemp = Double.parseDouble(row.get("mean_temp").toString()); // Prepares the data for writing to BigQuery by building a TableRow object TableRow outRow = new TableRow() .set("year", year) .set("month", month) .set("day", day) .set("mean_temp", meanTemp); c.output(outRow); } }
@ProcessElement public void processElement(ProcessContext c) { TableRow row = c.element(); String playName = (String) row.get("corpus"); String word = (String) row.get("word"); if (word.length() >= MIN_WORD_LENGTH) { c.output(KV.of(word, playName)); } else { // Track how many smaller words we're not including. This information will be // visible in the Monitoring UI. smallerWords.inc(); } } }
@ProcessElement public void processElement(ProcessContext c) { Double meanTemp = Double.parseDouble(c.element().get("mean_temp").toString()); Double gTemp = c.sideInput(globalMeanTemp); if (meanTemp < gTemp) { c.output(c.element()); } } })
List<TableRow> convertStringsToLong(List<TableRow> toConvert) { // The numbers come back as String after JSON serialization. Change them back to // longs so that we can assert the output. List<TableRow> converted = Lists.newArrayList(); for (TableRow entry : toConvert) { TableRow convertedEntry = entry.clone(); convertedEntry.set("number", Long.parseLong((String) convertedEntry.get("number"))); converted.add(convertedEntry); } return converted; }
@ProcessElement public void processElement(ProcessContext c) throws Exception { TableRow row = c.element(); long timestamp = (Integer) row.get("timestamp"); String userName = (String) row.get("contributor_username"); if (userName != null) { // Sets the timestamp field to be used in windowing. c.outputWithTimestamp(userName, new Instant(timestamp * 1000L)); } } }))
@ProcessElement public void processElement(ProcessContext c) throws Exception { TableRow row = c.element(); long timestamp = (Integer) row.get("timestamp"); String userName = (String) row.get("contributor_username"); if (userName != null) { // Sets the timestamp field to be used in windowing. c.outputWithTimestamp(userName, new Instant(timestamp * 1000L)); } } }))
@ProcessElement public void processElement(ProcessContext c) throws Exception { TableRow row = c.element(); long timestamp = (Integer) row.get("timestamp"); String userName = (String) row.get("contributor_username"); if (userName != null) { // Sets the timestamp field to be used in windowing. c.outputWithTimestamp(userName, new Instant(timestamp * 1000L)); } } }))
/** * Convert TableRow to Entity * * @param row TableRow of bigquery * @return converted Entity * @throws ParseException */ public Entity convertTableRowToEntity(TableRow row) throws ParseException { String keyName = row.get(keyColumn).toString(); Key key = getKey(keyName); Entity.Builder builder = Entity.newBuilder().setKey(key); Set<Map.Entry<String, Object>> entries = row.entrySet(); for (Map.Entry<String, Object> entry : entries) { // Skip on the key column if (entry.getKey().equals(keyColumn)) { continue; } // Put a value in the builder String propertyName = entry.getKey(); Object value = entry.getValue(); Value v = convertToDatastoreValue(propertyName, value); if (v != null) { builder.putProperties(propertyName, v); } } return builder.build(); }
@ProcessElement public void processElement(ProcessContext c) throws Exception { TableRow element = c.element(); TableRow row = new TableRow() .set("trigger_type", element.get("trigger_type")) .set("freeway", element.get("freeway")) .set("total_flow", element.get("total_flow")) .set("number_of_records", element.get("number_of_records")) .set("isFirst", element.get("isFirst")) .set("isLast", element.get("isLast")) .set("timing", element.get("timing")) .set("window", element.get("window")); c.output(canonicalFormat(row)); } }
@Test public void testToTableRow_array_row() { TableRow row = toTableRow().apply(ARRAY_ROW_ROW); assertThat(row.size(), equalTo(1)); row = ((List<TableRow>) row.get("rows")).get(0); assertThat(row.size(), equalTo(5)); assertThat(row, hasEntry("id", 123L)); assertThat(row, hasEntry("value", 123.456)); assertThat(row, hasEntry("name", "test")); assertThat(row, hasEntry("valid", false)); }
@Test public void testToTableRow_row() { TableRow row = toTableRow().apply(ROW_ROW); assertThat(row.size(), equalTo(1)); row = (TableRow) row.get("row"); assertThat(row.size(), equalTo(5)); assertThat(row, hasEntry("id", 123L)); assertThat(row, hasEntry("value", 123.456)); assertThat(row, hasEntry("name", "test")); assertThat(row, hasEntry("valid", false)); }