@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { List<Row> results = new ArrayList<>(); for (Row row : rows) { int idx1 = row.find(col1); int idx2 = row.find(col2); if (idx1 != -1 && idx2 != -1) { StringBuilder builder = new StringBuilder(); builder.append(row.getValue(idx1)); builder.append(delimiter); builder.append(row.getValue(idx2)); row.add(dest, builder.toString()); } results.add(row); } return results; } }
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { List<Row> results = new ArrayList<>(); for (Row row : rows) { Row masked = new Row(row); int idx = row.find(column); if (idx != -1) { masked.setValue(idx, maskShuffle((String) row.getValue(idx), 0)); } else { throw new DirectiveExecutionException(toString() + " : '" + column + "' column is not defined. Please check the wrangling step." ); } results.add(masked); } return results; }
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { for (Row row : rows) { for (int i = 0; i < row.length(); ++i) { String name = row.getColumn(i); if (toLower) { row.setColumn(i, name.toLowerCase()); } else { row.setColumn(i, name.toUpperCase()); } } } return rows; } }
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { for (Row row : rows) { int idx = row.find(column); if (idx == -1) { row.add(column, value); continue; } Object object = row.getValue(idx); if (object == null) { row.setValue(idx, value); } else { if (object instanceof String) { if (((String) object).isEmpty()) { row.setValue(idx, value); } } else if (object instanceof JSONObject) { if (JSONObject.NULL.equals(object)) { row.setValue(idx, value); } } } } return rows; } }
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { for (Row row : rows) { int idx = row.find(column); if (idx != -1) { Object object = row.getValue(idx); if (object instanceof String) { if (object != null) { String value = (String) object; row.setValue(idx, value.toLowerCase()); } } } } return rows; } }
List<Row> results = new ArrayList<>(); for (Row row : rows) { int idx = row.find(col); if (idx != -1) { Object object = row.getValue(idx); if (object instanceof String) { String data = (String) object; Row newRow = new Row(row); int recPosition = index; int colid = 1; val = val.replaceAll(padding, ""); newRow.add(String.format("%s_%d", col, colid), val); recPosition += width; colid+=1;
locations[count] = row.find(column); ++count; for (int i = 0; i < count; ++i) { if (locations[i] != -1) { Object value = row.getValue(locations[i]); int m = -1; if (value instanceof JsonArray) { Row r = new Row(row); for (int i = 0; i < count; ++i) { if (locations[i] != -1) { Object value = row.getValue(locations[i]); if (value == null) { r.add(columns[i], null); } else { Object v = null; r.addOrSet(columns[i], null); } else { if (v instanceof JsonElement) { r.setValue(locations[i], JsParser.getValue((JsonElement)v)); } else { r.setValue(locations[i], v); r.addOrSet(columns[i], null);
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException, ErrorRowException { for (Row row : rows) { int idx = row.find(source); if (idx != -1) { Object object = row.getValue(idx); if (object == null || !(object instanceof Double)) { continue; } Double value = (Double) object; String format = this.format.format(value); row.addOrSet(destination, format); } } return rows; } }
for (Row row : rows) { ++count; for (int i = 0; i < row.length(); ++i) { String column = row.getColumn(i); Object object = row.getValue(i); Row rowTypes = new Row(); for (String column : types.getColumns()) { rowTypes.add(column, types.percentage(column, count)); Row rowStats = new Row(); for (String column : stats.getColumns()) { rowStats.add(column, stats.percentage(column, count)); Row row = new Row(); row.add("types", rowTypes); row.add("stats", rowStats); row.add("total", count);
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { List<Row> results = new ArrayList<>(); for (Row row : rows) { boolean missingOrNull = true; for (String column : columns) { int idx = row.find(column.trim()); if (idx != -1) { Object value = row.getValue(idx); if (value != null) { missingOrNull = false; } } else { results.add(row); } } if (!missingOrNull) { results.add(row); } } return results; } }
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException, ErrorRowException { List<Row> results = new ArrayList<>(); for (Row row : rows) { int idx = row.find(column); if (idx == -1) { continue; } Object object = row.getValue(idx); if (object instanceof String) { String body = (String) object; String[] lines = body.split(delimiter); int i = 0; for (String line : lines) { if (i > limit) { break; } results.add(new Row(column, line)); i++; } } } return results; } }
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { for (Row row : rows) { try { final ByteArrayOutputStream bOut = new ByteArrayOutputStream(); try(Writer out = new BufferedWriter(new OutputStreamWriter(bOut))) { CSVPrinter csvPrinter = new CSVPrinter(out, CSVFormat.DEFAULT); for (int i = 0; i < row.length(); ++i) { csvPrinter.print(row.getValue(i)); } csvPrinter.flush(); csvPrinter.close(); } catch (Exception e) { bOut.close(); } row.add(column, bOut.toString()); } catch (IOException e) { throw new DirectiveExecutionException(toString() + " : Failed to write CSV row. " + e.getMessage()); } } return rows; } }
@Override public List<Row> execute(List<Row> rows, ExecutorContext context) throws DirectiveExecutionException { for (Row row : rows) { int idx = row.find(col); if (idx != -1) { Object object = row.getValue(idx); if (object == null) { throw new DirectiveExecutionException(toString() + " : Did not find '" + col + "' in the row."); } try { if (object instanceof String) { JsonObject element = gson.fromJson(XML.toJSONObject((String) object).toString(), JsonElement.class).getAsJsonObject(); JsParser.jsonFlatten(element, col, 1, depth, row); row.remove(idx); } else { throw new DirectiveExecutionException( String.format("%s : Invalid type '%s' of column '%s'. Should be of type String.", toString(), col, object != null ? object.getClass().getName() : "null") ); } } catch (JSONException e) { throw new DirectiveExecutionException(toString() + " : " + e.getMessage()); } } } return rows; }
/** * Flattens the {@link GenericRecord}. * * @param genericRecord to be flattened. * @param row to be flattened into * @param name of the field to be flattened. */ private void add(GenericRecord genericRecord, Row row, String name) { List<Schema.Field> fields = genericRecord.getSchema().getFields(); String colname; for (Schema.Field field : fields) { Object v = genericRecord.get(field.name()); if (name != null) { colname = String.format("%s_%s", name, field.name()); } else { colname = field.name(); } if (v instanceof GenericRecord) { add((GenericRecord) v, row, colname); } else if (v instanceof Map || v instanceof List) { row.add(colname, gson.toJson(v)); } else if (v instanceof Utf8) { row.add(colname, v.toString()); } else { row.add(colname, genericRecord.get(field.name())); } } } }
/** * Adds or sets the value to the beginning. * * @param index at which the column need to be inserted. * @param name of the field to be either set or added to record. * @param value to be added. */ public void addOrSetAtIndex(int index, String name, Object value) { int idx = find(name); if (idx != -1) { setValue(idx, value); } else { if (index < columns.size() && index < values.size()) { columns.add(index, name); values.add(index, value); } } } }