@Override public RecordSet getRecords() throws MalformedSourceException { RecordSet ret = new RecordSet(); for (DataLoader loader : dataLoaders_) { RecordSet cSet = loader.getRecords(); ret.addAll(cSet); } return ret; }
private DataLoadingSpec generateNextLoadingSpec(TransformationSpec spec, int noOfRecordsWeAlreadyHave) { SimpleDataLoadingSpec ret = new SimpleDataLoadingSpec(); ret.setNumberOfRecords(spec.getNumberOfRecords()-noOfRecordsWeAlreadyHave); ret.setOffset(spec.getOffset() + current_offset); ret.setDataSetName(spec.getDataSetName()); ret.setFromDate(spec.getFromDate()); ret.setUntilDate(spec.getUntilDate()); ret.setIdentifier(spec.getIdentifier()); return ret; }
@Override public RecordSet run(RecordSet records) { RecordSet recs = records; for (ProcessingStep step : process) { recs = step.execute(recs); } return recs; }
/** * Executes this {@link ProcessingStep}. * <br> * <br> * For each {@link Record} contained in the {@link RecordSet} * given as argument, converts it to {@link MutableRecord} calls * {@link AbstractModifier#modify}, and adds the modified record * in the return result. * * @see ProcessingStep#execute * * @param recs The initial set of records * @return A {@link RecordSet} containing the modified records */ @Override public RecordSet execute(RecordSet recs) { RecordSet ret = new RecordSet(); Record modified_record; for (Record rec : recs) { modified_record = this.modify(rec.makeMutable()); ret.addRecord(modified_record); } return recs; }
@Override public Record modify(MutableRecord rec) { List<Value> values = rec.getValues(field); if (values != null) { List<String> converted_values = new ArrayList<String>(); for (Value val : values) { converted_values.add(val.getAsString()); } List<Value> final_value = new ArrayList<Value>(); String v = StringUtils.join(converted_values.iterator(), separator + (whitespaceAfter ? " " : "")); final_value.add(new StringValue(v)); rec.updateField(field, final_value); } return rec; }
public static String getPrintableString(Record record) { StringBuilder result = new StringBuilder(); result.append("\nPublication {\n"); for (String field : record.getFields()) { result.append("--" + field + ":\n"); List<Value> values = record.getValues(field); for (Value value : values) { result.append("\t" + value.getAsString() + "\n"); } } result.append("}\n"); return result.toString(); } }
/** * Executes this {@link ProcessingStep}. * <br> * <br> * For each {@link Record} contained in the {@link RecordSet} * given as argument, if {@link AbstractFilter#isIncluded} returns * <code>true</code> the Record is included in the return result, * otherwise it is ignored. * * @see ProcessingStep#execute * * @param recs The initial set of records * @return The resulting set after the filter is applied */ @Override public RecordSet execute(RecordSet recs) { RecordSet ret = new RecordSet(); for (Record rec : recs) { if (this.isIncluded(rec)) { ret.addRecord(rec); } } return ret; }
private boolean checkSpec(TransformationSpec spec) { // The spec must have EITHER an id OR other info. if (spec.getIdentifier() != null && (spec.getNumberOfRecords() != 1 || spec.getOffset() != 0 || spec.getDataSetName() != null || spec.getFromDate() != null || spec.getUntilDate() != null)) { return false; } return true; } }
public static String getFirstValue(Record rec, String field) { List<Value> values = rec.getValues(field); String value = null; if (values != null && values.size() > 0) { value = values.get(0).getAsString(); } return value; }
@Override public boolean equals(Object other) { Value val = (Value)other; return value.equals(val.getAsString()); }
@Override public Record modify(MutableRecord rec) { List<Value> old_values = rec.getValues("language"); if (old_values == null || old_values.size() == 0) { return rec; } List<Value> new_values = new ArrayList<Value>(); for (Value value : old_values) { String lang3 = value.getAsString(); String lang2 = lang3.length() == 3 ? getLang2(lang3) : lang3; new_values.add(new StringValue(lang2)); } rec.updateField("language", new_values); return rec; }
@Override public List<String> generateOutput(RecordSet recs) { System.out.println("=================Records================="); for (Record rec : recs) { System.out.println("--------------Record--------------"); for (String str : rec.getFields()) { List<Value> vals = rec.getValues(str); if (vals != null) { System.out.println(str + ": "); for (Value val : vals) { System.out.println("----: " + val.getAsString()); } } } } System.out.println("=================Records end================="); return new ArrayList<String>(); }
public static List<String> getValues(Record rec, String field) { List<String> result = new ArrayList<String>(); List<Value> values = rec.getValues(field); if (values != null && values.size() > 0) { for (Value value : values) { result.add(value.getAsString()); } } return result; }