@Override public int length() { return internalRow.length(); }
private List<List<Object>> convertRowsToObjects(Row[] rows) { List<List<Object>> result = new ArrayList<>(); List<Object> objects; for (Row row : rows) { objects = new ArrayList<>(); for (int i = 0; i < row.length(); i++) { objects.add(row.get(i)); } // Set<PosixFilePermission> perms = new HashSet<>(); result.add(objects); } return result; }
public static Map<String, Object> convertRowAndSchemaToValuesMap(Row row, StructType schema) { String[] colNames = schema.fieldNames(); Map<String, Object> result = new HashMap<>(); for (int i = 0; i < row.length(); i++) { result.put(colNames[i], row.get(i)); } return result; }
@SuppressWarnings({ "unchecked", "rawtypes" }) @Override public int compare(Row r1, Row r2) { for (int i = 0; i < r1.length(); i++) { Comparable r1FieldValue = (Comparable)r1.get(i); Comparable r2FieldValue = (Comparable)r2.get(i); int comparison = r1FieldValue.compareTo(r2FieldValue); if (comparison != 0) { return comparison; } } return 0; } }
private MinValuesWithPath generateGafferObjectsIndex(final Row minRow, final String path) throws StoreException { final int numOfCols = minRow.length(); final Object[] min = new Object[numOfCols]; for (int i = 0; i < numOfCols; i++) { min[i] = minRow.get(i); } return new MinValuesWithPath(min, path); } }
StructType objStructType = inputDataFrame.schema(); StructField []arrStructField=objStructType.fields(); List<StructField> fields = new ArrayList<StructField>(); List<StructField> newfields = new ArrayList<StructField>(); List <StructField> listFields = Arrays.asList(arrStructField); StructField a = DataTypes.createStructField(leftCol,DataTypes.StringType, true); fields.add(a); newfields.addAll(listFields); newfields.addAll(fields); final int size = objStructType.size(); JavaRDD<Row> rowRDD = inputDataFrame.javaRDD().map(new Function<Row, Row>() { private static final long serialVersionUID = 3280804931696581264L; public Row call(Row tblRow) throws Exception { Object[] newRow = new Object[size+1]; int rowSize= tblRow.length(); for (int itr = 0; itr < rowSize; itr++) { if(tblRow.apply(itr)!=null) { newRow[itr] = tblRow.apply(itr); } } newRow[size] = UUID.randomUUID().toString(); return RowFactory.create(newRow); } }); inputDataFrame = objsqlContext.createDataFrame(rowRDD, DataTypes.createStructType(newfields));
private static double[] transformRow2DataInstance(Row row, Attributes attributes) throws Exception { double[] instance = new double[row.length()]; for (int i = 0; i < row.length(); i++) { Attribute att = attributes.getFullListOfAttributes().get(i); StateSpaceType space = att.getStateSpaceType(); switch (space.getStateSpaceTypeEnum()) { case REAL: instance[i] = row.getDouble(i); break; case FINITE_SET: String state = row.getString(i); double index = ((FiniteStateSpace) space).getIndexOfState(state); instance[i] = index; break; default: // This should never execute throw new Exception("Unrecognized Error"); } } return instance; }
private Record convertRowAndSchemaToRecord(Row row, StructType schema, boolean global) { String[] colNames = schema.fieldNames(); Map<String, Object> result = new HashMap<>(); long timestamp = -1; int targetTenantId = this.tId; boolean globalTenantProcessed = false; for (int i = 0; i < row.length(); i++) { if (colNames[i].equals(AnalyticsConstants.TIMESTAMP_FIELD)) { timestamp = row.getLong(i); } else if (global && colNames[i].equals(AnalyticsConstants.TENANT_ID_FIELD)) { targetTenantId = row.getInt(i); globalTenantProcessed = true; } else { result.put(colNames[i], row.get(i)); } } if (global && !globalTenantProcessed) { throw new RuntimeException("The field '" + AnalyticsConstants.TENANT_ID_FIELD + "' is not found in row: " + row + " with schema: " + schema + " when creating a global tenant access record"); } if (timestamp < 0) { // timestamp has not being set return new Record(targetTenantId, this.tName, result); } else { return new Record(targetTenantId, this.tName, result, timestamp); } }
public static Row set(Row row, String fieldName, Object replacement) { Object[] values = new Object[row.length()]; for (int i = 0; i < row.schema().fields().length; i++) { if (i == row.fieldIndex(fieldName)) { values[i] = replacement; } else { values[i] = row.get(i); } } return new RowWithSchema(row.schema(), values); }
@Test public void testGetRowFromCells() { Cells cells = new Cells("namespace", Cell.create("cell1", 1), Cell.create("cell2", "test"), Cell.create("cell3", new ArrayList<String>())); Row row = CellsUtils.getRowFromCells(cells); assertEquals(row.length(), cells.size()); assertEquals(row.get(0), 1); assertEquals(row.get(1), "test"); assertEquals(row.get(2), new ArrayList<String>()); }
Row row = (Row) item; for (int i = 0; i < row.length(); i++) { Object value = row.get(i);
@Test public void testSubsetRowNoFields() { StructField field1 = DataTypes.createStructField("field1", DataTypes.StringType, true); StructField field2 = DataTypes.createStructField("field2", DataTypes.IntegerType, true); StructField field3 = DataTypes.createStructField("field3", DataTypes.FloatType, true); StructType schema = DataTypes.createStructType(Lists.newArrayList(field1, field2, field3)); StructType subsetSchema = DataTypes.createStructType(Lists.<StructField>newArrayList()); Row row = new RowWithSchema(schema, "hello", 1, 2.0); Row subsetRow = RowUtils.subsetRow(row, subsetSchema); assertEquals(subsetRow.length(), 0); }
@Test public void testSubsetRowSomeFields() { StructField field1 = DataTypes.createStructField("field1", DataTypes.StringType, true); StructField field2 = DataTypes.createStructField("field2", DataTypes.IntegerType, true); StructField field3 = DataTypes.createStructField("field3", DataTypes.FloatType, true); StructType schema = DataTypes.createStructType(Lists.newArrayList(field1, field2, field3)); StructType subsetSchema = DataTypes.createStructType(Lists.newArrayList(field1, field3)); Row row = new RowWithSchema(schema, "hello", 1, 2.0); Row subsetRow = RowUtils.subsetRow(row, subsetSchema); assertEquals(subsetRow.length(), 2); assertEquals(subsetRow.get(0), "hello"); assertEquals(subsetRow.get(1), 2.0); }
@Test public void testSet() { StructField field1 = DataTypes.createStructField("field1", DataTypes.StringType, true); StructField field2 = DataTypes.createStructField("field2", DataTypes.IntegerType, true); StructField field3 = DataTypes.createStructField("field3", DataTypes.FloatType, true); StructType schema = DataTypes.createStructType(Lists.newArrayList(field1, field2, field3)); Row row = new RowWithSchema(schema, "hello", 1, 2.0); Row setRow = RowUtils.set(row, "field2", 100); setRow = RowUtils.set(setRow, "field1", "world"); assertEquals(setRow.length(), 3); assertEquals(RowUtils.get(setRow, "field1"), "world"); assertEquals(RowUtils.get(setRow, "field2"), 100); assertEquals(RowUtils.get(setRow, "field3"), 2.0); }
@Test public void testAppendWithoutSchema() { Row row = RowFactory.create("hello", 1, true); Row appendRow = RowUtils.append(row, -50.0); assertEquals(appendRow.length(), 4); assertEquals(appendRow.get(0), "hello"); assertEquals(appendRow.get(1), 1); assertEquals(appendRow.get(2), true); assertEquals(appendRow.get(3), -50.0); }
@Test public void testAppendWithSchema() { StructField field1 = DataTypes.createStructField("field1", DataTypes.StringType, true); StructField field2 = DataTypes.createStructField("field2", DataTypes.IntegerType, true); StructField field3 = DataTypes.createStructField("field3", DataTypes.FloatType, true); StructType schema = DataTypes.createStructType(Lists.newArrayList(field1, field2, field3)); Row row = new RowWithSchema(schema, "hello", 1, 2.0); Row appendRow = RowUtils.append(row, "field4", DataTypes.BooleanType, true); appendRow = RowUtils.append(appendRow, "field5", DataTypes.StringType, "world"); assertEquals(appendRow.length(), 5); assertEquals(RowUtils.get(appendRow, "field1"), "hello"); assertEquals(RowUtils.get(appendRow, "field2"), 1); assertEquals(RowUtils.get(appendRow, "field3"), 2.0); assertEquals(RowUtils.get(appendRow, "field4"), true); assertEquals(RowUtils.get(appendRow, "field5"), "world"); }