/** * Read interceptor value from parquet. * * @param g Interceptor group. */ private static double readInterceptor(SimpleGroup g) { double interceptor; final SimpleGroup interceptVector = (SimpleGroup)g.getGroup(2, 0); final SimpleGroup interceptVectorVal = (SimpleGroup)interceptVector.getGroup(3, 0); final SimpleGroup interceptVectorValElement = (SimpleGroup)interceptVectorVal.getGroup(0, 0); interceptor = interceptVectorValElement.getDouble(0, 0); return interceptor; }
/** * Read coefficient matrix from parquet. * * @param g Coefficient group. * @return Vector of coefficients. */ private static Vector readCoefficients(SimpleGroup g) { Vector coefficients; final int amountOfCoefficients = g.getGroup(3, 0).getGroup(5, 0).getFieldRepetitionCount(0); coefficients = new DenseVector(amountOfCoefficients); for (int j = 0; j < amountOfCoefficients; j++) { double coefficient = g.getGroup(3, 0).getGroup(5, 0).getGroup(0, j).getDouble(0, 0); coefficients.set(j, coefficient); } return coefficients; }
/** * Form the node data according data in parquet row. * * @param g The given group presenting the node data from Spark DT model. */ @NotNull private static SparkModelParser.NodeData extractNodeDataFromParquetRow(SimpleGroup g) { NodeData nodeData = new NodeData(); nodeData.id = g.getInteger(0, 0); nodeData.prediction = g.getDouble(1, 0); nodeData.leftChildId = g.getInteger(5, 0); nodeData.rightChildId = g.getInteger(6, 0); if (nodeData.leftChildId == -1 && nodeData.rightChildId == -1) { nodeData.featureIdx = -1; nodeData.threshold = -1; nodeData.isLeafNode = true; } else { final SimpleGroup splitGrp = (SimpleGroup)g.getGroup(7, 0); nodeData.featureIdx = splitGrp.getInteger(0, 0); nodeData.threshold = splitGrp.getGroup(1, 0).getGroup(0, 0).getDouble(0, 0); } return nodeData; }
final SimpleGroup g = (SimpleGroup)recordReader.read(); final int treeID = g.getInteger(0, 0); final SimpleGroup nodeDataGroup = (SimpleGroup)g.getGroup(1, 0);
/** * Read coefficient matrix from parquet. * * @param g Coefficient group. * @return Vector of coefficients. */ private static Vector readSVMCoefficients(SimpleGroup g) { Vector coefficients; Group coeffGroup = g.getGroup(0, 0).getGroup(3, 0); final int amountOfCoefficients = coeffGroup.getFieldRepetitionCount(0); coefficients = new DenseVector(amountOfCoefficients); for (int j = 0; j < amountOfCoefficients; j++) { double coefficient = coeffGroup.getGroup(0, j).getDouble(0, 0); coefficients.set(j, coefficient); } return coefficients; }
/** * Read coefficient matrix from parquet. * * @param g Coefficient group. * @return Vector of coefficients. */ private static Vector readLinRegCoefficients(SimpleGroup g) { Vector coefficients; Group coeffGroup = g.getGroup(1, 0).getGroup(3, 0); final int amountOfCoefficients = coeffGroup.getFieldRepetitionCount(0); coefficients = new DenseVector(amountOfCoefficients); for (int j = 0; j < amountOfCoefficients; j++) { double coefficient = coeffGroup.getGroup(0, j).getDouble(0, 0); coefficients.set(j, coefficient); } return coefficients; }
final SimpleGroup g = (SimpleGroup)recordReader.read(); final int treeID = g.getInteger(0, 0); final SimpleGroup nodeDataGroup = (SimpleGroup)g.getGroup(1, 0); NodeData nodeData = extractNodeDataFromParquetRow(nodeDataGroup);
private static Model loadKMeansModel(String pathToMdl) { Vector[] centers = null; try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) { PageReadStore pages; final MessageType schema = r.getFooter().getFileMetaData().getSchema(); final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema); while (null != (pages = r.readNextRowGroup())) { final int rows = (int)pages.getRowCount(); final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema)); centers = new DenseVector[rows]; for (int i = 0; i < rows; i++) { final SimpleGroup g = (SimpleGroup)recordReader.read(); // final int clusterIdx = g.getInteger(0, 0); Group clusterCenterCoeff = g.getGroup(1, 0).getGroup(3, 0); final int amountOfCoefficients = clusterCenterCoeff.getFieldRepetitionCount(0); centers[i] = new DenseVector(amountOfCoefficients); for (int j = 0; j < amountOfCoefficients; j++) { double coefficient = clusterCenterCoeff.getGroup(0, j).getDouble(0, 0); centers[i].set(j, coefficient); } } } } catch (IOException e) { System.out.println("Error reading parquet file."); e.printStackTrace(); } return new KMeansModel(centers, new EuclideanDistance()); }