org.apache.parquet.io.RecordReader java code examples

Add the Codota plugin to your IDE and get smart completions

private void myMethod () {
 Gson g =new Gson()
GsonBuilder gsonBuilder;gsonBuilder.create()
new GsonBuilder().create()
Smart code suggestions by Codota
}

final SimpleGroup g = (SimpleGroup)recordReader.read();
final int treeID = g.getInteger(0, 0);
final SimpleGroup nodeDataGroup = (SimpleGroup)g.getGroup(1, 0);

 currentValue = recordReader.read();
} catch (RecordMaterializationException e) {
if (recordReader.shouldSkipCurrentRecord()) {

 currentValue = recordReader.read();
} catch (RecordMaterializationException e) {
if (recordReader.shouldSkipCurrentRecord()) {

/**
 * Load Decision Tree model.
 *
 * @param pathToMdl Path to model.
 */
private static Model loadDecisionTreeModel(String pathToMdl) {
  try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
    PageReadStore pages;
    final MessageType schema = r.getFooter().getFileMetaData().getSchema();
    final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
    final Map<Integer, NodeData> nodes = new TreeMap<>();
    while (null != (pages = r.readNextRowGroup())) {
      final long rows = pages.getRowCount();
      final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
      for (int i = 0; i < rows; i++) {
        final SimpleGroup g = (SimpleGroup)recordReader.read();
        NodeData nodeData = extractNodeDataFromParquetRow(g);
        nodes.put(nodeData.id, nodeData);
      }
    }
    return buildDecisionTreeModel(nodes);
  }
  catch (IOException e) {
    System.out.println("Error reading parquet file.");
    e.printStackTrace();
  }
  return null;
}

 currentValue = recordReader.read();
} catch (RecordMaterializationException e) {
if (recordReader.shouldSkipCurrentRecord()) {

final RecordReader recordReader = colIO.getRecordReader(pagesMetaData, new GroupRecordConverter(schema));
for (int i = 0; i < rows; i++) {
  final SimpleGroup g = (SimpleGroup)recordReader.read();
  int treeId = g.getInteger(0, 0);
  double treeWeight = g.getDouble(2, 0);
final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
for (int i = 0; i < rows; i++) {
  final SimpleGroup g = (SimpleGroup)recordReader.read();
  final int treeID = g.getInteger(0, 0);
  final SimpleGroup nodeDataGroup = (SimpleGroup)g.getGroup(1, 0);

 currentValue = recordReader.read();
} catch (RecordMaterializationException e) {
if (recordReader.shouldSkipCurrentRecord()) {

/**
 * Load logistic regression model.
 *
 * @param pathToMdl Path to model.
 */
private static Model loadLogRegModel(String pathToMdl) {
  Vector coefficients = null;
  double interceptor = 0;
  try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
    PageReadStore pages;
    final MessageType schema = r.getFooter().getFileMetaData().getSchema();
    final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
    while (null != (pages = r.readNextRowGroup())) {
      final long rows = pages.getRowCount();
      final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
      for (int i = 0; i < rows; i++) {
        final SimpleGroup g = (SimpleGroup)recordReader.read();
        interceptor = readInterceptor(g);
        coefficients = readCoefficients(g);
      }
    }
  }
  catch (IOException e) {
    System.out.println("Error reading parquet file.");
    e.printStackTrace();
  }
  return new LogisticRegressionModel(coefficients, interceptor);
}

/**
 * Load linear regression model.
 *
 * @param pathToMdl Path to model.
 */
private static Model loadLinRegModel(String pathToMdl) {
  Vector coefficients = null;
  double interceptor = 0;
  try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
    PageReadStore pages;
    final MessageType schema = r.getFooter().getFileMetaData().getSchema();
    final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
    while (null != (pages = r.readNextRowGroup())) {
      final long rows = pages.getRowCount();
      final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
      for (int i = 0; i < rows; i++) {
        final SimpleGroup g = (SimpleGroup)recordReader.read();
        interceptor = readLinRegInterceptor(g);
        coefficients = readLinRegCoefficients(g);
      }
    }
  }
  catch (IOException e) {
    System.out.println("Error reading parquet file.");
    e.printStackTrace();
  }
  return new LinearRegressionModel(coefficients, interceptor);
}

/**
 * Load SVM model.
 *
 * @param pathToMdl Path to model.
 */
private static Model loadLinearSVMModel(String pathToMdl) {
  Vector coefficients = null;
  double interceptor = 0;
  try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
    PageReadStore pages;
    final MessageType schema = r.getFooter().getFileMetaData().getSchema();
    final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
    while (null != (pages = r.readNextRowGroup())) {
      final long rows = pages.getRowCount();
      final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
      for (int i = 0; i < rows; i++) {
        final SimpleGroup g = (SimpleGroup)recordReader.read();
        interceptor = readSVMInterceptor(g);
        coefficients = readSVMCoefficients(g);
      }
    }
  }
  catch (IOException e) {
    System.out.println("Error reading parquet file.");
    e.printStackTrace();
  }
  return new SVMLinearClassificationModel(coefficients, interceptor);
}

private static Model loadKMeansModel(String pathToMdl) {
  Vector[] centers = null;
  try (ParquetFileReader r = ParquetFileReader.open(HadoopInputFile.fromPath(new Path(pathToMdl), new Configuration()))) {
    PageReadStore pages;
    final MessageType schema = r.getFooter().getFileMetaData().getSchema();
    final MessageColumnIO colIO = new ColumnIOFactory().getColumnIO(schema);
    while (null != (pages = r.readNextRowGroup())) {
      final int rows = (int)pages.getRowCount();
      final RecordReader recordReader = colIO.getRecordReader(pages, new GroupRecordConverter(schema));
      centers = new DenseVector[rows];
      for (int i = 0; i < rows; i++) {
        final SimpleGroup g = (SimpleGroup)recordReader.read();
        // final int clusterIdx = g.getInteger(0, 0);
        Group clusterCenterCoeff = g.getGroup(1, 0).getGroup(3, 0);
        final int amountOfCoefficients = clusterCenterCoeff.getFieldRepetitionCount(0);
        centers[i] = new DenseVector(amountOfCoefficients);
        for (int j = 0; j < amountOfCoefficients; j++) {
          double coefficient = clusterCenterCoeff.getGroup(0, j).getDouble(0, 0);
          centers[i].set(j, coefficient);
        }
      }
    }
  }
  catch (IOException e) {
    System.out.println("Error reading parquet file.");
    e.printStackTrace();
  }
  return new KMeansModel(centers, new EuclideanDistance());
}

recordReader.read();
count++;
totalRead++;

recordReader.read();
count++;
totalRead++;

private List<IColumn> loadColumns(ParquetMetadata md) {
  try {
    MessageType schema = md.getFileMetaData().getSchema();
    List<IAppendableColumn> cols = createColumns(md);
    ParquetFileReader r = ParquetFileReader.open(this.configuration, this.path);
    MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema);
    PageReadStore pages;
    while (null != (pages = r.readNextRowGroup())) {
      final long rows = pages.getRowCount();
      RecordReader<Group> recordReader = columnIO.getRecordReader(
          pages, new GroupRecordConverter(schema));
      for (int i = 0; i < rows; i++) {
        Group g = recordReader.read();
        appendGroup(cols, g, md.getFileMetaData().getSchema().getColumns());
      }
    }
    for (IAppendableColumn c : cols)
      c.seal();
    r.close();
    return Linq.map(cols, e -> e);
  } catch (IOException ex) {
    throw new RuntimeException(ex);
  }
}

Javadoc

used to read reassembled records

Most used methods

read
Reads one record and returns it.
shouldSkipCurrentRecord
Returns whether the current record should be skipped (dropped) Will be called *after* read()

Popular in Java

Reading from database using SQL prepared statement
getResourceAsStream (ClassLoader)
setContentView (Activity)
getSupportFragmentManager (FragmentActivity)
Return the FragmentManager for interacting with fragments associated with this activity.
URL (java.net)
A Uniform Resource Locator that identifies the location of an Internet resource as specified by RFC
SecureRandom (java.security)
This class generates cryptographically secure pseudo-random numbers. It is best to invoke SecureRand
Format (java.text)
The base class for all formats. This is an abstract base class which specifies the protocol for clas
Dictionary (java.util)
The Dictionary class is the abstract parent of any class, such as Hashtable, which maps keys to valu
Collectors (java.util.stream)
BoxLayout (javax.swing)

How to useRecordReader in org.apache.parquet.io

Best Java code snippets using org.apache.parquet.io.RecordReader (Showing top 14 results out of 315)

How to use
RecordReader
in
org.apache.parquet.io