private static AvroExternalTable buildSnapshotTable() throws IOException { return buildAvroExternalTable(SNAPSHOT); }
private static void compact() throws IOException { SerialCompactor sc = new SerialCompactor.Builder().withSnapshot(buildSnapshotTable()).withDeltas(buildDeltaTables()) .withOutputTableName(jobProperties.getProperty(OUTPUT + NAME)) .withOutputDataLocationInHdfs(jobProperties.getProperty(OUTPUT + DATALOCATION)).build(); sc.compact(); }
jobProperties = ConfigurationConverter.getProperties(jobConfig); long startTime = System.nanoTime(); compact(); long endTime = System.nanoTime(); long elapsedTime = endTime - startTime;
private static void compact() throws IOException { SerialCompactor sc = new SerialCompactor.Builder().withSnapshot(buildSnapshotTable()).withDeltas(buildDeltaTables()) .withOutputTableName(jobProperties.getProperty(OUTPUT + NAME)) .withOutputDataLocationInHdfs(jobProperties.getProperty(OUTPUT + DATALOCATION)).build(); sc.compact(); }
private static List<AvroExternalTable> buildDeltaTables() throws IOException { List<AvroExternalTable> deltas = new ArrayList<>(); for (int i = 1;; i++) { String deltai = DELTA + "." + i; if (jobProperties.getProperty(deltai + DATALOCATION) == null) { break; } deltas.add(buildAvroExternalTable(deltai)); } return deltas; }
jobProperties = ConfigurationConverter.getProperties(jobConfig); long startTime = System.nanoTime(); compact(); long endTime = System.nanoTime(); long elapsedTime = endTime - startTime;
private static AvroExternalTable buildSnapshotTable() throws IOException { return buildAvroExternalTable(SNAPSHOT); }
private static List<AvroExternalTable> buildDeltaTables() throws IOException { List<AvroExternalTable> deltas = new ArrayList<>(); for (int i = 1;; i++) { String deltai = DELTA + "." + i; if (jobProperties.getProperty(deltai + DATALOCATION) == null) { break; } deltas.add(buildAvroExternalTable(deltai)); } return deltas; }