@SuppressWarnings("unchecked") static StandardTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); if (tablePb.getNumRows() != null) { builder.setNumRows(tablePb.getNumRows().longValue()); } if (tablePb.getStreamingBuffer() != null) { builder.setStreamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); } if (tablePb.getTimePartitioning() != null) { builder.setTimePartitioning(TimePartitioning.fromPb(tablePb.getTimePartitioning())); } if (tablePb.getClustering() != null) { builder.setClustering(Clustering.fromPb(tablePb.getClustering())); } return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } }
BuilderImpl(Table tablePb) { this.tableId = TableId.fromPb(tablePb.getTableReference()); if (tablePb.getLastModifiedTime() != null) { this.setLastModifiedTime(tablePb.getLastModifiedTime().longValue()); } this.description = tablePb.getDescription(); this.expirationTime = tablePb.getExpirationTime(); this.friendlyName = tablePb.getFriendlyName(); this.creationTime = tablePb.getCreationTime(); this.etag = tablePb.getEtag(); this.generatedId = tablePb.getId(); this.selfLink = tablePb.getSelfLink(); this.numBytes = tablePb.getNumBytes(); this.numRows = tablePb.getNumRows(); this.definition = TableDefinition.fromPb(tablePb); if (tablePb.getEncryptionConfiguration() != null) { this.encryptionConfiguration = new EncryptionConfiguration.Builder(tablePb.getEncryptionConfiguration()).build(); } this.labels = Labels.fromPb(tablePb.getLabels()); }
@Override public List<InputSplit> getSplits(JobContext context) throws IOException { long numTableRows = tableToExport.getNumRows().longValue(); List<String> paths = getExportPaths(); int pathCount = paths.size(); List<InputSplit> splits = new ArrayList<>(); for (String exportPattern : paths) { splits.add( new ShardedInputSplit( new Path(exportPattern), Math.max(1, numTableRows / pathCount))); } return splits; }
private Builder(Table tablePb) { super(tablePb); if (tablePb.getNumRows() != null) { this.numRows(tablePb.getNumRows().longValue()); } this.numBytes = tablePb.getNumBytes(); this.location = tablePb.getLocation(); if (tablePb.getStreamingBuffer() != null) { this.streamingBuffer = StreamingBuffer.fromPb(tablePb.getStreamingBuffer()); } }
private Builder(Table tablePb) { super(tablePb); if (tablePb.getNumRows() != null) { this.numRows(tablePb.getNumRows().longValue()); } this.numBytes = tablePb.getNumBytes(); this.location = tablePb.getLocation(); if (tablePb.getStreamingBuffer() != null) { this.streamingBuffer = StreamingBuffer.fromPb(tablePb.getStreamingBuffer()); } if (tablePb.getTimePartitioning() != null) { this.timePartitioning = TimePartitioning.fromPb(tablePb.getTimePartitioning()); } }
@Override public List<String> getExportPaths() throws IOException { List<String> paths = new ArrayList<>(); long numTableRows = tableToExport.getNumRows().longValue(); long numTableBytes = tableToExport.getNumBytes(); int numShards = computeNumShards(numTableBytes); logger.atInfo().log("Computed '%s' shards for sharded BigQuery export.", numShards); for (int i = 0; i < numShards; ++i) { String exportPattern = String.format( "%s/shard-%d/%s", gcsPath, i, fileFormat.getFilePattern()); paths.add(exportPattern); } logger.atInfo().log( "Table '%s' to be exported has %s rows and %s bytes", BigQueryStrings.toString(tableToExport.getTableReference()), numTableRows, numTableBytes); return paths; }
@SuppressWarnings("unchecked") static StandardTableDefinition fromPb(Table tablePb) { Builder builder = newBuilder().table(tablePb); if (tablePb.getNumRows() != null) { builder.setNumRows(tablePb.getNumRows().longValue()); } if (tablePb.getStreamingBuffer() != null) { builder.setStreamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); } if (tablePb.getTimePartitioning() != null) { builder.setTimePartitioning(TimePartitioning.fromPb(tablePb.getTimePartitioning())); } if (tablePb.getClustering() != null) { builder.setClustering(Clustering.fromPb(tablePb.getClustering())); } return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } }
BuilderImpl(Table tablePb) { this.tableId = TableId.fromPb(tablePb.getTableReference()); if (tablePb.getLastModifiedTime() != null) { this.setLastModifiedTime(tablePb.getLastModifiedTime().longValue()); } this.description = tablePb.getDescription(); this.expirationTime = tablePb.getExpirationTime(); this.friendlyName = tablePb.getFriendlyName(); this.creationTime = tablePb.getCreationTime(); this.etag = tablePb.getEtag(); this.generatedId = tablePb.getId(); this.selfLink = tablePb.getSelfLink(); this.numBytes = tablePb.getNumBytes(); this.numRows = tablePb.getNumRows(); this.definition = TableDefinition.fromPb(tablePb); if (tablePb.getEncryptionConfiguration() != null) { this.encryptionConfiguration = new EncryptionConfiguration.Builder(tablePb.getEncryptionConfiguration()).build(); } this.labels = Labels.fromPb(tablePb.getLabels()); }