entries.put(entry.getKey(), new DataFileValue(entry.getValue().estimatedSize, 0L)); files.add(entry.getKey().path().toString());
long highSize = entry.getValue().getSize(); long highEntries = entry.getValue().getNumEntries(); highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime())); } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) { long lowSize = entry.getValue().getSize(); long lowEntries = entry.getValue().getNumEntries(); lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime())); long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio)); long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio)); lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime())); long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio))); long highEntries = (long) Math .ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio))); highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
@Override void load(List<TabletMetadata> tablets, Files files) throws MutationsRejectedException { byte[] fam = TextUtil.getBytes(DataFileColumnFamily.NAME); for (TabletMetadata tablet : tablets) { if (tablet.getLocation() != null) { unloadingTablets.increment(tablet.getLocation().getHostAndPort(), 1L); continue; } Mutation mutation = new Mutation(tablet.getExtent().getMetadataEntry()); for (final Bulk.FileInfo fileInfo : files) { String fullPath = new Path(bulkDir, fileInfo.getFileName()).toString(); byte[] val = new DataFileValue(fileInfo.getEstFileSize(), fileInfo.getEstNumEntries()) .encode(); mutation.put(fam, fullPath.getBytes(UTF_8), val); } bw.addMutation(mutation); } }
private static void createEntriesForTablet(TreeMap<Key,Value> map, Tablet tablet) { Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue(); Text extent = new Text(TabletsSection.getRow(tablet.tableId, tablet.endRow)); addEntry(map, extent, DIRECTORY_COLUMN, new Value(tablet.dir.getBytes(UTF_8))); addEntry(map, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8))); addEntry(map, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tablet.prevEndRow)); for (String file : tablet.files) { addEntry(map, extent, new ColumnFQ(DataFileColumnFamily.NAME, new Text(file)), EMPTY_SIZE); } }
DataFileValue dfv = new DataFileValue(columnUpdate.getValue()); if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) { violations = addViolation(violations, 1);
/** * Calculates the total size of input files in the compaction plan */ private Long calculateTotalSize(MajorCompactionRequest request, CompactionPlan plan) { long totalSize = 0; Map<FileRef,DataFileValue> allFiles = request.getFiles(); for (FileRef fileRef : plan.inputFiles) { totalSize += allFiles.get(fileRef).getSize(); } return totalSize; }
private static Mutation createTablet(String tid, String endRow, String prevRow, String dir, String file) throws Exception { KeyExtent ke = new KeyExtent(tid, endRow == null ? null : new Text(endRow), prevRow == null ? null : new Text(prevRow)); Mutation mut = ke.getPrevRowUpdateMutation(); TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0".getBytes())); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes())); mut.put(DataFileColumnFamily.NAME.toString(), file, new DataFileValue(10, 200).encodeAsString()); return mut; }
if (dfv.getNumEntries() == 0) { tablet.getTabletServer().getFileSystem().deleteRecursively(tmpDatafile.path()); } else { if (dfv.getNumEntries() > 0) { datafileSizes.put(newDatafile, dfv); tablet.getExtent().toString())); long splitSize = tablet.getTableConfiguration().getAsBytes(Property.TABLE_SPLIT_THRESHOLD); if (dfv.getSize() > splitSize) { log.debug(String.format("Minor Compaction wrote out file larger than split threshold." + " split threshold = %,d file size = %,d", splitSize, dfv.getSize()));
Mutation m = new Mutation(extent.getMetadataEntry()); if (dfv.getNumEntries() > 0) { m.put(DataFileColumnFamily.NAME, path.meta(), new Value(dfv.encode())); TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8)));
private Size fetch() { long tabletSize = 0; while (iterator.hasNext()) { Entry<Key,Value> entry = iterator.next(); Key key = entry.getKey(); if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { tabletSize += new DataFileValue(entry.getValue().get()).getSize(); } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) { KeyExtent extent = new KeyExtent(key.getRow(), entry.getValue()); return new Size(extent, tabletSize); } } return null; }
public Value encodeAsValue() { return new Value(encode()); }
deletes += summary.getStatistics().get(DELETES_STAT); } else { long numEntries = entry.getValue().getNumEntries(); if (numEntries == 0 && !proceed_bns) { shouldCompact = false; } else { total += entry.getValue().getNumEntries();
public byte[] encode() { return encodeAsString().getBytes(UTF_8); }
DataFileValue dfv = new DataFileValue(columnUpdate.getValue()); if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) { violations = addViolation(violations, 1);
public abstract boolean shouldCompact(long fsize, long esize); }
private static void createEntriesForTablet(TreeMap<Key,Value> map, Tablet tablet) { Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue(); Text extent = new Text(KeyExtent.getMetadataEntry(tablet.tableId, tablet.endRow)); addEntry(map, extent, DIRECTORY_COLUMN, new Value(tablet.dir.getBytes(UTF_8))); addEntry(map, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8))); addEntry(map, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tablet.prevEndRow)); for (String file : tablet.files) { addEntry(map, extent, new ColumnFQ(DataFileColumnFamily.NAME, new Text(file)), EMPTY_SIZE); } }
new Value("/default_tablet".getBytes())); mut.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf", new DataFileValue(1, 200).encodeAsString()); mut2.putDelete(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf"); mut2.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/1_0.rf", new DataFileValue(2, 300).encodeAsString());
minorCompaction.data("numEntries", Long.toString(this.stats.getNumEntries())); minorCompaction.data("size", Long.toString(this.stats.getSize())); minorCompaction.stop();
public static void replaceDatafiles(ServerContext context, KeyExtent extent, Set<FileRef> datafilesToDelete, Set<FileRef> scanFiles, FileRef path, Long compactionId, DataFileValue size, String address, TServerInstance lastLocation, ZooLock zooLock, boolean insertDeleteFlags) { if (insertDeleteFlags) { // add delete flags for those paths before the data file reference is removed MetadataTableUtil.addDeleteEntries(extent, datafilesToDelete, context); } // replace data file references to old mapfiles with the new mapfiles Mutation m = new Mutation(extent.getMetadataEntry()); for (FileRef pathToRemove : datafilesToDelete) m.putDelete(DataFileColumnFamily.NAME, pathToRemove.meta()); for (FileRef scanFile : scanFiles) m.put(ScanFileColumnFamily.NAME, scanFile.meta(), new Value(new byte[0])); if (size.getNumEntries() > 0) m.put(DataFileColumnFamily.NAME, path.meta(), new Value(size.encode())); if (compactionId != null) TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value(("" + compactionId).getBytes())); TServerInstance self = getTServerInstance(address, zooLock); self.putLastLocation(m); // remove the old location if (lastLocation != null && !lastLocation.equals(self)) lastLocation.clearLastLocation(m); MetadataTableUtil.update(context, zooLock, m, extent); }