long highEntries = entry.getValue().getNumEntries(); highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime())); } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) { long lowEntries = entry.getValue().getNumEntries(); lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime())); long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio)); lowDatafileSizes.put(entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime())); .ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio))); highDatafileSizes.put(entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
private static void createEntriesForTablet(TreeMap<Key,Value> map, Tablet tablet) { Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue(); Text extent = new Text(TabletsSection.getRow(tablet.tableId, tablet.endRow)); addEntry(map, extent, DIRECTORY_COLUMN, new Value(tablet.dir.getBytes(UTF_8))); addEntry(map, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8))); addEntry(map, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tablet.prevEndRow)); for (String file : tablet.files) { addEntry(map, extent, new ColumnFQ(DataFileColumnFamily.NAME, new Text(file)), EMPTY_SIZE); } }
entries.put(entry.getKey(), new DataFileValue(entry.getValue().estimatedSize, 0L)); files.add(entry.getKey().path().toString());
public static SortedMap<FileRef,DataFileValue> getDataFileSizes(KeyExtent extent, ServerContext context) { TreeMap<FileRef,DataFileValue> sizes = new TreeMap<>(); try (Scanner mdScanner = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY)) { mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME); Text row = extent.getMetadataEntry(); Key endKey = new Key(row, DataFileColumnFamily.NAME, new Text("")); endKey = endKey.followingKey(PartialKey.ROW_COLFAM); mdScanner.setRange(new Range(new Key(row), endKey)); for (Entry<Key,Value> entry : mdScanner) { if (!entry.getKey().getRow().equals(row)) break; DataFileValue dfv = new DataFileValue(entry.getValue().get()); sizes.put(new FileRef(context.getVolumeManager(), entry.getKey()), dfv); } return sizes; } }
dataFiles.put(ref, new DataFileValue(entry.getValue().get())); } else if (DIRECTORY_COLUMN.hasColumns(key)) { directory = value.toString();
continue; DataFileValue dfv = new DataFileValue(0, 0); sizes.put(new FileRef(fileStatus.getPath().toString(), fileStatus.getPath()), dfv);
filesBuilder.put(qual, new DataFileValue(val)); break; case BulkFileColumnFamily.STR_NAME:
if (entry.getKey().compareColumnFamily(DataFileColumnFamily.NAME) == 0) { origDatafileSizes.put(new FileRef(context.getVolumeManager(), entry.getKey()), new DataFileValue(entry.getValue().get()));
String filename = path.getName(); FileRef ref = new FileRef(location + "/" + filename, path); DataFileValue dfv = new DataFileValue(0, 0); dataFiles.put(ref, dfv);
@Override void load(List<TabletMetadata> tablets, Files files) throws MutationsRejectedException { byte[] fam = TextUtil.getBytes(DataFileColumnFamily.NAME); for (TabletMetadata tablet : tablets) { if (tablet.getLocation() != null) { unloadingTablets.increment(tablet.getLocation().getHostAndPort(), 1L); continue; } Mutation mutation = new Mutation(tablet.getExtent().getMetadataEntry()); for (final Bulk.FileInfo fileInfo : files) { String fullPath = new Path(bulkDir, fileInfo.getFileName()).toString(); byte[] val = new DataFileValue(fileInfo.getEstFileSize(), fileInfo.getEstNumEntries()) .encode(); mutation.put(fam, fullPath.getBytes(UTF_8), val); } bw.addMutation(mutation); } }
try { getDatafileManager().bringMinorCompactionOnline(tmpDatafile, newDatafile, mergeFile, new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()), commitSession, flushId); } finally { span.stop(); return new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()); } catch (Exception | Error e) { failed = true;
DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten())); new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
private static Mutation createTablet(String tid, String endRow, String prevRow, String dir, String file) throws Exception { KeyExtent ke = new KeyExtent(tid, endRow == null ? null : new Text(endRow), prevRow == null ? null : new Text(prevRow)); Mutation mut = ke.getPrevRowUpdateMutation(); TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0".getBytes())); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes())); mut.put(DataFileColumnFamily.NAME.toString(), file, new DataFileValue(10, 200).encodeAsString()); return mut; }
private Size fetch() { long tabletSize = 0; while (iterator.hasNext()) { Entry<Key,Value> entry = iterator.next(); Key key = entry.getKey(); if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { tabletSize += new DataFileValue(entry.getValue().get()).getSize(); } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) { KeyExtent extent = new KeyExtent(key.getRow(), entry.getValue()); return new Size(extent, tabletSize); } } return null; }
private static void createEntriesForTablet(TreeMap<Key,Value> map, Tablet tablet) { Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue(); Text extent = new Text(KeyExtent.getMetadataEntry(tablet.tableId, tablet.endRow)); addEntry(map, extent, DIRECTORY_COLUMN, new Value(tablet.dir.getBytes(UTF_8))); addEntry(map, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8))); addEntry(map, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tablet.prevEndRow)); for (String file : tablet.files) { addEntry(map, extent, new ColumnFQ(DataFileColumnFamily.NAME, new Text(file)), EMPTY_SIZE); } }
public static SortedMap<FileRef,DataFileValue> getDataFileSizes(KeyExtent extent, ClientContext context) throws IOException { TreeMap<FileRef,DataFileValue> sizes = new TreeMap<>(); try (Scanner mdScanner = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY)) { mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME); Text row = extent.getMetadataEntry(); VolumeManager fs = VolumeManagerImpl.get(); Key endKey = new Key(row, DataFileColumnFamily.NAME, new Text("")); endKey = endKey.followingKey(PartialKey.ROW_COLFAM); mdScanner.setRange(new Range(new Key(row), endKey)); for (Entry<Key,Value> entry : mdScanner) { if (!entry.getKey().getRow().equals(row)) break; DataFileValue dfv = new DataFileValue(entry.getValue().get()); sizes.put(new FileRef(fs, entry.getKey()), dfv); } return sizes; } }
String filename = path.getName(); FileRef ref = new FileRef(location.toString() + "/" + filename, path); DataFileValue dfv = new DataFileValue(0, 0); dataFiles.put(ref, dfv);
private void runSplitRecoveryTest(AccumuloServerContext context, int failPoint, String mr, int extentToSplit, ZooLock zl, KeyExtent... extents) throws Exception { Text midRow = new Text(mr); SortedMap<FileRef,DataFileValue> splitMapFiles = null; for (int i = 0; i < extents.length; i++) { KeyExtent extent = extents[i]; String tdir = ServerConstants.getTablesDirs()[0] + "/" + extent.getTableId() + "/dir_" + i; MetadataTableUtil.addTablet(extent, tdir, context, TabletTime.LOGICAL_TIME_ID, zl); SortedMap<FileRef,DataFileValue> mapFiles = new TreeMap<>(); mapFiles.put(new FileRef(tdir + "/" + RFile.EXTENSION + "_000_000"), new DataFileValue(1000017 + i, 10000 + i)); if (i == extentToSplit) { splitMapFiles = mapFiles; } int tid = 0; TransactionWatcher.ZooArbitrator.start(Constants.BULK_ARBITRATOR_TYPE, tid); MetadataTableUtil.updateTabletDataFile(tid, extent, mapFiles, "L0", context, zl); } KeyExtent extent = extents[extentToSplit]; KeyExtent high = new KeyExtent(extent.getTableId(), extent.getEndRow(), midRow); KeyExtent low = new KeyExtent(extent.getTableId(), midRow, extent.getPrevEndRow()); splitPartiallyAndRecover(context, extent, high, low, .4, splitMapFiles, midRow, "localhost:1234", failPoint, zl); }
@Test public void testMerge() throws Exception { Connector conn = getConnector(); String tableName = getUniqueNames(1)[0]; conn.tableOperations().create(tableName); BatchWriter bw1 = conn.createBatchWriter(tableName, new BatchWriterConfig()); bw1.addMutation(createTablet("0", "m", null, "/d1", "/d1/file1")); bw1.addMutation(createTablet("0", null, "m", "/d2", "/d2/file2")); bw1.flush(); BatchWriter bw2 = conn.createBatchWriter(tableName, new BatchWriterConfig()); MetadataTableUtil.initializeClone(tableName, "0", "1", conn, bw2); bw1.addMutation(deleteTablet("0", "m", null, "/d1", "/d1/file1")); Mutation mut = createTablet("0", null, null, "/d2", "/d2/file2"); mut.put(DataFileColumnFamily.NAME.toString(), "/d1/file1", new DataFileValue(10, 200).encodeAsString()); bw1.addMutation(mut); bw1.flush(); try { MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2); assertTrue(false); } catch (TabletIterator.TabletDeletedException tde) {} }