public static KeyExtent fixSplit(ServerContext context, Text metadataEntry, SortedMap<ColumnFQ,Value> columns, ZooLock lock) throws AccumuloException { log.info("Incomplete split {} attempting to fix", metadataEntry); Value oper = columns.get(TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN); if (columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN) == null) { throw new IllegalArgumentException( "Metadata entry does not have split ratio (" + metadataEntry + ")"); } double splitRatio = Double.parseDouble( new String(columns.get(TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN).get(), UTF_8)); Value prevEndRowIBW = columns.get(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN); if (prevEndRowIBW == null) { throw new IllegalArgumentException( "Metadata entry does not have prev row (" + metadataEntry + ")"); } Value time = columns.get(TabletsSection.ServerColumnFamily.TIME_COLUMN); if (time == null) { throw new IllegalArgumentException( "Metadata entry does not have time (" + metadataEntry + ")"); } Text metadataPrevEndRow = KeyExtent.decodePrevEndRow(prevEndRowIBW); Table.ID tableId = (new KeyExtent(metadataEntry, (Text) null)).getTableId(); return fixSplit(context, tableId, metadataEntry, metadataPrevEndRow, oper, splitRatio, lock); }
if (filesInUseByScans.size() > 0) log.debug("Adding scan refs to metadata {} {}", extent, filesInUseByScans); MasterMetadataUtil.replaceDatafiles(tablet.getContext(), extent, oldDatafiles, filesInUseByScans, newDatafile, compactionId, dfv, tablet.getTabletServer().getClientAddressString(), lastLocation,
/** * new data file update function adds one data file to a tablet's list * * @param path * should be relative to the table directory * */ public static void updateTabletDataFile(ServerContext context, KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time, Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) { if (extent.isRootTablet()) { if (unusedWalLogs != null) { updateRootTabletDataFile(context, unusedWalLogs); } return; } Mutation m = getUpdateForTabletDataFile(extent, path, mergeFile, dfv, time, filesInUseByScans, address, zooLock, unusedWalLogs, lastLocation, flushId); MetadataTableUtil.update(context, zooLock, m, extent); }
public static void replaceDatafiles(ServerContext context, KeyExtent extent, Set<FileRef> datafilesToDelete, Set<FileRef> scanFiles, FileRef path, Long compactionId, DataFileValue size, String address, TServerInstance lastLocation, ZooLock zooLock, boolean insertDeleteFlags) { if (insertDeleteFlags) { // add delete flags for those paths before the data file reference is removed MetadataTableUtil.addDeleteEntries(extent, datafilesToDelete, context); } // replace data file references to old mapfiles with the new mapfiles Mutation m = new Mutation(extent.getMetadataEntry()); for (FileRef pathToRemove : datafilesToDelete) m.putDelete(DataFileColumnFamily.NAME, pathToRemove.meta()); for (FileRef scanFile : scanFiles) m.put(ScanFileColumnFamily.NAME, scanFile.meta(), new Value(new byte[0])); if (size.getNumEntries() > 0) m.put(DataFileColumnFamily.NAME, path.meta(), new Value(size.encode())); if (compactionId != null) TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value(("" + compactionId).getBytes())); TServerInstance self = getTServerInstance(address, zooLock); self.putLastLocation(m); // remove the old location if (lastLocation != null && !lastLocation.equals(self)) lastLocation.clearLastLocation(m); MetadataTableUtil.update(context, zooLock, m, extent); }
MasterMetadataUtil.addNewTablet(getTabletServer().getContext(), low, lowDirectory, getTabletServer().getTabletSession(), lowDatafileSizes, getBulkIngestedFiles(), time, lastFlushID, lastCompactID, getTabletServer().getLock());
public void updateTabletDataFile(long maxCommittedTime, FileRef newDatafile, FileRef absMergeFile, DataFileValue dfv, Set<String> unusedWalLogs, Set<FileRef> filesInUseByScans, long flushId) { synchronized (timeLock) { if (maxCommittedTime > persistedTime) persistedTime = maxCommittedTime; String time = tabletTime.getMetadataValue(persistedTime); MasterMetadataUtil.updateTabletDataFile(getTabletServer().getContext(), extent, newDatafile, absMergeFile, dfv, time, filesInUseByScans, tabletServer.getClientAddressString(), tabletServer.getLock(), unusedWalLogs, lastLocation, flushId); } }
TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8))); TServerInstance self = getTServerInstance(address, zooLock); self.putLastLocation(m);
public static void replaceDatafiles(ClientContext context, KeyExtent extent, Set<FileRef> datafilesToDelete, Set<FileRef> scanFiles, FileRef path, Long compactionId, DataFileValue size, String address, TServerInstance lastLocation, ZooLock zooLock, boolean insertDeleteFlags) throws IOException { if (insertDeleteFlags) { // add delete flags for those paths before the data file reference is removed MetadataTableUtil.addDeleteEntries(extent, datafilesToDelete, context); } // replace data file references to old mapfiles with the new mapfiles Mutation m = new Mutation(extent.getMetadataEntry()); for (FileRef pathToRemove : datafilesToDelete) m.putDelete(DataFileColumnFamily.NAME, pathToRemove.meta()); for (FileRef scanFile : scanFiles) m.put(ScanFileColumnFamily.NAME, scanFile.meta(), new Value(new byte[0])); if (size.getNumEntries() > 0) m.put(DataFileColumnFamily.NAME, path.meta(), new Value(size.encode())); if (compactionId != null) TabletsSection.ServerColumnFamily.COMPACT_COLUMN.put(m, new Value(("" + compactionId).getBytes())); TServerInstance self = getTServerInstance(address, zooLock); self.putLastLocation(m); // remove the old location if (lastLocation != null && !lastLocation.equals(self)) lastLocation.clearLastLocation(m); MetadataTableUtil.update(context, zooLock, m, extent); }
Map<Long,? extends Collection<FileRef>> bulkFiles = MetadataTableUtil .getBulkFilesLoaded(context, extent); MasterMetadataUtil.addNewTablet(context, low, "/lowDir", instance, lowDatafileSizes, bulkFiles, TabletTime.LOGICAL_TIME_ID + "0", -1l, -1l, zl);
public void updateTabletDataFile(long maxCommittedTime, FileRef newDatafile, FileRef absMergeFile, DataFileValue dfv, Set<String> unusedWalLogs, Set<FileRef> filesInUseByScans, long flushId) { synchronized (timeLock) { if (maxCommittedTime > persistedTime) persistedTime = maxCommittedTime; String time = tabletTime.getMetadataValue(persistedTime); MasterMetadataUtil.updateTabletDataFile(getTabletServer(), extent, newDatafile, absMergeFile, dfv, time, filesInUseByScans, tabletServer.getClientAddressString(), tabletServer.getLock(), unusedWalLogs, lastLocation, flushId); } }
TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8))); TServerInstance self = getTServerInstance(address, zooLock); self.putLastLocation(m);
tabletEntries = MetadataTableUtil.getTabletEntries(tabletsKeyValues, columnsToFetch); KeyExtent fke = MasterMetadataUtil.fixSplit(context, metadataEntry, tabletEntries.get(metadataEntry), lock);
MasterMetadataUtil.addNewTablet(getTabletServer(), low, lowDirectory, getTabletServer().getTabletSession(), lowDatafileSizes, getBulkIngestedFiles(), time, lastFlushID, lastCompactID, getTabletServer().getLock());
/** * new data file update function adds one data file to a tablet's list * * @param path * should be relative to the table directory * */ public static void updateTabletDataFile(ClientContext context, KeyExtent extent, FileRef path, FileRef mergeFile, DataFileValue dfv, String time, Set<FileRef> filesInUseByScans, String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) { if (extent.isRootTablet()) { if (unusedWalLogs != null) { updateRootTabletDataFile(extent, path, mergeFile, dfv, time, filesInUseByScans, address, zooLock, unusedWalLogs, lastLocation, flushId); } return; } Mutation m = getUpdateForTabletDataFile(extent, path, mergeFile, dfv, time, filesInUseByScans, address, zooLock, unusedWalLogs, lastLocation, flushId); MetadataTableUtil.update(context, zooLock, m, extent); }
if (filesInUseByScans.size() > 0) log.debug("Adding scan refs to metadata " + extent + " " + filesInUseByScans); MasterMetadataUtil.replaceDatafiles(tablet.getTabletServer(), extent, oldDatafiles, filesInUseByScans, newDatafile, compactionId, dfv, tablet.getTabletServer().getClientAddressString(), lastLocation,
return fixSplit(context, table, metadataEntry, metadataPrevEndRow, oper, splitRatio, tserver, time.toString(), initFlushID, initCompactID, lock);
fke = MasterMetadataUtil.fixSplit(context, metadataEntry, tabletEntries.get(metadataEntry), instance, lock); } catch (IOException e) {