@Override public void close() throws IOException { sync(); put(SENTINEL); try { diskWriter.join(); checkForWriterException(); } catch (Throwable e) { throw new RuntimeException(e); } checkForWriterException(); }
/** * If we have marked that the column family is being replaced, when we start the next row, * we should sync out the previous partition and create a new row based on the current value. */ @Override boolean shouldStartNewRow() throws IOException { if (needsSync) { needsSync = false; super.sync(); return true; } return super.shouldStartNewRow(); }
setTypes(value.get(0)); prepareWriter(); writer.newRow(keybuff); for (Mutation mut : value) { if (cfType == CFType.SUPER) { writer.newSuperColumn(mut.getColumn_or_supercolumn().getSuper_column().name); if (colType == ColType.COUNTER) for (CounterColumn column : mut.getColumn_or_supercolumn().getCounter_super_column().columns) writer.addCounterColumn(column.name, column.value); else { for (Column column : mut.getColumn_or_supercolumn().getSuper_column().columns) { if (column.ttl == 0) writer.addColumn(column.name, column.value, column.timestamp); else writer.addExpiringColumn(column.name, column.value, column.timestamp, column.ttl, System.currentTimeMillis() + ((long) column.ttl * 1000)); writer.addCounterColumn(mut.getColumn_or_supercolumn().counter_column.name, mut.getColumn_or_supercolumn().counter_column.value); else { if (mut.getColumn_or_supercolumn().column.ttl == 0) writer.addColumn(mut.getColumn_or_supercolumn().column.name, mut.getColumn_or_supercolumn().column.value, mut.getColumn_or_supercolumn().column.timestamp); else writer.addExpiringColumn(mut.getColumn_or_supercolumn().column.name, mut.getColumn_or_supercolumn().column.value, mut.getColumn_or_supercolumn().column.timestamp, mut.getColumn_or_supercolumn().column.ttl, System.currentTimeMillis() + ((long) (mut.getColumn_or_supercolumn().column.ttl) * 1000));
private void prepareWriter() throws IOException { if (outputDir == null) { String keyspace = ConfigHelper.getOutputKeyspace(conf); //dir must be named by ks/cf for the loader outputDir = new File(getOutputLocation() + File.separator + keyspace + File.separator + ConfigHelper.getOutputColumnFamily(conf)); outputDir.mkdirs(); } if (writer == null) { AbstractType<?> subcomparator = null; if (cfType == CFType.SUPER) subcomparator = BytesType.instance; writer = new SSTableSimpleUnsortedWriter( outputDir, ConfigHelper.getOutputPartitioner(conf), ConfigHelper.getOutputKeyspace(conf), ConfigHelper.getOutputColumnFamily(conf), BytesType.instance, subcomparator, Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")), ConfigHelper.getOutputCompressionParamaters(conf)); this.loader = new SSTableLoader(outputDir, new ExternalClient(conf), new NullOutputHandler()); } }
protected void replaceColumnFamily() throws IOException { sync(); }
private void put(Buffer buffer) throws IOException { while (true) { checkForWriterException(); try { if (writeQueue.offer(buffer, 1, TimeUnit.SECONDS)) break; } catch (InterruptedException e) { throw new RuntimeException(e); } } }
protected void sync() throws IOException { if (buffer.isEmpty()) return; put(buffer); buffer = new Buffer(); currentSize = 0; }
protected void sync() throws IOException { if (buffer.isEmpty()) return; columnFamily = null; put(buffer); buffer = new Buffer(); currentSize = 0; columnFamily = getColumnFamily(); buffer.setFirstInsertedKey(currentKey); }
PartitionUpdate getUpdateFor(DecoratedKey key) { assert key != null; PartitionUpdate previous = buffer.get(key); if (previous == null) { previous = createPartitionUpdate(key); currentSize += PartitionUpdate.serializer.serializedSize(previous, formatType.info.getLatestVersion().correspondingMessagingVersion()); previous.allowNewUpdates(); buffer.put(key, previous); } return previous; }
public void run() { while (true) { try { Buffer b = writeQueue.take(); if (b == SENTINEL) return; try (SSTableTxnWriter writer = createWriter()) { for (Map.Entry<DecoratedKey, PartitionUpdate> entry : b.entrySet()) writer.append(entry.getValue().unfilteredIterator()); writer.finish(false); } } catch (Throwable e) { JVMStabilityInspector.inspectThrowable(e); // Keep only the first exception if (exception == null) exception = e; } } } }
prepareWriter(); SSTableSimpleUnsortedWriter ssWriter = (SSTableSimpleUnsortedWriter) writer; ssWriter.newRow(keybuff); for (Mutation mut : value) ssWriter.newSuperColumn(mut.getColumn_or_supercolumn().getSuper_column().name); if (colType == ColType.COUNTER) for (CounterColumn column : mut.getColumn_or_supercolumn().getCounter_super_column().columns) ssWriter.addCounterColumn(column.name, column.value); else ssWriter.addColumn(column.name, column.value, column.timestamp); else ssWriter.addExpiringColumn(column.name, column.value, column.timestamp, column.ttl, System.currentTimeMillis() + ((long)column.ttl * 1000)); ssWriter.addCounterColumn(mut.getColumn_or_supercolumn().counter_column.name, mut.getColumn_or_supercolumn().counter_column.value); else ssWriter.addColumn(mut.getColumn_or_supercolumn().column.name, mut.getColumn_or_supercolumn().column.value, mut.getColumn_or_supercolumn().column.timestamp); else ssWriter.addExpiringColumn(mut.getColumn_or_supercolumn().column.name, mut.getColumn_or_supercolumn().column.value, mut.getColumn_or_supercolumn().column.timestamp, mut.getColumn_or_supercolumn().column.ttl, System.currentTimeMillis() + ((long)(mut.getColumn_or_supercolumn().column.ttl) * 1000));
private void prepareWriter() throws IOException { if (outputdir == null) { String keyspace = ConfigHelper.getOutputKeyspace(conf); //dir must be named by ks/cf for the loader outputdir = new File(getOutputLocation() + File.separator + keyspace + File.separator + ConfigHelper.getOutputColumnFamily(conf)); outputdir.mkdirs(); } if (writer == null) { AbstractType<?> subcomparator = null; ExternalClient externalClient = null; String username = ConfigHelper.getOutputKeyspaceUserName(conf); String password = ConfigHelper.getOutputKeyspacePassword(conf); if (cfType == CFType.SUPER) subcomparator = BytesType.instance; this.writer = new SSTableSimpleUnsortedWriter( outputdir, ConfigHelper.getOutputPartitioner(conf), ConfigHelper.getOutputKeyspace(conf), ConfigHelper.getOutputColumnFamily(conf), BytesType.instance, subcomparator, Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")), ConfigHelper.getOutputCompressionParamaters(conf)); externalClient = new ExternalClient(ConfigHelper.getOutputInitialAddress(conf), ConfigHelper.getOutputRpcPort(conf), username, password); this.loader = new SSTableLoader(outputdir, externalClient, new NullOutputHandler()); } }
private void maybeSync() throws SyncException { try { if (currentSize > bufferSize) sync(); } catch (IOException e) { // addColumn does not throw IOException but we want to report this to the user, // so wrap it in a temporary RuntimeException that we'll catch in rawAddRow above. throw new SyncException(e); } }
private void put(Buffer buffer) throws IOException { while (true) { checkForWriterException(); try { if (writeQueue.offer(buffer, 1, TimeUnit.SECONDS)) break; } catch (InterruptedException e) { throw new RuntimeException(e); } } }
protected void sync() throws IOException { if (buffer.isEmpty()) return; put(buffer); buffer = new Buffer(); currentSize = 0; }
PartitionUpdate getUpdateFor(DecoratedKey key) { assert key != null; PartitionUpdate previous = buffer.get(key); if (previous == null) { previous = createPartitionUpdate(key); currentSize += PartitionUpdate.serializer.serializedSize(previous, formatType.info.getLatestVersion().correspondingMessagingVersion()); previous.allowNewUpdates(); buffer.put(key, previous); } return previous; }
public void run() { while (true) { try { Buffer b = writeQueue.take(); if (b == SENTINEL) return; try (SSTableTxnWriter writer = createWriter()) { for (Map.Entry<DecoratedKey, PartitionUpdate> entry : b.entrySet()) writer.append(entry.getValue().unfilteredIterator()); writer.finish(false); } } catch (Throwable e) { JVMStabilityInspector.inspectThrowable(e); // Keep only the first exception if (exception == null) exception = e; } } } }
@Override public void close() throws IOException { sync(); put(SENTINEL); try { diskWriter.join(); checkForWriterException(); } catch (Throwable e) { throw new RuntimeException(e); } checkForWriterException(); }
: new SSTableSimpleUnsortedWriter(directory, cfMetaData, preparedInsert.left.updatedColumns(), bufferSizeInMB);