void mutate(List<Mutation> mutations, int mutationCount) { Iterator<Mutation> iter = mutations.iterator(); while (iter.hasNext()) { wlock.lock(); try { if (nmPointer == 0) { throw new IllegalStateException("Native Map Deleted"); } modCount++; int count = 0; while (iter.hasNext() && count < 10) { Mutation mutation = iter.next(); mutationCount = _mutate(mutation, mutationCount); count += mutation.size(); } } finally { wlock.unlock(); } } }
/** * Push a mutation into a table. If table is null, the defaultTable will be used. If * {@link AccumuloOutputFormat#canCreateTables(JobConf)} is set, the table will be created if it * does not exist. The table name must only contain alphanumerics and underscore. */ @Override public void write(Text table, Mutation mutation) throws IOException { if (table == null || table.toString().isEmpty()) table = this.defaultTableName; if (!simulate && table == null) throw new IOException("No table or default table specified. Try simulation mode next time"); ++mutCount; valCount += mutation.size(); printMutation(table, mutation); if (simulate) return; if (!bws.containsKey(table)) try { addTable(table); } catch (final Exception e) { log.error("Could not add table '" + table + "'", e); throw new IOException(e); } try { bws.get(table).addMutation(mutation); } catch (MutationsRejectedException e) { throw new IOException(e); } }
/** * Push a mutation into a table. If table is null, the defaultTable will be used. If * {@link AccumuloOutputFormat#canCreateTables(JobContext)} is set, the table will be created if * it does not exist. The table name must only contain alphanumerics and underscore. */ @Override public void write(Text table, Mutation mutation) throws IOException { if (table == null || table.toString().isEmpty()) table = this.defaultTableName; if (!simulate && table == null) throw new IOException("No table or default table specified. Try simulation mode next time"); ++mutCount; valCount += mutation.size(); printMutation(table, mutation); if (simulate) return; if (!bws.containsKey(table)) try { addTable(table); } catch (Exception e) { log.error("Could not add table '" + table + "'", e); throw new IOException(e); } try { bws.get(table).addMutation(mutation); } catch (MutationsRejectedException e) { throw new IOException(e); } }
public void commit(CommitSession commitSession, List<Mutation> mutations) { int totalCount = 0; long totalBytes = 0; // write the mutation to the in memory table for (Mutation mutation : mutations) { totalCount += mutation.size(); totalBytes += mutation.numBytes(); } getTabletMemory().mutate(commitSession, mutations, totalCount); synchronized (this) { if (writesInProgress < 1) { throw new IllegalStateException( "commiting mutations after logging, but not waiting for any log messages"); } if (isCloseComplete()) { throw new IllegalStateException("tablet closed with outstanding messages to the logger"); } getTabletMemory().updateMemoryUsageStats(); // decrement here in case an exception is thrown below writesInProgress--; if (writesInProgress == 0) this.notifyAll(); commitSession.decrementCommitsInProgress(); numEntries += totalCount; numEntriesInMemory += totalCount; ingestCount += totalCount; ingestBytes += totalBytes; } }
checkArgument(m != null, "m is null"); if (m.size() == 0) throw new IllegalArgumentException("Can not add empty mutations");
if (m.size() == 0) throw new IllegalArgumentException("Can not add empty mutations");
private int countKVPairs(List<Mutation> mutations) { int count = 0; for (Mutation m : mutations) { count += m.size(); } return count; }
private void addMutationNoCloseCheck(Mutation m) throws MutationsRejectedException { if (m.size() == 0) { throw new IllegalArgumentException("Cannot add empty mutation."); } mutationBuffer.addMutation(writerId, m); }
void mutate(List<Mutation> mutations, int mutationCount) { Iterator<Mutation> iter = mutations.iterator(); while (iter.hasNext()) { wlock.lock(); try { if (nmPointer == 0) { throw new IllegalStateException("Native Map Deleted"); } modCount++; int count = 0; while (iter.hasNext() && count < 10) { Mutation mutation = iter.next(); mutationCount = _mutate(mutation, mutationCount); count += mutation.size(); } } finally { wlock.unlock(); } } }
public void mutate(List<Mutation> mutations, int mutationCount) { Iterator<Mutation> iter = mutations.iterator(); while (iter.hasNext()) { wlock.lock(); try { if (nmPointer == 0) { throw new IllegalStateException("Native Map Deleted"); } modCount++; int count = 0; while (iter.hasNext() && count < 10) { Mutation mutation = iter.next(); _mutate(mutation, mutationCount); mutationCount++; count += mutation.size(); } } finally { wlock.unlock(); } } }
/** * Applies changes to a row in the InMemoryMap * */ public void mutate(List<Mutation> mutations) { int numKVs = 0; for (int i = 0; i < mutations.size(); i++) numKVs += mutations.get(i).size(); // Can not update mutationCount while writes that started before // are in progress, this would cause partial mutations to be seen. // Also, can not continue until mutation count is updated, because // a read may not see a successful write. Therefore writes must // wait for writes that started before to finish. // // using separate lock from this map, to allow read/write in parallel synchronized (writeSerializer) { int kv = nextKVCount.getAndAdd(numKVs); try { map.mutate(mutations, kv); } finally { kvCount.set(kv + numKVs - 1); } } }
/** * Applies changes to a row in the InMemoryMap * */ public void mutate(List<Mutation> mutations) { int numKVs = 0; for (int i = 0; i < mutations.size(); i++) numKVs += mutations.get(i).size(); // Can not update mutationCount while writes that started before // are in progress, this would cause partial mutations to be seen. // Also, can not continue until mutation count is updated, because // a read may not see a successful write. Therefore writes must // wait for writes that started before to finish. // // using separate lock from this map, to allow read/write in parallel synchronized (writeSerializer) { int kv = nextKVCount.getAndAdd(numKVs); try { map.mutate(mutations, kv); } finally { kvCount.set(kv + numKVs - 1); } } }
public static void index(int numPartitions, Text docId, String doc, String splitRegex, BatchWriter bw) throws Exception { String[] tokens = doc.split(splitRegex); Text partition = genPartition(doc.hashCode() % numPartitions); Mutation m = new Mutation(partition); HashSet<String> tokensSeen = new HashSet<String>(); for (String token : tokens) { token = token.toLowerCase(); if (!tokensSeen.contains(token)) { tokensSeen.add(token); m.put(new Text(token), docId, new Value(new byte[0])); } } if (m.size() > 0) bw.addMutation(m); }
/** * Push a mutation into a table. If table is null, the defaultTable will be used. If * {@link AccumuloOutputFormat#canCreateTables(JobContext)} is set, the table will be created if * it does not exist. The table name must only contain alphanumerics and underscore. */ @Override public void write(Text table, Mutation mutation) throws IOException { if (table == null || table.toString().isEmpty()) table = this.defaultTableName; if (!simulate && table == null) throw new IOException("No table or default table specified. Try simulation mode next time"); ++mutCount; valCount += mutation.size(); printMutation(table, mutation); if (simulate) return; if (!bws.containsKey(table)) try { addTable(table); } catch (Exception e) { log.error("Could not add table '" + table + "'", e); throw new IOException(e); } try { bws.get(table).addMutation(mutation); } catch (MutationsRejectedException e) { throw new IOException(e); } }
/** * Push a mutation into a table. If table is null, the defaultTable will be used. If * {@link AccumuloOutputFormat#canCreateTables(JobConf)} is set, the table will be created if it * does not exist. The table name must only contain alphanumerics and underscore. */ @Override public void write(Text table, Mutation mutation) throws IOException { if (table == null || table.toString().isEmpty()) table = this.defaultTableName; if (!simulate && table == null) throw new IOException("No table or default table specified. Try simulation mode next time"); ++mutCount; valCount += mutation.size(); printMutation(table, mutation); if (simulate) return; if (!bws.containsKey(table)) try { addTable(table); } catch (final Exception e) { log.error("Could not add table '" + table + "'", e); throw new IOException(e); } try { bws.get(table).addMutation(mutation); } catch (MutationsRejectedException e) { throw new IOException(e); } }
/** * Push a mutation into a table. If table is null, the defaultTable will be used. If * {@link AccumuloOutputFormat#canCreateTables(JobContext)} is set, the table will be created if * it does not exist. The table name must only contain alphanumerics and underscore. */ @Override public void write(Text table, Mutation mutation) throws IOException { if (table == null || table.toString().isEmpty()) table = this.defaultTableName; if (!simulate && table == null) throw new IOException("No table or default table specified. Try simulation mode next time"); ++mutCount; valCount += mutation.size(); printMutation(table, mutation); if (simulate) return; if (!bws.containsKey(table)) try { addTable(table); } catch (Exception e) { log.error("Could not add table '" + table + "'", e); throw new IOException(e); } try { bws.get(table).addMutation(mutation); } catch (MutationsRejectedException e) { throw new IOException(e); } }
valCount += mutation.size(); printMutation(table, mutation);
public void commit(CommitSession commitSession, List<Mutation> mutations) { int totalCount = 0; long totalBytes = 0; // write the mutation to the in memory table for (Mutation mutation : mutations) { totalCount += mutation.size(); totalBytes += mutation.numBytes(); } tabletMemory.mutate(commitSession, mutations); synchronized (this) { if (writesInProgress < 1) { throw new IllegalStateException("commiting mutations after logging, but not waiting for any log messages"); } if (closed && closeComplete) { throw new IllegalStateException("tablet closed with outstanding messages to the logger"); } tabletMemory.updateMemoryUsageStats(); // decrement here in case an exception is thrown below writesInProgress--; if (writesInProgress == 0) this.notifyAll(); commitSession.decrementCommitsInProgress(); numEntries += totalCount; numEntriesInMemory += totalCount; ingestCount += totalCount; ingestBytes += totalBytes; } }
static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions, boolean delete) throws Exception { String[] tokens = doc.split("\\W+"); String partition = genPartition(doc.hashCode() % numPartitions); Mutation m = new Mutation(partition); HashSet<String> tokensSeen = new HashSet<>(); for (String token : tokens) { token = token.toLowerCase(); if (!tokensSeen.contains(token)) { tokensSeen.add(token); if (delete) m.putDelete(token, docId); else m.put(token, docId, new Value(new byte[0])); } } if (m.size() > 0) bw.addMutation(m); }
synchronized void addMutation(Mutation m) { if (m.size() == 0) throw new IllegalArgumentException("Can not add empty mutations"); long now = System.currentTimeMillis(); mutationCount++; for (ColumnUpdate u : m.getUpdates()) { Key key = new Key(m.getRow(), 0, m.getRow().length, u.getColumnFamily(), 0, u.getColumnFamily().length, u.getColumnQualifier(), 0, u.getColumnQualifier().length, u.getColumnVisibility(), 0, u.getColumnVisibility().length, u.getTimestamp()); if (u.isDeleted()) key.setDeleted(true); if (!u.hasTimestamp()) if (timeType.equals(TimeType.LOGICAL)) key.setTimestamp(mutationCount); else key.setTimestamp(now); table.put(new MockMemKey(key, mutationCount), new Value(u.getValue())); } }