private void checkOutputSpecs(Configuration conf) { if (ConfigHelper.getOutputKeyspace(conf) == null) { throw new UnsupportedOperationException("you must set the keyspace with setTable()"); } }
/** * Connects to the given server:port and returns a client based on the given socket that points to the configured * keyspace, and is logged in with the configured credentials. * * @param host fully qualified host name to connect to * @param port RPC port of the server * @param conf a job configuration * @return a cassandra client * @throws Exception set of thrown exceptions may be implementation defined, * depending on the used transport factory */ public static Cassandra.Client createAuthenticatedClient(String host, int port, Configuration conf) throws Exception { logger.debug("Creating authenticated client for CF output format"); TTransport transport = ConfigHelper.getClientTransportFactory(conf).openTransport(host, port); TProtocol binaryProtocol = new TBinaryProtocol(transport, true, true); Cassandra.Client client = new Cassandra.Client(binaryProtocol); client.set_keyspace(ConfigHelper.getOutputKeyspace(conf)); String user = ConfigHelper.getOutputKeyspaceUserName(conf); String password = ConfigHelper.getOutputKeyspacePassword(conf); if ((user != null) && (password != null)) login(user, password, client); logger.debug("Authenticated client for CF output format created successfully"); return client; }
if (client != null) client.set_keyspace(ConfigHelper.getOutputKeyspace(conf)); String user = ConfigHelper.getOutputKeyspaceUserName(conf); String password = ConfigHelper.getOutputKeyspacePassword(conf);
private void checkOutputSpecs(Configuration conf) { if (ConfigHelper.getOutputKeyspace(conf) == null) { throw new UnsupportedOperationException("you must set the keyspace with setTable()"); } }
private void checkOutputSpecs(Configuration conf) { if (ConfigHelper.getOutputKeyspace(conf) == null) { throw new UnsupportedOperationException("you must set the keyspace with setTable()"); } }
private void checkOutputSpecs(Configuration conf) { if (ConfigHelper.getOutputKeyspace(conf) == null) { throw new UnsupportedOperationException("you must set the keyspace with setColumnFamily()"); } }
protected void checkOutputSpecs(Configuration conf) { if (ConfigHelper.getOutputKeyspace(conf) == null) throw new UnsupportedOperationException("You must set the keyspace with setOutputKeyspace()"); if (ConfigHelper.getOutputPartitioner(conf) == null) throw new UnsupportedOperationException("You must set the output partitioner to the one used by your Cassandra cluster"); if (ConfigHelper.getOutputInitialAddress(conf) == null) throw new UnsupportedOperationException("You must set the initial output address to a Cassandra node"); }
protected void checkOutputSpecs(Configuration conf) { if (ConfigHelper.getOutputKeyspace(conf) == null) throw new UnsupportedOperationException("You must set the keyspace with setOutputKeyspace()"); if (ConfigHelper.getOutputPartitioner(conf) == null) throw new UnsupportedOperationException("You must set the output partitioner to the one used by your Cassandra cluster"); if (ConfigHelper.getOutputInitialAddress(conf) == null) throw new UnsupportedOperationException("You must set the initial output address to a Cassandra node"); }
protected void checkOutputSpecs(Configuration conf) { if (ConfigHelper.getOutputKeyspace(conf) == null) throw new UnsupportedOperationException("You must set the keyspace with setOutputKeyspace()"); if (ConfigHelper.getOutputPartitioner(conf) == null) throw new UnsupportedOperationException("You must set the output partitioner to the one used by your Cassandra cluster"); if (ConfigHelper.getOutputInitialAddress(conf) == null) throw new UnsupportedOperationException("You must set the initial output address to a Cassandra node"); }
protected void checkOutputSpecs(Configuration conf) { if (ConfigHelper.getOutputKeyspace(conf) == null) throw new UnsupportedOperationException("You must set the keyspace with setOutputKeyspace()"); if (ConfigHelper.getOutputPartitioner(conf) == null) throw new UnsupportedOperationException("You must set the output partitioner to the one used by your Cassandra cluster"); if (ConfigHelper.getOutputInitialAddress(conf) == null) throw new UnsupportedOperationException("You must set the initial output address to a Cassandra node"); }
public NativeRingCache(Configuration conf, Metadata metadata) { this.partitioner = ConfigHelper.getOutputPartitioner(conf); this.metadata = metadata; String keyspace = ConfigHelper.getOutputKeyspace(conf); this.rangeMap = metadata.getTokenRanges() .stream() .collect(toMap(p -> p, p -> metadata.getReplicas('"' + keyspace + '"', p))); }
public NativeRingCache(Configuration conf, Metadata metadata) { this.partitioner = ConfigHelper.getOutputPartitioner(conf); this.metadata = metadata; String keyspace = ConfigHelper.getOutputKeyspace(conf); this.rangeMap = metadata.getTokenRanges() .stream() .collect(toMap(p -> p, p -> metadata.getReplicas('"' + keyspace + '"', p))); }
public NativeRingCache(Configuration conf, Metadata metadata) { this.partitioner = ConfigHelper.getOutputPartitioner(conf); this.metadata = metadata; String keyspace = ConfigHelper.getOutputKeyspace(conf); this.rangeMap = metadata.getTokenRanges() .stream() .collect(toMap(p -> p, p -> metadata.getReplicas('"' + keyspace + '"', p))); }
String keyspace = ConfigHelper.getOutputKeyspace(conf); String cfName = ConfigHelper.getOutputColumnFamily(conf); String query = "SELECT key_validator," +
CqlRecordWriter(Configuration conf) { this.conf = conf; this.queueSize = conf.getInt(CqlOutputFormat.QUEUE_SIZE, 32 * FBUtilities.getAvailableProcessors()); batchThreshold = conf.getLong(CqlOutputFormat.BATCH_THRESHOLD, 32); this.clients = new HashMap<>(); String keyspace = ConfigHelper.getOutputKeyspace(conf); try (Cluster cluster = CqlConfigHelper.getOutputCluster(ConfigHelper.getOutputInitialAddress(conf), conf)) { Metadata metadata = cluster.getMetadata(); ringCache = new NativeRingCache(conf, metadata); TableMetadata tableMetadata = metadata.getKeyspace(Metadata.quote(keyspace)).getTable(ConfigHelper.getOutputColumnFamily(conf)); clusterColumns = tableMetadata.getClusteringColumns(); partitionKeyColumns = tableMetadata.getPartitionKey(); String cqlQuery = CqlConfigHelper.getOutputCql(conf).trim(); if (cqlQuery.toLowerCase(Locale.ENGLISH).startsWith("insert")) throw new UnsupportedOperationException("INSERT with CqlRecordWriter is not supported, please use UPDATE/DELETE statement"); cql = appendKeyWhereClauses(cqlQuery); } catch (Exception e) { throw new RuntimeException(e); } }
CqlRecordWriter(Configuration conf) { this.conf = conf; this.queueSize = conf.getInt(CqlOutputFormat.QUEUE_SIZE, 32 * FBUtilities.getAvailableProcessors()); batchThreshold = conf.getLong(CqlOutputFormat.BATCH_THRESHOLD, 32); this.clients = new HashMap<>(); String keyspace = ConfigHelper.getOutputKeyspace(conf); try (Cluster cluster = CqlConfigHelper.getOutputCluster(ConfigHelper.getOutputInitialAddress(conf), conf)) { Metadata metadata = cluster.getMetadata(); ringCache = new NativeRingCache(conf, metadata); TableMetadata tableMetadata = metadata.getKeyspace(Metadata.quote(keyspace)).getTable(ConfigHelper.getOutputColumnFamily(conf)); clusterColumns = tableMetadata.getClusteringColumns(); partitionKeyColumns = tableMetadata.getPartitionKey(); String cqlQuery = CqlConfigHelper.getOutputCql(conf).trim(); if (cqlQuery.toLowerCase(Locale.ENGLISH).startsWith("insert")) throw new UnsupportedOperationException("INSERT with CqlRecordWriter is not supported, please use UPDATE/DELETE statement"); cql = appendKeyWhereClauses(cqlQuery); } catch (Exception e) { throw new RuntimeException(e); } }
private void prepareWriter() throws IOException { if (outputDir == null) { String keyspace = ConfigHelper.getOutputKeyspace(conf); //dir must be named by ks/cf for the loader outputDir = new File(getOutputLocation() + File.separator + keyspace + File.separator + ConfigHelper.getOutputColumnFamily(conf)); outputDir.mkdirs(); } if (writer == null) { AbstractType<?> subcomparator = null; if (cfType == CFType.SUPER) subcomparator = BytesType.instance; writer = new SSTableSimpleUnsortedWriter( outputDir, ConfigHelper.getOutputPartitioner(conf), ConfigHelper.getOutputKeyspace(conf), ConfigHelper.getOutputColumnFamily(conf), BytesType.instance, subcomparator, Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")), ConfigHelper.getOutputCompressionParamaters(conf)); this.loader = new SSTableLoader(outputDir, new ExternalClient(conf), new NullOutputHandler()); } }
CqlRecordWriter(Configuration conf) { this.conf = conf; this.queueSize = conf.getInt(CqlOutputFormat.QUEUE_SIZE, 32 * FBUtilities.getAvailableProcessors()); batchThreshold = conf.getLong(CqlOutputFormat.BATCH_THRESHOLD, 32); this.clients = new HashMap<>(); String keyspace = ConfigHelper.getOutputKeyspace(conf); try (Cluster cluster = CqlConfigHelper.getOutputCluster(ConfigHelper.getOutputInitialAddress(conf), conf)) { Metadata metadata = cluster.getMetadata(); ringCache = new NativeRingCache(conf, metadata); TableMetadata tableMetadata = metadata.getKeyspace(Metadata.quote(keyspace)).getTable(ConfigHelper.getOutputColumnFamily(conf)); clusterColumns = tableMetadata.getClusteringColumns(); partitionKeyColumns = tableMetadata.getPartitionKey(); String cqlQuery = CqlConfigHelper.getOutputCql(conf).trim(); if (cqlQuery.toLowerCase(Locale.ENGLISH).startsWith("insert")) throw new UnsupportedOperationException("INSERT with CqlRecordWriter is not supported, please use UPDATE/DELETE statement"); cql = appendKeyWhereClauses(cqlQuery); } catch (Exception e) { throw new RuntimeException(e); } }
private void setConfigs() throws IOException keyspace = ConfigHelper.getOutputKeyspace(conf); table = ConfigHelper.getOutputColumnFamily(conf);
private void setConfigs() throws IOException { // if anything is missing, exceptions will be thrown here, instead of on write() keyspace = ConfigHelper.getOutputKeyspace(conf); columnFamily = ConfigHelper.getOutputColumnFamily(conf); schema = CqlBulkOutputFormat.getColumnFamilySchema(conf, columnFamily); insertStatement = CqlBulkOutputFormat.getColumnFamilyInsertStatement(conf, columnFamily); outputDir = getColumnFamilyDirectory(); deleteSrc = CqlBulkOutputFormat.getDeleteSourceOnSuccess(conf); }