String cfName = ConfigHelper.getOutputColumnFamily(conf); String query = "SELECT key_validator," + " key_aliases," +
CqlRecordWriter(Configuration conf) { this.conf = conf; this.queueSize = conf.getInt(CqlOutputFormat.QUEUE_SIZE, 32 * FBUtilities.getAvailableProcessors()); batchThreshold = conf.getLong(CqlOutputFormat.BATCH_THRESHOLD, 32); this.clients = new HashMap<>(); String keyspace = ConfigHelper.getOutputKeyspace(conf); try (Cluster cluster = CqlConfigHelper.getOutputCluster(ConfigHelper.getOutputInitialAddress(conf), conf)) { Metadata metadata = cluster.getMetadata(); ringCache = new NativeRingCache(conf, metadata); TableMetadata tableMetadata = metadata.getKeyspace(Metadata.quote(keyspace)).getTable(ConfigHelper.getOutputColumnFamily(conf)); clusterColumns = tableMetadata.getClusteringColumns(); partitionKeyColumns = tableMetadata.getPartitionKey(); String cqlQuery = CqlConfigHelper.getOutputCql(conf).trim(); if (cqlQuery.toLowerCase(Locale.ENGLISH).startsWith("insert")) throw new UnsupportedOperationException("INSERT with CqlRecordWriter is not supported, please use UPDATE/DELETE statement"); cql = appendKeyWhereClauses(cqlQuery); } catch (Exception e) { throw new RuntimeException(e); } }
CqlRecordWriter(Configuration conf) { this.conf = conf; this.queueSize = conf.getInt(CqlOutputFormat.QUEUE_SIZE, 32 * FBUtilities.getAvailableProcessors()); batchThreshold = conf.getLong(CqlOutputFormat.BATCH_THRESHOLD, 32); this.clients = new HashMap<>(); String keyspace = ConfigHelper.getOutputKeyspace(conf); try (Cluster cluster = CqlConfigHelper.getOutputCluster(ConfigHelper.getOutputInitialAddress(conf), conf)) { Metadata metadata = cluster.getMetadata(); ringCache = new NativeRingCache(conf, metadata); TableMetadata tableMetadata = metadata.getKeyspace(Metadata.quote(keyspace)).getTable(ConfigHelper.getOutputColumnFamily(conf)); clusterColumns = tableMetadata.getClusteringColumns(); partitionKeyColumns = tableMetadata.getPartitionKey(); String cqlQuery = CqlConfigHelper.getOutputCql(conf).trim(); if (cqlQuery.toLowerCase(Locale.ENGLISH).startsWith("insert")) throw new UnsupportedOperationException("INSERT with CqlRecordWriter is not supported, please use UPDATE/DELETE statement"); cql = appendKeyWhereClauses(cqlQuery); } catch (Exception e) { throw new RuntimeException(e); } }
private void prepareWriter() throws IOException { if (outputDir == null) { String keyspace = ConfigHelper.getOutputKeyspace(conf); //dir must be named by ks/cf for the loader outputDir = new File(getOutputLocation() + File.separator + keyspace + File.separator + ConfigHelper.getOutputColumnFamily(conf)); outputDir.mkdirs(); } if (writer == null) { AbstractType<?> subcomparator = null; if (cfType == CFType.SUPER) subcomparator = BytesType.instance; writer = new SSTableSimpleUnsortedWriter( outputDir, ConfigHelper.getOutputPartitioner(conf), ConfigHelper.getOutputKeyspace(conf), ConfigHelper.getOutputColumnFamily(conf), BytesType.instance, subcomparator, Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")), ConfigHelper.getOutputCompressionParamaters(conf)); this.loader = new SSTableLoader(outputDir, new ExternalClient(conf), new NullOutputHandler()); } }
CqlRecordWriter(Configuration conf) { this.conf = conf; this.queueSize = conf.getInt(CqlOutputFormat.QUEUE_SIZE, 32 * FBUtilities.getAvailableProcessors()); batchThreshold = conf.getLong(CqlOutputFormat.BATCH_THRESHOLD, 32); this.clients = new HashMap<>(); String keyspace = ConfigHelper.getOutputKeyspace(conf); try (Cluster cluster = CqlConfigHelper.getOutputCluster(ConfigHelper.getOutputInitialAddress(conf), conf)) { Metadata metadata = cluster.getMetadata(); ringCache = new NativeRingCache(conf, metadata); TableMetadata tableMetadata = metadata.getKeyspace(Metadata.quote(keyspace)).getTable(ConfigHelper.getOutputColumnFamily(conf)); clusterColumns = tableMetadata.getClusteringColumns(); partitionKeyColumns = tableMetadata.getPartitionKey(); String cqlQuery = CqlConfigHelper.getOutputCql(conf).trim(); if (cqlQuery.toLowerCase(Locale.ENGLISH).startsWith("insert")) throw new UnsupportedOperationException("INSERT with CqlRecordWriter is not supported, please use UPDATE/DELETE statement"); cql = appendKeyWhereClauses(cqlQuery); } catch (Exception e) { throw new RuntimeException(e); } }
private void setConfigs() throws IOException { // if anything is missing, exceptions will be thrown here, instead of on write() keyspace = ConfigHelper.getOutputKeyspace(conf); columnFamily = ConfigHelper.getOutputColumnFamily(conf); schema = CqlBulkOutputFormat.getColumnFamilySchema(conf, columnFamily); insertStatement = CqlBulkOutputFormat.getColumnFamilyInsertStatement(conf, columnFamily); outputDir = getColumnFamilyDirectory(); deleteSrc = CqlBulkOutputFormat.getDeleteSourceOnSuccess(conf); }
table = ConfigHelper.getOutputColumnFamily(conf);
table = ConfigHelper.getOutputColumnFamily(conf);
table = ConfigHelper.getOutputColumnFamily(conf);