SchemaField keyField = rb.req.getSearcher().getSchema().getUniqueKeyField(); if (null != keyField) if (!returnFields.contains(keyField))
SchemaField uniqueField = core.getSchema().getUniqueKeyField();
private void validateSchema(IndexSchema schema) { if (schema.getUniqueKeyField() == null) { throw new MorphlineCompilationException("Solr schema.xml is missing unique key field", config); } if (!schema.getUniqueKeyField().isRequired()) { throw new MorphlineCompilationException("Solr schema.xml must contain a required unique key field", config); } }
private void validateSchema(IndexSchema schema) { if (schema.getUniqueKeyField() == null) { throw new MorphlineCompilationException("Solr schema.xml is missing unique key field", config); } if (!schema.getUniqueKeyField().isRequired()) { throw new MorphlineCompilationException("Solr schema.xml must contain a required unique key field", config); } }
private void validateSchema(IndexSchema schema) { if (schema.getUniqueKeyField() == null) { throw new MorphlineCompilationException("Solr schema.xml is missing unique key field", config); } if (!schema.getUniqueKeyField().isRequired()) { throw new MorphlineCompilationException("Solr schema.xml must contain a required unique key field", config); } }
private void validateSchema(IndexSchema schema) { if (schema.getUniqueKeyField() == null) { throw new MorphlineCompilationException("Solr schema.xml is missing unique key field", config); } if (!schema.getUniqueKeyField().isRequired()) { throw new MorphlineCompilationException("Solr schema.xml must contain a required unique key field", config); } }
public ValueSource getValueSource(SchemaField field, QParser parser) { // default key field to unique key SchemaField keyField = keyFieldName==null ? schema.getUniqueKeyField() : schema.getField(keyFieldName); return new FileFloatSource(field, keyField, defVal, parser); }
public UpdateHandler(SolrCore core) { this.core=core; schema = core.getSchema(); idField = schema.getUniqueKeyField(); idFieldType = idField!=null ? idField.getType() : null; idTerm = idField!=null ? new Term(idField.getName(),"") : null; parseEventListeners(); }
public String getPrintableId(IndexSchema schema) { SchemaField sf = schema.getUniqueKeyField(); if (indexedId != null) { return schema.getUniqueKeyField().getType().indexedToReadable(indexedId); } if (doc != null) { return schema.printableUniqueKey(doc); } if (solrDoc != null) { SolrInputField field = solrDoc.getField(sf.getName()); if (field != null) { return field.getFirstValue().toString(); } } return "(null)"; }
protected NamedList getCollapseGroupResult(int docId, NamedList result) { String uniqueIdFieldname = searcher.getSchema().getUniqueKeyField().getName(); try { String schemaId = searcher.doc(docId).get(uniqueIdFieldname); NamedList collapseGroupResult = (NamedList) result.get(schemaId); if (collapseGroupResult == null) { collapseGroupResult = new NamedList(); result.add(schemaId, collapseGroupResult); } return collapseGroupResult; } catch (IOException e) { throw new RuntimeException(e); } } }
@Override public void load(SolrInputDocument doc) throws IOException, SolrServerException { String uniqueKeyFieldName = getSchema().getUniqueKeyField().getName(); Object id = doc.getFieldValue(uniqueKeyFieldName); if (id == null) { throw new IllegalArgumentException("Missing value for (required) unique document key: " + uniqueKeyFieldName + " (see Solr schema.xml)"); } try { context.write(new Text(id.toString()), new SolrInputDocumentWritable(doc)); } catch (InterruptedException e) { throw new IOException("Interrupted while writing " + doc, e); } if (LOG.isDebugEnabled()) { long numParserOutputBytes = 0; for (SolrInputField field : doc.values()) { numParserOutputBytes += sizeOf(field.getValue()); } context.getCounter(MorphlineCounters.class.getName(), MorphlineCounters.PARSER_OUTPUT_BYTES.toString()).increment(numParserOutputBytes); } context.getCounter(MorphlineCounters.class.getName(), MorphlineCounters.DOCS_READ.toString()).increment(1); }
public GenerateSolrSequenceKey(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.baseIdFieldName = getConfigs().getString(config, "baseIdField", Fields.BASE_ID); this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); IndexSchema schema = locator.getIndexSchema(); SchemaField uniqueKey = schema.getUniqueKeyField(); uniqueKeyName = uniqueKey == null ? null : uniqueKey.getName(); String tmpIdPrefix = getConfigs().getString(config, "idPrefix", null); // for load testing only Random tmpRandomIdPrefx = null; if ("random".equals(tmpIdPrefix)) { // for load testing only tmpRandomIdPrefx = new Random(new SecureRandom().nextLong()); tmpIdPrefix = null; } idPrefix = tmpIdPrefix; randomIdPrefix = tmpRandomIdPrefx; validateArguments(); }
@Override public void load(SolrInputDocument doc) throws IOException, SolrServerException { String uniqueKeyFieldName = getSchema().getUniqueKeyField().getName(); Object id = doc.getFieldValue(uniqueKeyFieldName); if (id == null) { throw new IllegalArgumentException("Missing value for (required) unique document key: " + uniqueKeyFieldName + " (see Solr schema.xml)"); } try { context.write(new Text(id.toString()), new SolrInputDocumentWritable(doc)); } catch (InterruptedException e) { throw new IOException("Interrupted while writing " + doc, e); } if (LOG.isDebugEnabled()) { long numParserOutputBytes = 0; for (SolrInputField field : doc.values()) { numParserOutputBytes += sizeOf(field.getValue()); } context.getCounter(MorphlineCounters.class.getName(), MorphlineCounters.PARSER_OUTPUT_BYTES.toString()).increment(numParserOutputBytes); } context.getCounter(MorphlineCounters.class.getName(), MorphlineCounters.DOCS_READ.toString()).increment(1); }
public GenerateSolrSequenceKey(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.baseIdFieldName = getConfigs().getString(config, "baseIdField", Fields.BASE_ID); this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); IndexSchema schema = locator.getIndexSchema(); SchemaField uniqueKey = schema.getUniqueKeyField(); uniqueKeyName = uniqueKey == null ? null : uniqueKey.getName(); String tmpIdPrefix = getConfigs().getString(config, "idPrefix", null); // for load testing only Random tmpRandomIdPrefx = null; if ("random".equals(tmpIdPrefix)) { // for load testing only tmpRandomIdPrefx = new Random(new SecureRandom().nextLong()); tmpIdPrefix = null; } idPrefix = tmpIdPrefix; randomIdPrefix = tmpRandomIdPrefx; validateArguments(); }
public GenerateSolrSequenceKey(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.baseIdFieldName = getConfigs().getString(config, "baseIdField", Fields.BASE_ID); this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); IndexSchema schema = locator.getIndexSchema(); SchemaField uniqueKey = schema.getUniqueKeyField(); uniqueKeyName = uniqueKey == null ? null : uniqueKey.getName(); String tmpIdPrefix = getConfigs().getString(config, "idPrefix", null); // for load testing only Random tmpRandomIdPrefx = null; if ("random".equals(tmpIdPrefix)) { // for load testing only tmpRandomIdPrefx = new Random(new SecureRandom().nextLong()); tmpIdPrefix = null; } idPrefix = tmpIdPrefix; randomIdPrefix = tmpRandomIdPrefx; validateArguments(); }
public GenerateSolrSequenceKey(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.baseIdFieldName = getConfigs().getString(config, "baseIdField", Fields.BASE_ID); this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); IndexSchema schema = locator.getIndexSchema(); SchemaField uniqueKey = schema.getUniqueKeyField(); uniqueKeyName = uniqueKey == null ? null : uniqueKey.getName(); String tmpIdPrefix = getConfigs().getString(config, "idPrefix", null); // for load testing only Random tmpRandomIdPrefx = null; if ("random".equals(tmpIdPrefix)) { // for load testing only tmpRandomIdPrefx = new Random(new SecureRandom().nextLong()); tmpIdPrefix = null; } idPrefix = tmpIdPrefix; randomIdPrefix = tmpRandomIdPrefx; validateArguments(); }
@Override public String init(NamedList config, final SolrCore core) { String result = super.init(config, core); SolrParams initParams = SolrParams.toSolrParams(config); // Initialize Carrot2 controller. Pass initialization attributes, if any. HashMap<String, Object> initAttributes = new HashMap<String, Object>(); extractCarrotAttributes(initParams, initAttributes); this.controller.init(initAttributes); this.idFieldName = core.getSchema().getUniqueKeyField().getName(); // Make sure the requested Carrot2 clustering algorithm class is available String carrotAlgorithmClassName = initParams.get(CarrotParams.ALGORITHM); Class<?> algorithmClass = core.getResourceLoader().findClass(carrotAlgorithmClassName); if (!IClusteringAlgorithm.class.isAssignableFrom(algorithmClass)) { throw new IllegalArgumentException("Class provided as " + CarrotParams.ALGORITHM + " must implement " + IClusteringAlgorithm.class.getName()); } this.clusteringAlgorithmClass = (Class<? extends IClusteringAlgorithm>) algorithmClass; return result; }
public String getIndexedId(IndexSchema schema) { if (indexedId == null) { SchemaField sf = schema.getUniqueKeyField(); if (sf != null) { if (doc != null) { schema.getUniqueKeyField(); Field storedId = doc.getField(sf.getName()); indexedId = sf.getType().storedToIndexed(storedId); } if (solrDoc != null) { SolrInputField field = solrDoc.getField(sf.getName()); if (field != null) { indexedId = sf.getType().toInternal( field.getFirstValue().toString() ); } } } } return indexedId; }
public DocumentLoader getLoader() { if (context instanceof SolrMorphlineContext) { DocumentLoader loader = ((SolrMorphlineContext)context).getDocumentLoader(); if (loader != null) { return loader; } } SolrServer solrServer = getSolrServer(); if (solrServer instanceof CloudSolrServer) { try { ((CloudSolrServer)solrServer).setIdField(getIndexSchema().getUniqueKeyField().getName()); } catch (RuntimeException e) { try { solrServer.shutdown(); // release resources } catch (Exception ex2) { LOG.debug("Cannot get index schema and cannot shutdown CloudSolrServer", ex2); } throw new RuntimeException(e); // rethrow root cause } } return new SolrServerDocumentLoader(solrServer, batchSize); }
public void inform(SolrCore core) { schema = core.getLatestSchema(); // XML_STORE is not listed explicitly by the indexer informField (indexConfig.getField(FieldRole.XML_STORE), core); // This must be run before informField() registers default analyzers with the Schema registerXmlTextFields(); for (FieldDefinition xmlField : indexConfig.getFields()) { informField (xmlField, core); } if (xpathFieldConfig != null) { addXPathFields(); } SchemaField uniqueKeyField = schema.getUniqueKeyField(); if (uniqueKeyField == null) { logger.error("{} schema does not define any unique field", core.getName()); } else if (! uniqueKeyField.getName().equals(indexConfig.getFieldName(FieldRole.URI))) { logger.error("{} schema defines a different unique field than the uri field declared in lux configuration", core.getName()); } // must call this after making changes to the field map: schema.refreshAnalyzers(); }