Map<String, ExpressionFunction> readExpressions() { Map<String, ExpressionFunction> expressions = new HashMap<>(); ApplicationFile expressionPath = application.getFile(modelFiles.expressionsPath()); if ( ! expressionPath.exists() || ! expressionPath.isDirectory()) return Collections.emptyMap(); for (ApplicationFile expressionFile : expressionPath.listFiles()) { try (BufferedReader reader = new BufferedReader(expressionFile.createReader())){ String name = expressionFile.getPath().getName(); expressions.put(name, readExpression(name, reader)); } catch (IOException e) { throw new UncheckedIOException("Failed reading " + expressionFile.getPath(), e); } catch (ParseException e) { throw new IllegalStateException("Invalid stored expression in " + expressionFile, e); } } return expressions; }
/** * List the files in this directory, optionally list files for subdirectories recursively as well. * * @param recurse Set to true if all files in the directory tree should be returned. * @return a list of files in this directory. */ public List<ApplicationFile> listFiles(boolean recurse) { List<ApplicationFile> ret = new ArrayList<>(); List<ApplicationFile> files = listFiles(); ret.addAll(files); if (recurse) { for (ApplicationFile file : files) { if (file.isDirectory()) { ret.addAll(file.listFiles(recurse)); } } } return ret; }
private Collection<Element> getPermanentServices(DeployState deployState) throws IOException, SAXException { List<Element> permanentServices = new ArrayList<>(); Optional<ApplicationPackage> applicationPackage = deployState.getPermanentApplicationPackage(); if (applicationPackage.isPresent()) { ApplicationFile file = applicationPackage.get().getFile(Path.fromString(ApplicationPackage.PERMANENT_SERVICES)); if (file.exists()) { try (Reader reader = file.createReader()) { Element permanentServicesRoot = getServicesFromReader(reader); permanentServices.addAll(getServiceElements(permanentServicesRoot)); } } } return permanentServices; }
/** * Reads the information about all the large (aka ranking) constants stored in the application package * (the constant value itself is replicated with file distribution). */ List<RankingConstant> readLargeConstants() { try { List<RankingConstant> constants = new ArrayList<>(); for (ApplicationFile constantFile : application.getFile(modelFiles.largeConstantsInfoPath()).listFiles()) { String[] parts = IOUtils.readAll(constantFile.createReader()).split(":"); constants.add(new RankingConstant(parts[0], TensorType.fromSpec(parts[1]), parts[2])); } return constants; } catch (IOException e) { throw new UncheckedIOException(e); } }
/** * Creates a rank profile not attached to any search definition, for each imported model in the application package, * and adds it to the given rank profile registry. */ private void createGlobalRankProfiles(DeployLogger deployLogger, ImportedMlModels importedModels, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { if ( ! importedModels.all().isEmpty()) { // models/ directory is available for (ImportedMlModel model : importedModels.all()) { RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry); rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()), model.name(), profile, queryProfiles.getRegistry(), model); convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false)); } } else { // generated and stored model information may be available instead ApplicationFile generatedModelsDir = applicationPackage.getFile(ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR); for (ApplicationFile generatedModelDir : generatedModelsDir.listFiles()) { String modelName = generatedModelDir.getPath().last(); if (modelName.contains(".")) continue; // Name space: Not a global profile RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry); rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile); convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false)); } } new Processing().processRankProfiles(deployLogger, rankProfileRegistry, queryProfiles, true, false); }
private Optional<Reader> optionalFile(Path filePath) { try { return Optional.of(getFile(filePath).createReader()); } catch (FileNotFoundException e) { return Optional.empty(); } }
@Override public int compareTo(ApplicationFile other) { return this.getPath().getName().compareTo((other).getPath().getName()); }
/** * List the files under this directory. If this is file, an empty list is returned. * Only immediate files/subdirectories are returned. * * @return a list of files in this directory. */ public List<ApplicationFile> listFiles() { return listFiles(defaultFilter); }
/** * Adds this expression to the application package, such that it can be read later. * * @param name the name of this ranking expression - may have 1-3 parts separated by dot where the first part * is always the model name */ void writeExpression(String name, ExpressionFunction expression) { StringBuilder b = new StringBuilder(expression.getBody().getRoot().toString()); for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) b.append('\n').append(input.getKey()).append('\t').append(input.getValue()); application.getFile(modelFiles.expressionPath(name)).writeFile(new StringReader(b.toString())); }
/** Adds this function expression to the application package so it can be read later. */ public void writeFunction(String name, RankingExpression expression) { application.getFile(modelFiles.functionsPath()).appendFile(name + "\t" + expression.getRoot().toString() + "\n"); }
/** Returns whether a model store for this application and model name exists */ public boolean exists() { return application.getFile(modelFiles.storedModelReplicatedPath()).exists(); }
/** Reads the previously stored function expressions for these arguments */ List<Pair<String, RankingExpression>> readFunctions() { try { ApplicationFile file = application.getFile(modelFiles.functionsPath()); if ( ! file.exists()) return Collections.emptyList(); List<Pair<String, RankingExpression>> functions = new ArrayList<>(); try (BufferedReader reader = new BufferedReader(file.createReader())) { String line; while (null != (line = reader.readLine())) { String[] parts = line.split("\t"); String name = parts[0]; try { RankingExpression expression = new RankingExpression(parts[0], parts[1]); functions.add(new Pair<>(name, expression)); } catch (ParseException e) { throw new IllegalStateException("Could not parse " + name, e); } } return functions; } } catch (IOException e) { throw new UncheckedIOException(e); } }
private void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage application) throws FileNotFoundException { // TODO: Handle validation of URI soon too. if (rankingConstant.getPathType() == RankingConstant.PathType.FILE) { String constantFile = rankingConstant.getFileName(); if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) && constantFile.startsWith(FilesApplicationPackage.preprocessed)) { constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length()); } ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile)); new ConstantTensorJsonValidator().validate(constantFile, rankingConstant.getTensorType(), tensorApplicationFile.createReader()); } }
@Override public int compareTo(ApplicationFile other) { if (other == this) return 0; return this.getPath().getName().compareTo((other).getPath().getName()); }
/** * Adds this constant to the application package as a file, * such that it can be distributed using file distribution. * * @return the path to the stored constant, relative to the application package root */ Path writeLargeConstant(String name, Tensor constant) { Path constantsPath = modelFiles.largeConstantsContentPath(); // "tbf" ending for "typed binary format" - recognized by the nodes receiving the file: Path constantPath = constantsPath.append(name + ".tbf"); // Remember the constant in a file we replicate in ZooKeeper application.getFile(modelFiles.largeConstantsInfoPath().append(name + ".constant")) .writeFile(new StringReader(name + ":" + constant.type() + ":" + correct(constantPath))); // Write content explicitly as a file on the file system as this is distributed using file distribution // - but only if this is a global model to avoid writing the same constants for each rank profile // where they are used if (modelFiles.modelName.isGlobal()) { createIfNeeded(constantsPath); IOUtils.writeFile(application.getFileReference(constantPath), TypedBinaryFormat.encode(constant)); } return correct(constantPath); }
/** * Append this constant to the single file used for small constants distributed as config */ public void writeSmallConstant(String name, Tensor constant) { // Secret file format for remembering constants: application.getFile(modelFiles.smallConstantsPath()).appendFile(name + "\t" + constant.type().toString() + "\t" + constant.toString() + "\n"); }
private List<Pair<String, Tensor>> readSmallConstants() { try { ApplicationFile file = application.getFile(modelFiles.smallConstantsPath()); if ( ! file.exists()) return Collections.emptyList(); List<Pair<String, Tensor>> constants = new ArrayList<>(); BufferedReader reader = new BufferedReader(file.createReader()); String line; while (null != (line = reader.readLine())) { String[] parts = line.split("\t"); String name = parts[0]; TensorType type = TensorType.fromSpec(parts[1]); Tensor tensor = Tensor.from(type, parts[2]); constants.add(new Pair<>(name, tensor)); } return constants; } catch (IOException e) { throw new UncheckedIOException(e); } }