/** {@inheritDoc} */ @Override public String name() { return cfg.getName(); }
/** * Checks if given node is a IGFS node. * * @param node Node to check. * @return {@code True} if node has IGFS with this name, {@code false} otherwise. */ public boolean igfsNode(ClusterNode node) { return IgfsUtils.isIgfsNode(node, cfg.getName()); }
/** * Formats the file system removing all existing entries from it. * * @return Future. */ IgniteInternalFuture<?> clearAsync0() { GridFutureAdapter<?> fut = new GridFutureAdapter<>(); Thread t = new Thread(new FormatRunnable(fut), "igfs-format-" + cfg.getName() + "-" + FORMAT_THREAD_IDX_GEN.incrementAndGet()); t.setDaemon(true); t.start(); return fut; }
/** {@inheritDoc} */ @Override public void onKernalStop(boolean cancel) { for (IgfsContext igfsCtx : igfsCache.values()) { if (log.isDebugEnabled()) log.debug("Stopping igfs: " + igfsCtx.configuration().getName()); List<IgfsManager> mgrs = igfsCtx.managers(); for (ListIterator<IgfsManager> it = mgrs.listIterator(mgrs.size()); it.hasPrevious();) { IgfsManager mgr = it.previous(); mgr.onKernalStop(cancel); } } if (log.isDebugEnabled()) log.debug("Finished executing IGFS processor onKernalStop() callback."); }
/** * Executes IGFS task with overridden maximum range length (see * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information). * * @param task Task to execute. * @param rslvr Optional resolver to control split boundaries. * @param paths Collection of paths to be processed within this task. * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will * be ignored. Otherwise an exception will be thrown. * @param maxRangeLen Optional maximum range length. If {@code 0}, then by default all consecutive * IGFS blocks will be included. * @param arg Optional task argument. * @return Execution future. */ <T, R> IgniteInternalFuture<R> executeAsync0(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg) { return igfsCtx.kernalContext().task().execute(task, new IgfsTaskArgsImpl<>(cfg.getName(), paths, rslvr, skipNonExistentFiles, maxRangeLen, arg)); }
/** * Executes IGFS task asynchronously with overridden maximum range length (see * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information). * * @param taskCls Task class to execute. * @param rslvr Optional resolver to control split boundaries. * @param paths Collection of paths to be processed within this task. * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will * be ignored. Otherwise an exception will be thrown. * @param maxRangeLen Maximum range length. * @param arg Optional task argument. * @return Execution future. */ <T, R> IgniteInternalFuture<R> executeAsync0(Class<? extends IgfsTask<T, R>> taskCls, @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg) { return igfsCtx.kernalContext().task().execute((Class<IgfsTask<T, R>>)taskCls, new IgfsTaskArgsImpl<>(cfg.getName(), paths, rslvr, skipNonExistentFiles, maxRangeLen, arg)); }
/** * Get info for the given path. * * @param path Path. * @return Info. * @throws IgniteCheckedException If failed. */ @Nullable public IgfsEntryInfo infoForPath(IgfsPath path) throws IgniteCheckedException { return client ? runClientTask(new IgfsClientMetaInfoForPathCallable(cfg.getName(), IgfsUserContext.currentUser(), path)) : info(fileId(path)); }
/** {@inheritDoc} */ @Override public void stop(boolean cancel) { // Stop IGFS instances. for (IgfsContext igfsCtx : igfsCache.values()) { if (log.isDebugEnabled()) log.debug("Stopping igfs: " + igfsCtx.configuration().getName()); List<IgfsManager> mgrs = igfsCtx.managers(); for (ListIterator<IgfsManager> it = mgrs.listIterator(mgrs.size()); it.hasPrevious();) { IgfsManager mgr = it.previous(); mgr.stop(cancel); } igfsCtx.igfs().stop(cancel); } igfsCache.clear(); if (log.isDebugEnabled()) log.debug("IGFS processor stopped."); }
/** * Get IDs for the given path. * * @param path Path. * @return IDs. * @throws IgniteCheckedException If failed. */ public List<IgniteUuid> idsForPath(IgfsPath path) throws IgniteCheckedException { return client ? runClientTask(new IgfsClientMetaIdsForPathCallable(cfg.getName(), IgfsUserContext.currentUser(), path)) : fileIds(path); }
/** {@inheritDoc} */ @Override @Nullable public IgfsFile info(final IgfsPath path) { A.notNull(path, "path"); if (meta.isClient()) return meta.runClientTask(new IgfsClientInfoCallable(cfg.getName(), IgfsUserContext.currentUser(), path)); return safeOp(new Callable<IgfsFile>() { @Override public IgfsFile call() throws Exception { if (log.isDebugEnabled()) log.debug("Get file info: " + path); IgfsMode mode = resolveMode(path); return resolveFileInfo(path, mode); } }); }
/** {@inheritDoc} */ @Override public long size(final IgfsPath path) { A.notNull(path, "path"); if (meta.isClient()) return meta.runClientTask(new IgfsClientSizeCallable(cfg.getName(), IgfsUserContext.currentUser(), path)); return safeOp(new Callable<Long>() { @Override public Long call() throws Exception { return summary0(path).totalLength(); } }); }
/** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { metaCacheStartLatch = new CountDownLatch(1); cfg = igfsCtx.configuration(); evts = igfsCtx.kernalContext().event(); sampling = new IgfsSamplingKey(cfg.getName()); log = igfsCtx.kernalContext().log(IgfsMetaManager.class); metaCacheName = cfg.getMetaCacheConfiguration().getName(); }
/** * Get compute facade for client tasks. * * @return Compute facade. */ private IgniteCompute clientCompute() { assert client; IgniteCompute cliCompute0 = cliCompute; if (cliCompute0 == null) { IgniteEx ignite = igfsCtx.kernalContext().grid(); ClusterGroup cluster = ignite.cluster().forIgfsMetadataDataNodes(cfg.getName(), metaCacheName); cliCompute0 = ignite.compute(cluster); cliCompute = cliCompute0; } assert cliCompute0 != null; return cliCompute0; }
/** {@inheritDoc} */ @Override public void mkdirs(final IgfsPath path, @Nullable final Map<String, String> props) { A.notNull(path, "path"); if (meta.isClient()) { meta.runClientTask(new IgfsClientMkdirsCallable(cfg.getName(), IgfsUserContext.currentUser(), path, props)); return ; } safeOp(new Callable<Void>() { @Override public Void call() throws Exception { if (log.isDebugEnabled()) log.debug("Make directories: " + path); IgfsMode mode = resolveMode(path); switch (mode) { case PRIMARY: meta.mkdirs(path, props == null ? DFLT_DIR_META : new HashMap<>(props)); break; case DUAL_ASYNC: case DUAL_SYNC: await(path); meta.mkdirsDual(secondaryFs, path, props); break; case PROXY: secondaryFs.mkdirs(path, props); break; } return null; } }); }
/** {@inheritDoc} */ @Override public IgfsPathSummary summary(final IgfsPath path) { A.notNull(path, "path"); if (meta.isClient()) return meta.runClientTask(new IgfsClientSummaryCallable(cfg.getName(), IgfsUserContext.currentUser(), path)); return safeOp(new Callable<IgfsPathSummary>() { @Override public IgfsPathSummary call() throws Exception { if (log.isDebugEnabled()) log.debug("Calculating path summary: " + path); return summary0(path); } }); }
/** {@inheritDoc} */ @Override public void setTimes(final IgfsPath path, final long modificationTime, final long accessTime) { A.notNull(path, "path"); if (accessTime == -1 && modificationTime == -1) return; if (meta.isClient()) { meta.runClientTask(new IgfsClientSetTimesCallable(cfg.getName(), IgfsUserContext.currentUser(), path, accessTime, modificationTime)); return; } safeOp(new Callable<Void>() { @Override public Void call() throws Exception { IgfsMode mode = resolveMode(path); if (mode == PROXY) secondaryFs.setTimes(path, modificationTime, accessTime); else { meta.updateTimes(path, modificationTime, accessTime, IgfsUtils.isDualMode(mode) ? secondaryFs : null); } return null; } }); }
/** {@inheritDoc} */ @Override public boolean delete(final IgfsPath path, final boolean recursive) { A.notNull(path, "path"); if (meta.isClient()) return meta.runClientTask(new IgfsClientDeleteCallable(cfg.getName(), IgfsUserContext.currentUser(), path, recursive)); return safeOp(new Callable<Boolean>() { @Override public Boolean call() throws Exception { if (log.isDebugEnabled()) log.debug("Deleting file [path=" + path + ", recursive=" + recursive + ']'); if (F.eq(IgfsPath.ROOT, path)) return false; IgfsMode mode = resolveMode(path); if (mode == PROXY) return secondaryFs.delete(path, recursive); boolean dual = IgfsUtils.isDualMode(mode); if (dual) await(path); IgfsDeleteResult res = meta.softDelete(path, recursive, dual ? secondaryFs : null); // Record event if needed. if (res.success() && res.info() != null) IgfsUtils.sendEvents(igfsCtx.kernalContext(), path, res.info().isFile() ? EVT_IGFS_FILE_DELETED : EVT_IGFS_DIR_DELETED); return res.success(); } }); }
/** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { dataCacheStartLatch = new CountDownLatch(1); String igfsName = igfsCtx.configuration().getName(); topic = F.isEmpty(igfsName) ? TOPIC_IGFS : TOPIC_IGFS.topic(igfsName); igfsCtx.kernalContext().io().addMessageListener(topic, new GridMessageListener() { @Override public void onMessage(UUID nodeId, Object msg, byte plc) { if (msg instanceof IgfsBlocksMessage) processBlocksMessage(nodeId, (IgfsBlocksMessage)msg); else if (msg instanceof IgfsAckMessage) processAckMessage(nodeId, (IgfsAckMessage)msg); } }); igfsCtx.kernalContext().event().addLocalEventListener(new GridLocalEventListener() { @Override public void onEvent(Event evt) { assert evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT; DiscoveryEvent discoEvt = (DiscoveryEvent)evt; if (igfsCtx.igfsNode(discoEvt.eventNode())) { for (WriteCompletionFuture future : pendingWrites.values()) { future.onError(discoEvt.eventNode().id(), new ClusterTopologyCheckedException("Node left grid before write completed: " + evt.node().id())); } } } }, EVT_NODE_LEFT, EVT_NODE_FAILED); delWorker = new AsyncDeleteWorker(igfsCtx.kernalContext().igniteInstanceName(), "igfs-" + igfsName + "-delete-worker", log); dataCacheName = igfsCtx.configuration().getDataCacheConfiguration().getName(); }
/** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { if (!igfsCtx.configuration().isFragmentizerEnabled()) return; // We care only about node leave and fail events. igfsCtx.kernalContext().event().addLocalEventListener(new GridLocalEventListener() { @Override public void onEvent(Event evt) { assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED; DiscoveryEvent discoEvt = (DiscoveryEvent)evt; checkLaunchCoordinator(discoEvt); } }, EVT_NODE_LEFT, EVT_NODE_FAILED); fragmentizerWorker = new FragmentizerWorker(); String igfsName = igfsCtx.configuration().getName(); topic = F.isEmpty(igfsName) ? TOPIC_IGFS : TOPIC_IGFS.topic(igfsName); igfsCtx.kernalContext().io().addMessageListener(topic, fragmentizerWorker); new IgniteThread(fragmentizerWorker).start(); }
/** * Create data transfer object for IGFS configuration properties. * @param igfs IGFS configuration. */ public VisorIgfsConfiguration(FileSystemConfiguration igfs) { name = igfs.getName(); metaCacheName = igfs.getMetaCacheConfiguration().getName(); dataCacheName = igfs.getDataCacheConfiguration().getName(); blockSize = igfs.getBlockSize(); prefetchBlocks = igfs.getPrefetchBlocks(); streamBufSize = igfs.getBufferSize(); perNodeBatchSize = igfs.getPerNodeBatchSize(); perNodeParallelBatchCnt = igfs.getPerNodeParallelBatchCount(); dfltMode = igfs.getDefaultMode(); pathModes = igfs.getPathModes(); maxTaskRangeLen = igfs.getMaximumTaskRangeLength(); fragmentizerConcurrentFiles = igfs.getFragmentizerConcurrentFiles(); fragmentizerEnabled = igfs.isFragmentizerEnabled(); fragmentizerThrottlingBlockLen = igfs.getFragmentizerThrottlingBlockLength(); fragmentizerThrottlingDelay = igfs.getFragmentizerThrottlingDelay(); IgfsIpcEndpointConfiguration endpointCfg = igfs.getIpcEndpointConfiguration(); ipcEndpointCfg = endpointCfg != null ? endpointCfg.toString() : null; ipcEndpointEnabled = igfs.isIpcEndpointEnabled(); mgmtPort = igfs.getManagementPort(); seqReadsBeforePrefetch = igfs.getSequentialReadsBeforePrefetch(); colocateMeta = igfs.isColocateMetadata(); relaxedConsistency = igfs.isRelaxedConsistency(); updateFileLenOnFlush = igfs.isUpdateFileLengthOnFlush(); }