private void ensureTopicCreated(ZkUtils zkUtils, Set<String> allTopics, String topic, long retentionMs, int replicationFactor, int partitionCount) { Properties props = new Properties(); props.setProperty(LogConfig.RetentionMsProp(), Long.toString(retentionMs)); props.setProperty(LogConfig.CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY); if (!allTopics.contains(topic)) { AdminUtils.createTopic(zkUtils, topic, partitionCount, replicationFactor, props, RackAwareMode.Safe$.MODULE$); } else { try { AdminUtils.changeTopicConfig(zkUtils, topic, props); MetadataResponse.TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(JavaConversions.asScalaSet(Collections.singleton(topic)), zkUtils, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)).head(); maybeIncreaseTopicReplicationFactor(zkUtils, topicMetadata, replicationFactor, topic); maybeIncreaseTopicPartitionCount(zkUtils, topic, topicMetadata, partitionCount); } catch (RuntimeException re) { LOG.error("Skip updating topic " + topic + " configuration due to failure:" + re.getMessage() + "."); } } }
/** * Writes the supplied ACL information to ZK, where it will be picked up by the brokes authorizer. * * @param username the who. * @param permission the allow|deny. * @param resource the thing * @param ops the what. */ public void addUserAcl(final String username, final AclPermissionType permission, final ResourcePattern resource, final Set<AclOperation> ops) { final KafkaPrincipal principal = new KafkaPrincipal("User", username); final PermissionType scalaPermission = PermissionType$.MODULE$.fromJava(permission); final Set<Acl> javaAcls = ops.stream() .map(Operation$.MODULE$::fromJava) .map(op -> new Acl(principal, scalaPermission, "*", op)) .collect(Collectors.toSet()); final scala.collection.immutable.Set<Acl> scalaAcls = JavaConversions.asScalaSet(javaAcls).toSet(); final kafka.security.auth.ResourceType scalaResType = ResourceType$.MODULE$.fromJava(resource.resourceType()); final kafka.security.auth.Resource scalaResource = new kafka.security.auth.Resource(scalaResType, resource.name(), resource.patternType()); authorizer.addAcls(scalaAcls, scalaResource); addedAcls.add(scalaResource); }
.fetchTopicMetadataFromZk(JavaConversions.asScalaSet(Collections.singleton(topicName)), zkClient); if (topicMetadatas != null && topicMetadatas.size() > 0) { return JavaConversions.asJavaSet(topicMetadatas).iterator().next().partitionsMetadata().size();
public RRMultiWriter(String[] streams, DistributedLogClient client) { this.streams = streams; this.client = client; scala.collection.immutable.Set<ServiceFactory<VALUE, DLSN>> scalaSet = JavaConversions.asScalaSet(initializeServices(streams, client)).toSet(); this.service = Balancers.heap(new scala.util.Random(System.currentTimeMillis())) .newBalancer( Activity.value(scalaSet), NullStatsReceiver.get(), new NoBrokersAvailableException("No partitions available") ).toService(); }
/** * Deletes all ACLs for given user */ protected Future<ReconcileResult<Set<SimpleAclRule>>> internalDelete(String username, Set<SimpleAclRule> current) { try { HashMap<Resource, Set<Acl>> map = getResourceAclsMap(username, current); for (Map.Entry<Resource, Set<Acl>> entry: map.entrySet()) { scala.collection.mutable.Set remove = JavaConversions.asScalaSet(entry.getValue()); authorizer.removeAcls(remove.toSet(), entry.getKey()); } } catch (Exception e) { log.error("Deleting Acl rules for user {} failed", username, e); return Future.failedFuture(e); } return Future.succeededFuture(ReconcileResult.deleted()); }
/** * Cache pool based on a static list. * * @param cacheNodeSet static set of cache nodes to construct the cluster * @return a scala CachePoolCluster */ public static com.twitter.finagle.memcached.CachePoolCluster newStaticCluster( Set<CacheNode> cacheNodeSet) { scala.collection.immutable.Set<CacheNode> staticSet = JavaConversions.asScalaSet(cacheNodeSet).toSet(); return CachePoolCluster$.MODULE$.newStaticCluster(staticSet); }
/** * Create all ACLs for given user */ protected Future<ReconcileResult<Set<SimpleAclRule>>> internalCreate(String username, Set<SimpleAclRule> desired) { try { HashMap<Resource, Set<Acl>> map = getResourceAclsMap(username, desired); for (Map.Entry<Resource, Set<Acl>> entry: map.entrySet()) { scala.collection.mutable.Set add = JavaConversions.asScalaSet(entry.getValue()); authorizer.addAcls(add.toSet(), entry.getKey()); } } catch (Exception e) { log.error("Adding Acl rules for user {} failed", username, e); return Future.failedFuture(e); } return Future.succeededFuture(ReconcileResult.created(desired)); }
/** * Converts a {@link java.util.Set} to an immutable Scala Set. * * See scala.collection.JavaConversions.asScalaSet if you do * not need the returned Set to be immutable. * * @return an empty Set if the input is null. */ @SuppressWarnings("unchecked") public static <E> scala.collection.immutable.Set<E> asImmutableSet( java.util.Set<E> jSet ) { if (jSet == null) { return scala.collection.immutable.Set$.MODULE$.<E>empty(); } else { return asScalaSet(jSet).toSet(); } }
@SuppressWarnings({ "rawtypes", "unchecked" }) private static void verifyTopic(ZkUtils zkUtils, String topic) { Set topics = new HashSet(); topics.add(topic); // check # partition and the replication factor scala.collection.mutable.Map partitionAssignmentForTopics = zkUtils .getPartitionAssignmentForTopics(JavaConversions.asScalaSet(topics).toSeq()); scala.collection.Map partitionAssignment = (scala.collection.Map) partitionAssignmentForTopics.get(topic).get(); if (partitionAssignment.size() != 1) { throw new RuntimeException(String.format("The schema topic %s should have only 1 partition.", topic)); } // check the retention policy Properties prop = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic); String retentionPolicy = prop.getProperty(LogConfig.CleanupPolicyProp()); if (retentionPolicy == null || "compact".compareTo(retentionPolicy) != 0) { throw new RuntimeException(String.format("The retention policy of the schema topic %s must be compact.", topic)); } } }
private DirectedGraph<String> getDependencies(uk.gov.dstl.baleen.types.language.Sentence key) { List<WordToken> tokens = ImmutableList.copyOf(indexWords.get(key)); Set<Object> roots = new HashSet<>(); List<Edge<String>> edges = indexDependency .get(key) .stream() .peek( d -> { if (MaltParser.ROOT.equals(d.getDependencyType())) { roots.add(tokens.indexOf(d.getGovernor())); } }) .map( d -> { int source = tokens.indexOf(d.getGovernor()); int destination = tokens.indexOf(d.getDependent()); return new Edge<>(source, destination, d.getDependencyType().toLowerCase()); }) .collect(toList()); return new DirectedGraph<>( JavaConversions.asScalaBuffer(edges).toList(), JavaConversions.asScalaSet(roots).toSet()); } }
private DirectedGraph<String> getDependencies(uk.gov.dstl.baleen.types.language.Sentence key) { List<WordToken> tokens = ImmutableList.copyOf(indexWords.get(key)); Set<Object> roots = new HashSet<>(); List<Edge<String>> edges = indexDependency .get(key) .stream() .peek( d -> { if (MaltParser.ROOT.equals(d.getDependencyType())) { roots.add(tokens.indexOf(d.getGovernor())); } }) .map( d -> { int source = tokens.indexOf(d.getGovernor()); int destination = tokens.indexOf(d.getDependent()); return new Edge<>(source, destination, d.getDependencyType().toLowerCase()); }) .collect(toList()); return new DirectedGraph<>( JavaConversions.asScalaBuffer(edges).toList(), JavaConversions.asScalaSet(roots).toSet()); } }
private void verifySchemaTopic() { Set<String> topics = new HashSet<String>(); topics.add(topic); // check # partition and the replication factor scala.collection.Map partitionAssignment = zkUtils.getPartitionAssignmentForTopics( JavaConversions.asScalaSet(topics).toSeq()) .get(topic).get(); if (partitionAssignment.size() != 1) { log.warn("The schema topic " + topic + " should have only 1 partition."); } if (((Seq) partitionAssignment.get(0).get()).size() < desiredReplicationFactor) { log.warn("The replication factor of the schema topic " + topic + " is less than the " + "desired one of " + desiredReplicationFactor + ". If this is a production " + "environment, it's crucial to add more brokers and increase the replication " + "factor of the topic."); } // check the retention policy Properties prop = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic); String retentionPolicy = prop.getProperty(LogConfig.CleanupPolicyProp()); if (retentionPolicy == null || "compact".compareTo(retentionPolicy) != 0) { log.warn("The retention policy of the schema topic " + topic + " may be incorrect. " + "Please configure it with compact."); } }
/** * Zookeeper based cache pool cluster. * The cluster will monitor the underlying serverset changes and report the detected underlying * pool size. The cluster snapshot will be updated during cache-team's managed operation, and * the Future spool will be updated with corresponding changes. In case of zookeeper failure, * the backup pool will be used to fill the cluster after a certain timeout. * * @param zkPath the zookeeper path representing the cache pool * @param zkClient zookeeper client to read zookeeper * @param backupPool the backup static pool to use in case of ZK failure. Backup pool cannot be * null and empty backup pool means the same as no backup pool. * @param statsReceiver the destination to report the stats to * @return a scala CachePoolCluster */ public static com.twitter.finagle.memcached.CachePoolCluster newZkCluster( String zkPath, ZooKeeperClient zkClient, Set<CacheNode> backupPool, StatsReceiver statsReceiver) { scala.collection.immutable.Set<CacheNode> backupSet = JavaConversions.asScalaSet(backupPool).toSet(); return CachePoolCluster$.MODULE$.newZkCluster( zkPath, zkClient, scala.Option.apply(backupSet), statsReceiver); }
.fetchTopicMetadataFromZk(JavaConversions.asScalaSet(Collections.singleton(topicName)), zkClient); if (topicMetadatas != null && topicMetadatas.size() > 0) { return JavaConversions.asJavaSet(topicMetadatas).iterator().next().partitionsMetadata().size();
.fetchTopicMetadataFromZk(JavaConversions.asScalaSet(Collections.singleton(topicName)), zkClient); if (topicMetadatas != null && topicMetadatas.size() > 0) { return JavaConversions.asJavaSet(topicMetadatas).iterator().next().partitionsMetadata().size();
public RRMultiWriter(String[] streams, DistributedLogClient client) { this.streams = streams; this.client = client; scala.collection.immutable.Set<ServiceFactory<VALUE, DLSN>> scalaSet = JavaConversions.asScalaSet(initializeServices(streams, client)).toSet(); this.service = Balancers.heap(new scala.util.Random(System.currentTimeMillis())) .newBalancer( Activity.value(scalaSet), NullStatsReceiver.get(), new NoBrokersAvailableException("No partitions available") ).toService(); }
public RRMultiWriter(String[] streams, DistributedLogClient client) { this.streams = streams; this.client = client; scala.collection.immutable.Set<ServiceFactory<VALUE, DLSN>> scalaSet = JavaConversions.asScalaSet(initializeServices(streams, client)).toSet(); this.service = Balancers.heap(new scala.util.Random(System.currentTimeMillis())) .newBalancer( Activity.value(scalaSet), NullStatsReceiver.get(), new NoBrokersAvailableException("No partitions available") ).toService(); }
private java.util.Map<Resource, scala.collection.immutable.Set<Acl>> rolePrivilegesToResourceAcls(java.util.Map<String, scala.collection.immutable.Set<TSentryPrivilege>> rolePrivilegesMap) { final java.util.Map<Resource, scala.collection.immutable.Set<Acl>> resourceAclsMap = new HashMap<>(); for (String role : rolePrivilegesMap.keySet()) { scala.collection.immutable.Set<TSentryPrivilege> privileges = rolePrivilegesMap.get(role); final Iterator<TSentryPrivilege> iterator = privileges.iterator(); while (iterator.hasNext()) { TSentryPrivilege privilege = iterator.next(); final List<TAuthorizable> authorizables = privilege.getAuthorizables(); String host = null; String operation = privilege.getAction(); for (TAuthorizable tAuthorizable : authorizables) { if (tAuthorizable.getType().equals(KafkaAuthorizable.AuthorizableType.HOST.name())) { host = tAuthorizable.getName(); } else { Resource resource = new Resource(ResourceType$.MODULE$.fromString(tAuthorizable.getType()), tAuthorizable.getName()); if (operation.equals("*")) { operation = "All"; } Acl acl = new Acl(new KafkaPrincipal("role", role), Allow$.MODULE$, host, Operation$.MODULE$.fromString(operation)); Set<Acl> newAclsJava = new HashSet<Acl>(); newAclsJava.add(acl); addExistingAclsForResource(resourceAclsMap, resource, newAclsJava); final scala.collection.mutable.Set<Acl> aclScala = JavaConversions.asScalaSet(newAclsJava); resourceAclsMap.put(resource, aclScala.<Acl>toSet()); } } } } return resourceAclsMap; }