MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_PARTITION, new AddPartitionEvent(tbl, newParts, true, this)); MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_PARTITION, new AddPartitionEvent(tbl, newParts, true, this), null, transactionalListenerResponses, ms); new AddPartitionEvent(tbl, existingParts, false, this), null, null, ms); MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_PARTITION, new AddPartitionEvent(tbl, parts, false, this), null, null, ms);
@Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { // Subscriber can get notification of newly add partition in a // particular table by listening on a topic named "dbName.tableName" // and message selector string as "HCAT_EVENT = HCAT_ADD_PARTITION" if (partitionEvent.getStatus()) { Table table = partitionEvent.getTable(); String topicName = getTopicName(table); if (topicName != null && !topicName.equals("")) { send(messageFactory.buildAddPartitionMessage(table, partitionEvent.getPartitionIterator()), topicName); } else { LOG.info("Topic name not found in metastore. Suppressing HCatalog notification for " + partitionEvent.getTable().getDbName() + "." + partitionEvent.getTable().getTableName() + " To enable notifications for this table, please do alter table set properties (" + HCatConstants.HCAT_MSGBUS_TOPIC_NAME + "=<dbname>.<tablename>) or whatever you want topic name to be."); } } }
/** * @param partitionEvent partition event * @throws MetaException */ @Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { Table t = partitionEvent.getTable(); PartitionFilesIterator fileIter = MetaStoreUtils.isExternalTable(t) ? null : new PartitionFilesIterator(partitionEvent.getPartitionIterator(), t); EventMessage msg = MessageBuilder.getInstance() .buildAddPartitionMessage(t, partitionEvent.getPartitionIterator(), fileIter); MessageSerializer serializer = msgEncoder.getSerializer(); NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_PARTITION.toString(), serializer.serialize(msg)); event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); process(event, partitionEvent); }
assertEquals(notifyList.size(), listSize); AddPartitionEvent partEvent = (AddPartitionEvent)(notifyList.get(listSize-1)); assert partEvent.getStatus(); assertEquals(envContext, partEvent.getEnvironmentContext()); assertEquals(notifyList.size(), listSize); AddPartitionEvent appendPartEvent = (AddPartitionEvent)(notifyList.get(listSize-1)); assert appendPartEvent.getStatus(); assertEquals(envContext, appendPartEvent.getEnvironmentContext());
private void fireMetaStoreAddPartitionEvent(final Table tbl, final PartitionSpecProxy partitionSpec, final EnvironmentContext envContext, boolean success) throws MetaException { if (tbl != null && partitionSpec != null) { AddPartitionEvent addPartitionEvent = new AddPartitionEvent(tbl, partitionSpec, success, this); addPartitionEvent.setEnvironmentContext(envContext); for (MetaStoreEventListener listener : listeners) { listener.onAddPartition(addPartitionEvent); } } }
/** * Listener which fires when a partition is added. * * <p>For auditing purposes the read/write differential is the non-existence * and existence of the added partition respectively.</p> * * @param event The add partition event */ @Override public void onAddPartition(AddPartitionEvent event) throws MetaException { try { Table table = new Table(event.getTable()); Set<ReadEntity> readEntities = new HashSet<>(); Set<WriteEntity> writeEntities = new HashSet<>(); for (org.apache.hadoop.hive.metastore.api.Partition partition : event.getPartitions()) { writeEntities.add( new WriteEntity( new Partition(table, partition), WriteType.INSERT ) ); } run(readEntities, writeEntities, HiveOperation.THRIFT_ADD_PARTITION); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { // don't sync path if the operation has failed if (!partitionEvent.getStatus()) { LOGGER.debug("Skip syncing path with Sentry server for onAddPartition event," + " since the operation failed. \n"); return; } for (Partition part : partitionEvent.getPartitions()) { if (part.getSd() != null && part.getSd().getLocation() != null) { String authzObj = part.getDbName() + "." + part.getTableName(); String path = part.getSd().getLocation(); for (SentryMetastoreListenerPlugin plugin : sentryPlugins) { plugin.addPath(authzObj, path); } } } super.onAddPartition(partitionEvent); }
private void fireMetaStoreAddPartitionEvent(final Table tbl, final PartitionSpecProxy partitionSpec, final EnvironmentContext envContext, boolean success) throws MetaException { if (tbl != null && partitionSpec != null) { AddPartitionEvent addPartitionEvent = new AddPartitionEvent(tbl, partitionSpec, success, this); addPartitionEvent.setEnvironmentContext(envContext); for (MetaStoreEventListener listener : listeners) { listener.onAddPartition(addPartitionEvent); } } }
/** * @param partitionEvent partition event * @throws MetaException */ public void onAddPartition (AddPartitionEvent partitionEvent) throws MetaException { Table t = partitionEvent.getTable(); NotificationEvent event = new NotificationEvent(0, now(), HCatConstants.HCAT_ADD_PARTITION_EVENT, msgFactory.buildAddPartitionMessage(t, partitionEvent.getPartitionIterator()).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); enqueue(event); }
/** * Listener which fires when a partition is added. * * <p>For auditing purposes the read/write differential is the non-existence * and existence of the added partition respectively.</p> * * @param event The add partition event */ @Override public void onAddPartition(AddPartitionEvent event) throws MetaException { try { Table table = new Table(event.getTable()); Set<ReadEntity> readEntities = new HashSet<>(); Set<WriteEntity> writeEntities = new HashSet<>(); for (org.apache.hadoop.hive.metastore.api.Partition partition : event.getPartitions()) { writeEntities.add( new WriteEntity( new Partition(table, partition), WriteType.INSERT ) ); } run(readEntities, writeEntities, HiveOperation.THRIFT_ADD_PARTITION); } catch (Exception e) { throw new RuntimeException(e); } }
Assert.assertTrue(partEvent.getStatus()); Partition part = msc.getPartition("hive2038", "tmptbl", "b=2011"); Partition partAdded = partEvent.getPartitionIterator().next(); partAdded.setWriteId(part.getWriteId()); validateAddPartition(part, partAdded); validateTableInAddPartition(tbl, partEvent.getTable()); validateAddPartition(part, prePartEvent.getPartitions().get(0)); ++listSize; AddPartitionEvent multiplePartitionEvent = (AddPartitionEvent)(notifyList.get(listSize-1)); assertEquals("Unexpected table value.", table, multiplePartitionEvent.getTable()); List<Partition> multiParts = Lists.newArrayList(multiplePartitionEvent.getPartitionIterator()); assertEquals("Unexpected number of partitions in event!", 3, multiParts.size()); assertEquals("Unexpected partition value.", partition1.getValues(), multiParts.get(0).getValues()); Partition partAppended = appendPartEvent.getPartitionIterator().next(); validateAddPartition(newPart, partAppended);
private void fireMetaStoreAddPartitionEvent(final Table tbl, final List<Partition> parts, final EnvironmentContext envContext, boolean success) throws MetaException { if (tbl != null && parts != null && !parts.isEmpty()) { AddPartitionEvent addPartitionEvent = new AddPartitionEvent(tbl, parts, success, this); addPartitionEvent.setEnvironmentContext(envContext); for (MetaStoreEventListener listener : listeners) { listener.onAddPartition(addPartitionEvent); } } }
/** * @param partitionEvent partition event * @throws MetaException */ public void onAddPartition (AddPartitionEvent partitionEvent) throws MetaException { Table t = partitionEvent.getTable(); NotificationEvent event = new NotificationEvent(0, now(), HCatConstants.HCAT_ADD_PARTITION_EVENT, msgFactory.buildAddPartitionMessage(t, partitionEvent.getPartitionIterator()).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); enqueue(event); }
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_PARTITION, new AddPartitionEvent(tbl, partitionSpecProxy, true, this)); MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_PARTITION, new AddPartitionEvent(tbl, partitionSpecProxy, true, this), null, transactionalListenerResponses, ms);
@Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { // Subscriber can get notification of newly add partition in a // particular table by listening on a topic named "dbName.tableName" // and message selector string as "HCAT_EVENT = HCAT_ADD_PARTITION" if (partitionEvent.getStatus()) { Table table = partitionEvent.getTable(); String topicName = getTopicName(table); if (topicName != null && !topicName.equals("")) { send(messageFactory.buildAddPartitionMessage(table, partitionEvent.getPartitionIterator()), topicName); } else { LOG.info("Topic name not found in metastore. Suppressing HCatalog notification for " + partitionEvent.getTable().getDbName() + "." + partitionEvent.getTable().getTableName() + " To enable notifications for this table, please do alter table set properties (" + HCatConstants.HCAT_MSGBUS_TOPIC_NAME + "=<dbname>.<tablename>) or whatever you want topic name to be."); } } }
private void fireMetaStoreAddPartitionEvent(final Table tbl, final List<Partition> parts, final EnvironmentContext envContext, boolean success) throws MetaException { if (tbl != null && parts != null && !parts.isEmpty()) { AddPartitionEvent addPartitionEvent = new AddPartitionEvent(tbl, parts, success, this); addPartitionEvent.setEnvironmentContext(envContext); for (MetaStoreEventListener listener : listeners) { listener.onAddPartition(addPartitionEvent); } } }
/** * @param partitionEvent partition event * @throws MetaException */ @Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { Table t = partitionEvent.getTable(); String msg = msgFactory .buildAddPartitionMessage(t, partitionEvent.getPartitionIterator(), new PartitionFilesIterator(partitionEvent.getPartitionIterator(), t)).toString(); NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_PARTITION.toString(), msg); event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); process(event, partitionEvent); }
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, EventType.ADD_PARTITION, new AddPartitionEvent(tbl, Arrays.asList(part), true, this), envContext); MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ADD_PARTITION, new AddPartitionEvent(tbl, Arrays.asList(part), success, this), envContext, transactionalListenerResponses, ms);
@Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { // Subscriber can get notification of newly add partition in a // particular table by listening on a topic named "dbName.tableName" // and message selector string as "HCAT_EVENT = HCAT_ADD_PARTITION" if (partitionEvent.getStatus()) { Table table = partitionEvent.getTable(); String topicName = getTopicName(table); if (topicName != null && !topicName.equals("")) { send(messageFactory.buildAddPartitionMessage(table, partitionEvent.getPartitionIterator()), topicName); } else { LOG.info("Topic name not found in metastore. Suppressing HCatalog notification for " + partitionEvent.getTable().getDbName() + "." + partitionEvent.getTable().getTableName() + " To enable notifications for this table, please do alter table set properties (" + HCatConstants.HCAT_MSGBUS_TOPIC_NAME + "=<dbname>.<tablename>) or whatever you want topic name to be."); } } }
private void fireMetaStoreAddPartitionEvent(final RawStore ms, final Partition part, final EnvironmentContext envContext, boolean success) throws MetaException { final Table tbl = ms.getTable(part.getDbName(), part.getTableName()); for (MetaStoreEventListener listener : listeners) { AddPartitionEvent addPartitionEvent = new AddPartitionEvent(tbl, part, success, this); addPartitionEvent.setEnvironmentContext(envContext); listener.onAddPartition(addPartitionEvent); } }