/** * Makes a new event by copying properties from another event. * * @param other event to copy */ private InternalEvent(InternalEvent other) { this.eventString = new String(other.getEventString()); this.context = other.context; this.eventSha1Hash = other.getEventSha1Hash(); this.arrivalTime = other.getArrivalTime(); this.eventTime = other.getEventTime(); this.serialized = other.getSerialized(); if (other.getPartitions() == null) { this.partitions = null; } else { this.partitions = new LinkedHashMap<String, String>(other.getPartitions()); } /* * We rely on the implementation to provide a deep copy here. */ if (other.getEventObj() != null) { this.eventObj = other.getEventObj().copy(); } /* * Note that this is a shallow copy of the metadata. Some metadata elements are reused across * multiple events within a function invocation. */ this.metadata.putAll(other.metadata); }
@Override public LinkedHashMap<String, String> getPartitions() { LinkedHashMap<String, String> partitions = super.getPartitions(); if (partitions == null) { partitions = new LinkedHashMap<String, String>(1); super.setPartitions(partitions); } partitions.put(DYNAMODB_KEYS, this.stringKeys); return partitions; } }
/** * This is useful for operations that want to output multiple copies of the input event. * * @return new event with a copy of all properties from this event. */ public InternalEvent copy() { return new InternalEvent(this); } }
private BasicWrapper(final InternalEvent internal) { this.sha1Hash = internal.getEventSha1Hash(); this.timestamp = internal.getEventTime(); if (internal.getEventObj() != null) { this.payload = internal.getEventObj().getPayload(); } else { this.payload = null; } }
public void setEventTime(long eventTime) { this.eventTime = eventTime; this.addMetadata("eventEpochMs", new Long(eventTime)); this.addMetadata("sourceLagMs", new Long(System.currentTimeMillis() - this.getEventTime())); }
@Override public List<InternalEvent> perform(InternalEvent ievent) throws OperationException { if (ievent.getEventObj() == null) { throw new OperationException("Deserialized object is null"); payload = ievent.getEventObj().getField(this.path); } catch (FieldNotFoundException e) { throw new OperationException(e); LinkedHashMap<String, String> partitions = ievent.getPartitions(); try { InternalEvent newEvent = new InternalEvent(elm.toString(), ievent.getCtx(), ievent.getArrivalTime()); DeserializedEvent newDeserEvent = new GenericJsonEvent(elm.getAsJsonObject()); newEvent.setEventObj(newDeserEvent); newEvent.setEventTime(ievent.getEventTime()); }); newEvent.setPartitions(newPartitions);
@Test public void testGetEvaluatedPartitionsStatic() { List<PartitionSpec> partitionSpecs = new ArrayList<PartitionSpec>(1); PartitionSpec spec = new PartitionSpec("foo", Collections.emptyList(), PartitionSpec.Interpreter.STATIC, "123", 0); partitionSpecs.add(spec); PartitionOperation op = new PartitionOperation(partitionSpecs); InternalEvent ievent = new InternalEvent("foo", null, 1); DummyStringEvent devent = new DummyStringEvent(""); ievent.setEventObj(devent); op.perform(ievent); LinkedHashMap<String, String> actual = ievent.getPartitions(); LinkedHashMap<String, String> expected = new LinkedHashMap<String, String>(1); expected.put("foo", "123"); assertEquals(expected, actual); }
private S3Wrapper(final InternalEvent internal) { S3InternalEvent s3Event = ((S3InternalEvent) internal); this.s3Key = s3Event.getS3Key(); this.s3Bucket = s3Event.getS3Bucket(); this.s3KeyVersion = s3Event.getS3KeyVersion(); this.functionName = internal.getCtx().getContext().getFunctionName(); this.functionVersion = internal.getCtx().getContext().getFunctionVersion(); this.processingTime = System.currentTimeMillis(); this.timestamp = internal.getEventTime(); this.processingDelay = processingTime - timestamp; this.sha1Hash = internal.getEventSha1Hash(); if (internal.getEventObj() != null) { this.payload = internal.getEventObj().getPayload(); } else { this.payload = null; } }
String eventStr = ievent.getEventString(); DeserializedEvent data = deser.deserialize(ievent.getEventString()); logger.warn("Failed to deserialize: " + ievent.getEventString()); return null; ievent.setEventObj(data); return ievent; }).filter(Objects::nonNull); String raw = null; raw = this.ser.serialize(this.wrapper.getWrapped(ievent)); ievent.setSerialized(raw); return ievent; } catch (SerializationException e) { updateOldest(oldestArrivalTime, ievent.getArrivalTime()); updateOldest(oldestOccurrenceTime, ievent.getEventTime());
private PassthroughWrapper(final InternalEvent internal) { if (internal == null || internal.getEventObj() == null || internal.getEventObj().getPayload() == null) { wrapped = null; } else { wrapped = internal.getEventObj().getPayload(); } }
@Test(expected = OperationException.class) public void testNullField() throws FieldNotFoundException { InternalEvent ievent = new InternalEvent("foo", null, 1); DummyStringEvent devent = spy(new DummyStringEvent("")); ievent.setEventObj(devent); doThrow(FieldNotFoundException.class).when(devent).getFieldAsString("foo"); TimeOperation op = new TimeOperation("foo", TimeFieldType.SECONDS); op.perform(ievent); } }
private KinesisWrapper(final InternalEvent internal) { KinesisEventRecord eventRecord = ((KinesisInternalEvent) internal).getRecord(); Record record = eventRecord.getKinesis(); this.partitionKey = record.getPartitionKey(); this.sequenceNumber = record.getSequenceNumber(); this.eventSource = eventRecord.getEventSource(); this.sourceArn = eventRecord.getEventSourceARN(); this.functionName = internal.getCtx().getContext().getFunctionName(); this.functionVersion = internal.getCtx().getContext().getFunctionVersion(); this.processingTime = System.currentTimeMillis(); this.arrivalTime = record.getApproximateArrivalTimestamp().getTime(); this.timestamp = internal.getEventTime(); this.processingDelay = processingTime - timestamp; if (internal.getEventObj() != null) { this.payload = internal.getEventObj().getPayload(); } else { this.payload = null; } }
payload.append("\"_id\":"); payload.append("\""); payload.append(ievent.getEventSha1Hash()); payload.append("\","); if (this.usePartitionsForRouting && ievent.getPartitions() != null) { payload.append("\"" + this.routingFieldName + "\":"); payload.append("\""); payload.append(ievent.getPartitions().entrySet().stream().map(s -> s.getKey() + "=" + s.getValue()) .collect(Collectors.joining("/"))); payload.append("\","); payload.append(ievent.getSerialized()); payload.append("\n");
@Test public void testValidTime() throws FieldNotFoundException { InternalEvent ievent = new InternalEvent("foo", null, 1); DummyStringEvent devent = spy(new DummyStringEvent("")); ievent.setEventObj(devent); doReturn("1504728473").when(devent).getFieldAsString("foo"); TimeOperation op = new TimeOperation("foo", TimeFieldType.SECONDS); op.perform(ievent); assertEquals(1504728473000l, ievent.getEventTime()); }
if ((devent = ievent.getEventObj()) == null) { return null; if (ievent.getEventTime() > 1) { timestamp = ievent.getEventTime() / 1000.0; } else { return ievent;
@Override public byte[] serialize(InternalEvent ievent) { return (ievent.getSerialized() + '\n').getBytes(); } }
@Override public InternalEvent perform(InternalEvent ievent) { ievent.setPartitions(getPartitions(ievent.getEventObj())); return ievent; } }
@Override public InternalEvent perform(InternalEvent ievent) { String field; try { field = ievent.getEventObj().getFieldAsString(timeField); } catch (FieldNotFoundException e) { throw new OperationException( "time field " + timeField + " does not exist in " + ievent.getEventString()); } ievent.setEventTime(getTimestamp(field, timeFieldType)); return ievent; } }
@Override public byte[] serialize(InternalEvent ievent) { /* * Create a JSON line that describes the record for Splunk. */ StringBuilder payload = new StringBuilder(); payload.append("{"); if (this.index != null) { payload.append("\"index\":\""+ this.index +"\","); } payload.append("\"event\":"); payload.append(ievent.getSerialized()); payload.append(",\"time\": "); payload.append(String.format(Locale.US, "%.3f", ievent.getEventTime()/1000.0)); payload.append("}"); return payload.toString().getBytes(); } }
private String getIndexName(InternalEvent ievent) { if (this.dtFormat == null) { return this.index; } String index = this.index + this.dtFormat.print(ievent.getEventTime()); return index; } }