From 1974d4014b4e059ac8e3c7a3b8c78b4e841a9e51 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 14:14:33 -0400 Subject: [PATCH 01/50] DRILL-8556: Various PCAP and PCAP-NG Improvements PCAP-NG reader: - Stream blocks one at a time instead of reading the whole file into memory, and parse them directly, removing the pcapngdecoder dependency - Track interfaces per section: timestamps honor if_tsresol and if_tsoffset, and each packet is decoded for its interface's link type - Decode Ethernet (with 802.1Q/QinQ tags), raw IP, BSD loopback, PPP, Linux SLL/SLL2, and 802.11 / radiotap data frames - Add interface_id, interface_name, link_type, captured_length, comment, direction, reception_type, fcs_length, drop_count and packet_hash columns - Skip unknown block types instead of stopping, and tolerate a truncated final block - Decode each packet once per row instead of once per column - Stat queries: return null for missing options, add comment, make if_speed and if_tsoffset BIGINT, and use the interface's resolution for statistics timestamps - Support sessionizeTCPStreams Packet decoding (both readers): - Fix the TCP header length and payload offsets, bound payloads by the IP length and captured length, and walk IPv6 extension headers correctly; IPv6 TCP sequence numbers, flags and ports were misread - Detect ARP by EtherType and label ICMPv6 as ICMP; IPv6 packets with a hop-by-hop header were labeled ARP Format plugin: - Detect PCAP vs PCAP-NG per file; the format was a static field and was detected from the first file in the scan TCP sessions (both readers): - Share one TcpSessionizer; only TCP packets are tracked - Return sessions still open at end of file, with a new session_closed column, and ignore packets trailing a closed session - connection_time is null when the handshake was not captured Tests: replace the corrupted todo/ fixtures with valid captures, add fixtures for metadata, link types and sessionization, and correct expectations that encoded the old decoding bugs. --- contrib/format-pcapng/README.md | 31 +- contrib/format-pcapng/pom.xml | 5 - .../exec/store/pcap/PcapBatchReader.java | 173 ++--- .../drill/exec/store/pcap/TcpSessionizer.java | 161 ++++ .../drill/exec/store/pcap/decoder/Packet.java | 124 ++-- .../store/pcap/decoder/PacketConstants.java | 3 +- .../exec/store/pcap/decoder/TcpSession.java | 10 +- .../pcap/plugin/BasePcapFormatPlugin.java | 25 +- .../drill/exec/store/pcap/schema/Schema.java | 1 + .../exec/store/pcapng/PacketDecoder.java | 218 +++++- .../drill/exec/store/pcapng/PcapColumn.java | 692 +++++------------- .../exec/store/pcapng/PcapngBatchReader.java | 445 ++++++++++- .../drill/exec/store/pcapng/PcapngBlock.java | 48 ++ .../exec/store/pcap/TestPcapEVFReader.java | 3 +- .../exec/store/pcap/TestPcapRecordReader.java | 4 +- .../exec/store/pcap/TestSessionizePCAP.java | 45 +- .../store/pcapng/TestPcapngRecordReader.java | 192 ++++- .../pcapng/TestPcapngStatRecordReader.java | 23 +- .../test/resources/pcap/dataFromRemote.txt | 2 +- .../src/test/resources/pcapng/dhcp.pcapng | Bin 0 -> 1508 bytes .../resources/pcapng/dhcp_big_endian.pcapng | Bin 0 -> 1596 bytes .../resources/pcapng/encapsulations.pcapng | Bin 0 -> 896 bytes .../src/test/resources/pcapng/http.pcapng | Bin 0 -> 26596 bytes .../resources/pcapng/many_interfaces.pcapng | Bin 0 -> 20888 bytes .../src/test/resources/pcapng/metadata.pcapng | Bin 0 -> 1212 bytes .../src/test/resources/todo/dhcp.pcapng | Bin 1733 -> 0 bytes .../resources/todo/dhcp_big_endian.pcapng | Bin 1757 -> 0 bytes .../resources/todo/dhcp_little_endian.pcapng | Bin 1757 -> 0 bytes .../resources/todo/many_interfaces.pcapng | Bin 30743 -> 0 bytes 29 files changed, 1431 insertions(+), 774 deletions(-) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBlock.java create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/dhcp.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/dhcp_big_endian.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/encapsulations.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/http.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/many_interfaces.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/metadata.pcapng delete mode 100644 contrib/format-pcapng/src/test/resources/todo/dhcp.pcapng delete mode 100644 contrib/format-pcapng/src/test/resources/todo/dhcp_big_endian.pcapng delete mode 100644 contrib/format-pcapng/src/test/resources/todo/dhcp_little_endian.pcapng delete mode 100644 contrib/format-pcapng/src/test/resources/todo/many_interfaces.pcapng diff --git a/contrib/format-pcapng/README.md b/contrib/format-pcapng/README.md index 4b46c1c2caa..cfeea638de4 100644 --- a/contrib/format-pcapng/README.md +++ b/contrib/format-pcapng/README.md @@ -29,6 +29,31 @@ The following table lists configuration attributes: Attribute|Default Value|Description ---------|-------------|----------- stat|false|return the statistics data about the each pcapng file if true - -## ToDo -We plan to refactor the NG parser because of the `pcapngdecoder` so bad to process huge pcapng file. (Neither efficient nor support parser packet as stream) \ No newline at end of file +sessionizeTCPStreams|false|return one row per TCP session instead of one row per packet, for both PCAP and PCAP-NG. A session is written when it closes (FIN or RST). Sessions still open at the end of the file, such as ones that outlived the capture, are written last with `session_closed` false + +## PCAP-NG columns + +PCAP-NG files are streamed one block at a time, so file size is not limited by memory. + +In addition to the packet columns above, packet queries return the following metadata. +Timestamps honor each interface's `if_tsresol` and `if_tsoffset`. + +Column|Description +------|----------- +captured_length|Bytes of the packet stored in the file (`packet_length` is the original length on the wire) +interface_id|Index of the capturing interface within its section +interface_name|`if_name` of the capturing interface +link_type|[LINKTYPE_ value](https://www.tcpdump.org/linktypes.html) of the capturing interface +comment|Packet comments (`opt_comment`), newline separated +direction|`inbound` or `outbound` (`epb_flags`) +reception_type|`unicast`, `multicast`, `broadcast` or `promiscuous` (`epb_flags`) +fcs_length|Frame check sequence length in octets (`epb_flags`) +drop_count|Packets lost since the preceding packet (`epb_dropcount`) +packet_hash|`epb_hash` as `algorithm:hex`, such as `md5:9e107d9d...` + +IP, port and TCP columns are decoded for these link types: Ethernet (1), raw IP (101, 228, 229), +BSD loopback (0, 108), PPP (9), Linux cooked capture v1 and v2 (113, 276). On other link types they are null, +and the MAC address columns are only set for Ethernet. + +With `stat` set to true, options a block does not have are null. The `comment` column holds the section, interface, name resolution or +statistics block comment. diff --git a/contrib/format-pcapng/pom.xml b/contrib/format-pcapng/pom.xml index 6050fe23228..ada49f65cad 100644 --- a/contrib/format-pcapng/pom.xml +++ b/contrib/format-pcapng/pom.xml @@ -36,11 +36,6 @@ drill-java-exec ${project.version} - - fr.bmartel - pcapngdecoder - 1.2 - diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java index a09c3e3f272..379e5fceb6d 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java @@ -27,7 +27,6 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.store.pcap.decoder.Packet; import org.apache.drill.exec.store.pcap.decoder.PacketDecoder; -import org.apache.drill.exec.store.pcap.decoder.TcpSession; import org.apache.drill.exec.store.pcap.schema.Schema; import org.apache.drill.exec.store.pcap.plugin.PcapFormatConfig; import org.apache.drill.exec.vector.accessor.ScalarWriter; @@ -37,8 +36,6 @@ import java.io.IOException; import java.io.InputStream; import java.time.Instant; -import java.util.HashMap; -import java.util.Map; import static org.apache.drill.exec.store.pcap.PcapFormatUtils.parseBytesToASCII; @@ -82,26 +79,11 @@ public class PcapBatchReader implements ManagedReader { private ScalarWriter dataWriter; private ScalarWriter isCorruptWriter; private final PcapFormatConfig readerConfig; - // Writers for TCP Sessions - private ScalarWriter sessionStartTimeWriter; - private ScalarWriter sessionEndTimeWriter; - private ScalarWriter sessionDurationWriter; - private ScalarWriter connectionTimeWriter; - private ScalarWriter packetCountWriter; - private ScalarWriter originPacketCounterWriter; - private ScalarWriter remotePacketCounterWriter; - private ScalarWriter originDataVolumeWriter; - private ScalarWriter remoteDataVolumeWriter; - private ScalarWriter hostDataWriter; - private ScalarWriter remoteDataWriter; - private Map sessionQueue; + private TcpSessionizer sessionizer; public PcapBatchReader(PcapFormatConfig readerConfig, FileSchemaNegotiator negotiator) { this.readerConfig = readerConfig; - if (readerConfig.getSessionizeTCPStreams()) { - sessionQueue = new HashMap<>(); - } file = negotiator.file(); openFile(); @@ -113,14 +95,19 @@ public PcapBatchReader(PcapFormatConfig readerConfig, FileSchemaNegotiator negot // Creates writers for all fields (Since schema is known) rowWriter = loader.writer(); - populateColumnWriters(rowWriter); + if (readerConfig.getSessionizeTCPStreams()) { + sessionizer = new TcpSessionizer(rowWriter); + } else { + populateColumnWriters(rowWriter); + } } @Override public boolean next() { while (!rowWriter.isFull()) { if (!parseNextPacket(rowWriter)) { - return false; + // At end of file, more batches are needed only for open sessions that did not fit + return sessionizer != null && !sessionizer.writeOpenSessions(); } } return true; @@ -129,12 +116,6 @@ public boolean next() { @Override public void close() { - /* This warning could occur in the event of a corrupt or incomplete PCAP file. Specifically, - * if a session is started in one file and the end of the session is not captured in the same file. - */ - if (sessionQueue != null && !sessionQueue.isEmpty()) { - logger.warn("Unclosed sessions remaining in PCAP"); - } try { fsStream.close(); @@ -164,63 +145,39 @@ private void openFile() { } private void populateColumnWriters(RowSetLoader rowWriter) { - if (readerConfig.getSessionizeTCPStreams()) { - srcMacAddressWriter = rowWriter.scalar("src_mac_address"); - dstMacAddressWriter = rowWriter.scalar("dst_mac_address"); - dstIPWriter = rowWriter.scalar("dst_ip"); - srcIPWriter = rowWriter.scalar("src_ip"); - srcPortWriter = rowWriter.scalar("src_port"); - dstPortWriter = rowWriter.scalar("dst_port"); - sessionStartTimeWriter = rowWriter.scalar("session_start_time"); - sessionEndTimeWriter = rowWriter.scalar("session_end_time"); - sessionDurationWriter = rowWriter.scalar("session_duration"); - connectionTimeWriter = rowWriter.scalar("connection_time"); - tcpSessionWriter = rowWriter.scalar("tcp_session"); - packetCountWriter = rowWriter.scalar("total_packet_count"); - hostDataWriter = rowWriter.scalar("data_from_originator"); - remoteDataWriter = rowWriter.scalar("data_from_remote"); - - originPacketCounterWriter = rowWriter.scalar("packet_count_from_origin"); - remotePacketCounterWriter = rowWriter.scalar("packet_count_from_remote"); - originDataVolumeWriter = rowWriter.scalar("data_volume_from_origin"); - remoteDataVolumeWriter = rowWriter.scalar("data_volume_from_remote"); - isCorruptWriter = rowWriter.scalar("is_corrupt"); - - } else { - typeWriter = rowWriter.scalar("type"); - timestampWriter = rowWriter.scalar("packet_timestamp"); - timestampMicroWriter = rowWriter.scalar("timestamp_micro"); - networkWriter = rowWriter.scalar("network"); - srcMacAddressWriter = rowWriter.scalar("src_mac_address"); - dstMacAddressWriter = rowWriter.scalar("dst_mac_address"); - dstIPWriter = rowWriter.scalar("dst_ip"); - srcIPWriter = rowWriter.scalar("src_ip"); - srcPortWriter = rowWriter.scalar("src_port"); - dstPortWriter = rowWriter.scalar("dst_port"); - packetLengthWriter = rowWriter.scalar("packet_length"); - - //Writers for TCP Packets - tcpSessionWriter = rowWriter.scalar("tcp_session"); - tcpSequenceWriter = rowWriter.scalar("tcp_sequence"); - tcpAckNumberWriter = rowWriter.scalar("tcp_ack"); - tcpFlagsWriter = rowWriter.scalar("tcp_flags"); - tcpParsedFlagsWriter = rowWriter.scalar("tcp_parsed_flags"); - tcpNsWriter = rowWriter.scalar("tcp_flags_ns"); - tcpCwrWriter = rowWriter.scalar("tcp_flags_cwr"); - tcpEceWriter = rowWriter.scalar("tcp_flags_ece"); - tcpFlagsEceEcnCapableWriter = rowWriter.scalar("tcp_flags_ece_ecn_capable"); - tcpFlagsCongestionWriter = rowWriter.scalar("tcp_flags_ece_congestion_experienced"); - - tcpUrgWriter = rowWriter.scalar("tcp_flags_urg"); - tcpAckWriter = rowWriter.scalar("tcp_flags_ack"); - tcpPshWriter = rowWriter.scalar("tcp_flags_psh"); - tcpRstWriter = rowWriter.scalar("tcp_flags_rst"); - tcpSynWriter = rowWriter.scalar("tcp_flags_syn"); - tcpFinWriter = rowWriter.scalar("tcp_flags_fin"); - - dataWriter = rowWriter.scalar("data"); - isCorruptWriter = rowWriter.scalar("is_corrupt"); - } + typeWriter = rowWriter.scalar("type"); + timestampWriter = rowWriter.scalar("packet_timestamp"); + timestampMicroWriter = rowWriter.scalar("timestamp_micro"); + networkWriter = rowWriter.scalar("network"); + srcMacAddressWriter = rowWriter.scalar("src_mac_address"); + dstMacAddressWriter = rowWriter.scalar("dst_mac_address"); + dstIPWriter = rowWriter.scalar("dst_ip"); + srcIPWriter = rowWriter.scalar("src_ip"); + srcPortWriter = rowWriter.scalar("src_port"); + dstPortWriter = rowWriter.scalar("dst_port"); + packetLengthWriter = rowWriter.scalar("packet_length"); + + //Writers for TCP Packets + tcpSessionWriter = rowWriter.scalar("tcp_session"); + tcpSequenceWriter = rowWriter.scalar("tcp_sequence"); + tcpAckNumberWriter = rowWriter.scalar("tcp_ack"); + tcpFlagsWriter = rowWriter.scalar("tcp_flags"); + tcpParsedFlagsWriter = rowWriter.scalar("tcp_parsed_flags"); + tcpNsWriter = rowWriter.scalar("tcp_flags_ns"); + tcpCwrWriter = rowWriter.scalar("tcp_flags_cwr"); + tcpEceWriter = rowWriter.scalar("tcp_flags_ece"); + tcpFlagsEceEcnCapableWriter = rowWriter.scalar("tcp_flags_ece_ecn_capable"); + tcpFlagsCongestionWriter = rowWriter.scalar("tcp_flags_ece_congestion_experienced"); + + tcpUrgWriter = rowWriter.scalar("tcp_flags_urg"); + tcpAckWriter = rowWriter.scalar("tcp_flags_ack"); + tcpPshWriter = rowWriter.scalar("tcp_flags_psh"); + tcpRstWriter = rowWriter.scalar("tcp_flags_rst"); + tcpSynWriter = rowWriter.scalar("tcp_flags_syn"); + tcpFinWriter = rowWriter.scalar("tcp_flags_fin"); + + dataWriter = rowWriter.scalar("data"); + isCorruptWriter = rowWriter.scalar("is_corrupt"); } private boolean parseNextPacket(RowSetLoader rowWriter) { @@ -242,24 +199,8 @@ private boolean parseNextPacket(RowSetLoader rowWriter) { logger.debug("Invalid packet at offset {}", old); } - // If we are resessionizing the TCP Stream, add the packet to the stream - if (readerConfig.getSessionizeTCPStreams()) { - // If the session has not been seen before, add it to the queue - long sessionID = packet.getSessionHash(); - if (!sessionQueue.containsKey(sessionID)) { - logger.debug("Adding session {} to session queue.", sessionID); - sessionQueue.put(sessionID, new TcpSession(sessionID)); - } - - // When the session is closed, write it and remove it from the session queue. - sessionQueue.get(sessionID).addPacket(packet); - if (sessionQueue.get(sessionID).connectionClosed()) { - // Write out the session - addSessionDataToTable(sessionQueue.get(sessionID), rowWriter); - // Remove from the queue - sessionQueue.remove(sessionID); - } - + if (sessionizer != null) { + sessionizer.addPacket(packet); } else { addDataToTable(packet, decoder.getNetwork(), rowWriter); } @@ -295,34 +236,6 @@ private boolean getNextPacket(RowSetLoader rowWriter) { return true; } - private void addSessionDataToTable(TcpSession session, RowSetLoader rowWriter) { - rowWriter.start(); - - sessionStartTimeWriter.setTimestamp(session.getSessionStartTime()); - sessionEndTimeWriter.setTimestamp(session.getSessionEndTime()); - sessionDurationWriter.setPeriod(session.getSessionDuration()); - connectionTimeWriter.setPeriod(session.getConnectionTime()); - - srcMacAddressWriter.setString(session.getSrcMac()); - dstMacAddressWriter.setString(session.getDstMac()); - srcIPWriter.setString(session.getSrcIP()); - dstIPWriter.setString(session.getDstIP()); - srcPortWriter.setInt(session.getSrcPort()); - dstPortWriter.setInt(session.getDstPort()); - tcpSessionWriter.setLong(session.getSessionID()); - packetCountWriter.setInt(session.getPacketCount()); - - originPacketCounterWriter.setInt(session.getPacketCountFromOrigin()); - remotePacketCounterWriter.setInt(session.getPacketCountFromRemote()); - originDataVolumeWriter.setInt(session.getDataFromOriginator().length); - remoteDataVolumeWriter.setInt(session.getDataFromRemote().length); - isCorruptWriter.setBoolean(session.hasCorruptedData()); - - hostDataWriter.setString(session.getDataFromOriginatorAsString()); - remoteDataWriter.setString(session.getDataFromRemoteAsString()); - rowWriter.save(); - } - private void addDataToTable(Packet packet, int networkType, RowSetLoader rowWriter) { rowWriter.start(); diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java new file mode 100644 index 00000000000..e9993dbf416 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.drill.exec.physical.resultSet.RowSetLoader; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; +import org.apache.drill.exec.vector.accessor.ScalarWriter; + +/** + * Groups TCP packets into sessions and writes one row per session, using the + * sessionized schema of {@link org.apache.drill.exec.store.pcap.schema.Schema}, + * when the session closes. Shared by the PCAP and PCAP-NG readers. + */ +public class TcpSessionizer { + // In order of each session's first packet + private final Map openSessions = new LinkedHashMap<>(); + // Sessions already written, so the ACKs that trail a FIN do not start a new one + private final Set closedSessions = new HashSet<>(); + private final RowSetLoader rowWriter; + private final ScalarWriter srcMacAddressWriter; + private final ScalarWriter dstMacAddressWriter; + private final ScalarWriter dstIPWriter; + private final ScalarWriter srcIPWriter; + private final ScalarWriter srcPortWriter; + private final ScalarWriter dstPortWriter; + private final ScalarWriter sessionStartTimeWriter; + private final ScalarWriter sessionEndTimeWriter; + private final ScalarWriter sessionDurationWriter; + private final ScalarWriter connectionTimeWriter; + private final ScalarWriter tcpSessionWriter; + private final ScalarWriter packetCountWriter; + private final ScalarWriter hostDataWriter; + private final ScalarWriter remoteDataWriter; + private final ScalarWriter originPacketCounterWriter; + private final ScalarWriter remotePacketCounterWriter; + private final ScalarWriter originDataVolumeWriter; + private final ScalarWriter remoteDataVolumeWriter; + private final ScalarWriter isCorruptWriter; + private final ScalarWriter sessionClosedWriter; + + public TcpSessionizer(RowSetLoader rowWriter) { + this.rowWriter = rowWriter; + srcMacAddressWriter = rowWriter.scalar("src_mac_address"); + dstMacAddressWriter = rowWriter.scalar("dst_mac_address"); + dstIPWriter = rowWriter.scalar("dst_ip"); + srcIPWriter = rowWriter.scalar("src_ip"); + srcPortWriter = rowWriter.scalar("src_port"); + dstPortWriter = rowWriter.scalar("dst_port"); + sessionStartTimeWriter = rowWriter.scalar("session_start_time"); + sessionEndTimeWriter = rowWriter.scalar("session_end_time"); + sessionDurationWriter = rowWriter.scalar("session_duration"); + connectionTimeWriter = rowWriter.scalar("connection_time"); + tcpSessionWriter = rowWriter.scalar("tcp_session"); + packetCountWriter = rowWriter.scalar("total_packet_count"); + hostDataWriter = rowWriter.scalar("data_from_originator"); + remoteDataWriter = rowWriter.scalar("data_from_remote"); + originPacketCounterWriter = rowWriter.scalar("packet_count_from_origin"); + remotePacketCounterWriter = rowWriter.scalar("packet_count_from_remote"); + originDataVolumeWriter = rowWriter.scalar("data_volume_from_origin"); + remoteDataVolumeWriter = rowWriter.scalar("data_volume_from_remote"); + isCorruptWriter = rowWriter.scalar("is_corrupt"); + sessionClosedWriter = rowWriter.scalar("session_closed"); + } + + /** + * Adds a packet to its session, writing the session's row if the packet + * closes it. Packets other than TCP are ignored. + */ + public void addPacket(Packet packet) { + if (!packet.isTcpPacket()) { + return; + } + long sessionId = packet.getSessionHash(); + if (closedSessions.contains(sessionId)) { + if (!packet.getSynFlag()) { + return; + } + // The same addresses and ports opening a new connection + closedSessions.remove(sessionId); + } + TcpSession session = openSessions.computeIfAbsent(sessionId, TcpSession::new); + session.addPacket(packet); + if (session.connectionClosed()) { + write(session); + openSessions.remove(sessionId); + closedSessions.add(sessionId); + } + } + + /** + * Writes the sessions that never saw a FIN or RST, such as ones still open + * when the capture stopped, with session_closed false. Call at end of file, + * once per batch until it returns true. + * + * @return true when every open session has been written + */ + public boolean writeOpenSessions() { + Iterator sessions = openSessions.values().iterator(); + while (sessions.hasNext() && !rowWriter.isFull()) { + TcpSession session = sessions.next(); + // Assembles the payload, as a closing handshake would + session.closeSession(); + write(session); + sessions.remove(); + } + return openSessions.isEmpty(); + } + + private void write(TcpSession session) { + rowWriter.start(); + + sessionStartTimeWriter.setTimestamp(session.getSessionStartTime()); + sessionEndTimeWriter.setTimestamp(session.getSessionEndTime()); + sessionDurationWriter.setPeriod(session.getSessionDuration()); + if (session.getConnectionTime() != null) { + connectionTimeWriter.setPeriod(session.getConnectionTime()); + } + + srcMacAddressWriter.setString(session.getSrcMac()); + dstMacAddressWriter.setString(session.getDstMac()); + srcIPWriter.setString(session.getSrcIP()); + dstIPWriter.setString(session.getDstIP()); + srcPortWriter.setInt(session.getSrcPort()); + dstPortWriter.setInt(session.getDstPort()); + tcpSessionWriter.setLong(session.getSessionID()); + packetCountWriter.setInt(session.getPacketCount()); + + originPacketCounterWriter.setInt(session.getPacketCountFromOrigin()); + remotePacketCounterWriter.setInt(session.getPacketCountFromRemote()); + originDataVolumeWriter.setInt(session.getDataFromOriginator().length); + remoteDataVolumeWriter.setInt(session.getDataFromRemote().length); + isCorruptWriter.setBoolean(session.hasCorruptedData()); + sessionClosedWriter.setBoolean(session.connectionClosed()); + + hostDataWriter.setString(session.getDataFromOriginatorAsString()); + remoteDataWriter.setString(session.getDataFromRemoteAsString()); + rowWriter.save(); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java index 7c0fc92724f..0e7f0df9a8e 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java @@ -25,6 +25,7 @@ import java.io.InputStream; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.Arrays; import java.util.Formatter; import static org.apache.drill.exec.store.pcap.PcapFormatUtils.convertInt; @@ -48,14 +49,15 @@ public class Packet implements Comparable { protected byte[] raw; // index into the raw data where the current ethernet packet starts - private int etherOffset; + protected int etherOffset; // index into the raw data where the current IP packet starts. Should be just after etherOffset protected int ipOffset; + // index into the raw data where the TCP/UDP header starts, or -1 if not IPv4/IPv6 + protected int transportOffset = -1; private int packetLength; protected int etherProtocol; protected int protocol; - protected boolean isRoutingV6; protected boolean isCorrupt = false; private static final Logger logger = LoggerFactory.getLogger(Packet.class); @@ -131,12 +133,13 @@ public boolean isUdpPacket() { @SuppressWarnings("WeakerAccess") public boolean isArpPacket() { - return protocol == PacketConstants.ARP_PROTOCOL; + return etherProtocol == PacketConstants.ARP_TYPE; } @SuppressWarnings("WeakerAccess") public boolean isIcmpPacket() { - return protocol == PacketConstants.ICMP_PROTOCOL; + return isIpV4Packet() && protocol == PacketConstants.ICMP_PROTOCOL + || isIpV6Packet() && protocol == PacketConstants.ICMPV6_PROTOCOL; } public long getSessionHash() { @@ -171,6 +174,14 @@ public long getTimestampMicro() { return timestampMicro; } + /** + * For decoders that read the timestamp from somewhere other than a PCAP record header. + */ + protected void setTimestampMicro(long timestampMicro) { + this.timestampMicro = timestampMicro; + this.timestamp = timestampMicro / 1000L; + } + public int getPacketLength() { return packetLength; } @@ -353,12 +364,6 @@ public int getSrc_port() { if (isPPPoV6Packet()) { return getPort(64); } - if (isIpV6Packet()) { - if (isRoutingV6) { - return getPort(136); - } - return getPort(40); - } return getPort(0); } @@ -366,12 +371,6 @@ public int getDst_port() { if (isPPPoV6Packet()) { return getPort(66); } - if (isIpV6Packet()) { - if (isRoutingV6) { - return getPort(138); - } - return getPort(42); - } return getPort(2); } @@ -380,26 +379,44 @@ public boolean isCorrupt(){ } public byte[] getData() { - int payloadDataStart = getIPHeaderLength(); + int payloadStart = ipOffset + getIPHeaderLength(); if (isTcpPacket()) { - payloadDataStart += this.getTCPHeaderLength(raw); + payloadStart += getTCPHeaderLength(raw); } else if (isUdpPacket()) { - payloadDataStart += this.getUDPHeaderLength(); + payloadStart += getUDPHeaderLength(); } else { return null; } - byte[] data = null; - if (packetLength >= payloadDataStart) { - data = new byte[packetLength - payloadDataStart]; - try { - System.arraycopy(raw, ipOffset + payloadDataStart, data, 0, data.length); - } catch (Exception e) { - isCorrupt = true; - String message = "Error while parsing PCAP data: {}"; - logger.debug(message, e.getMessage()); - logger.trace(message, e); } + // The IP length excludes link-layer padding and trailers; the captured + // length bounds packets truncated by the snapshot length + int capturedEnd = Math.min(getFrameEnd(), raw.length); + if (payloadStart > capturedEnd) { + // The headers themselves were cut off + isCorrupt = true; + return null; + } + int payloadEnd = Math.min(getIpPacketEnd(), capturedEnd); + return payloadStart < payloadEnd ? Arrays.copyOfRange(raw, payloadStart, payloadEnd) : null; + } + + /** + * @return index just past the captured bytes of this packet + */ + protected int getFrameEnd() { + return etherOffset + originalLength; + } + + private int getIpPacketEnd() { + if (isIpV4Packet()) { + int totalLength = convertShort(raw, ipOffset + 2); + // Zero when the capture happened before TCP segmentation offload filled it in + return totalLength == 0 ? Integer.MAX_VALUE : ipOffset + totalLength; + } else if (isIpV6Packet()) { + int payloadLength = convertShort(raw, ipOffset + 4); + // Zero for jumbograms + return payloadLength == 0 ? Integer.MAX_VALUE : ipOffset + 40 + payloadLength; } - return data; + return Integer.MAX_VALUE; } private InetAddress getIPAddress(final boolean src) { @@ -417,16 +434,13 @@ private InetAddress getIPAddress(final boolean src) { private byte[] getIpAddressBytes(final boolean src) { int srcPos; byte[] ipBuffer; - int byteShift = 0; if (isIpV4Packet()) { ipBuffer = new byte[4]; srcPos = src ? PacketConstants.IP4_SRC_OFFSET : PacketConstants.IP4_DST_OFFSET; } else if (isIpV6Packet()) { + // Always in the fixed header, whatever extension headers follow ipBuffer = new byte[16]; - if (isRoutingV6) { - byteShift = 96; - } - srcPos = src ? PacketConstants.IP6_SRC_OFFSET + byteShift : PacketConstants.IP6_DST_OFFSET + byteShift; + srcPos = src ? PacketConstants.IP6_SRC_OFFSET : PacketConstants.IP6_DST_OFFSET; } else if (isPPPoV6Packet()) { ipBuffer = new byte[16]; srcPos = src ? PacketConstants.IP6_SRC_OFFSET + PacketConstants.PPPoV6_IP_OFFSET : PacketConstants.IP6_DST_OFFSET + PacketConstants.PPPoV6_IP_OFFSET; @@ -437,7 +451,13 @@ private byte[] getIpAddressBytes(final boolean src) { return ipBuffer; } + /** + * @return length of the IPv4 header, or of the IPv6 header plus extension headers + */ private int getIPHeaderLength() { + if (transportOffset >= 0) { + return transportOffset - ipOffset; + } return (raw[etherOffset + PacketConstants.VER_IHL_OFFSET] & 0xF) * 4; } @@ -445,8 +465,9 @@ private int getTCPHeaderLength(final byte[] packet) { final int inTCPHeaderDataOffset = 12; // tcp packet header can have options - int dataOffset = etherOffset + getIPHeaderLength() + inTCPHeaderDataOffset; - return 20 + ((packet[dataOffset] >> 4) & 0xF) * 4; + int dataOffset = ipOffset + getIPHeaderLength() + inTCPHeaderDataOffset; + // Data offset is the whole header length in 32-bit words + return ((packet[dataOffset] >> 4) & 0xF) * 4; } private int getUDPHeaderLength() { @@ -490,6 +511,7 @@ private void decodeEtherPacket() { protected int processIpV4Packet() { validateIpV4Packet(); + transportOffset = ipOffset + ipV4HeaderLength(); return getByte(raw, ipOffset + 9); } @@ -497,34 +519,34 @@ protected int processIpV6Packet() { Preconditions.checkState(ipVersion() == 6, "Should have seen IP version 6, got %d", ipVersion()); int headerLength = 40; int nextHeader = raw[ipOffset + 6] & 0xff; - while (nextHeader != PacketConstants.TCP_PROTOCOL && nextHeader != PacketConstants.UDP_PROTOCOL && nextHeader != PacketConstants.NO_NEXT_HEADER) { + while (true) { + int extension = ipOffset + headerLength; switch (nextHeader) { case PacketConstants.FRAGMENT_V6: - nextHeader = getByte(raw, ipOffset + headerLength); + nextHeader = getByte(raw, extension); headerLength += 8; break; case PacketConstants.ROUTING_V6: - isRoutingV6 = true; - nextHeader = getByte(raw, ipOffset + headerLength + 15); - headerLength += (getByte(raw, ipOffset + headerLength) + 1) * 8; - break; case PacketConstants.HOP_BY_HOP_EXTENSION_V6: case PacketConstants.DESTINATION_OPTIONS_V6: - case PacketConstants.AUTHENTICATION_V6: - case PacketConstants.ENCAPSULATING_SECURITY_V6: case PacketConstants.MOBILITY_EXTENSION_V6: case PacketConstants.HOST_IDENTITY_PROTOCOL: case PacketConstants.SHIM6_PROTOCOL: - nextHeader = getByte(raw, ipOffset + headerLength); - headerLength += (getByte(raw, ipOffset + headerLength) + 1) * 8; + // Next header, then length in 8-octet units not counting the first 8 + nextHeader = getByte(raw, extension); + headerLength += (getByte(raw, extension + 1) + 1) * 8; + break; + case PacketConstants.AUTHENTICATION_V6: + // Length in 4-octet units not counting the first 8 + nextHeader = getByte(raw, extension); + headerLength += (getByte(raw, extension + 1) + 2) * 4; break; default: - //noinspection ConstantConditions - logger.warn("Unknown V6 extension or protocol: {}", nextHeader); - return getByte(raw, ipOffset + headerLength); + // Upper-layer protocol, or ESP whose contents are encrypted + transportOffset = extension; + return nextHeader; } } - return nextHeader; } private void validateIpV4Packet() { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketConstants.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketConstants.java index 9525d3ebd3b..cbe37927ac4 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketConstants.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketConstants.java @@ -29,8 +29,8 @@ public final class PacketConstants { public static final int PACKET_PROTOCOL_OFFSET = 12; - public static final byte ARP_PROTOCOL = 0; public static final byte ICMP_PROTOCOL = 1; + public static final byte ICMPV6_PROTOCOL = 58; public static final byte TCP_PROTOCOL = 6; public static final byte UDP_PROTOCOL = 17; @@ -52,6 +52,7 @@ public final class PacketConstants { public static final int ETHER_TYPE_OFFSET = 12; public static final int IPv4_TYPE = 0x800; + public static final int ARP_TYPE = 0x806; public static final int IPv6_TYPE = 0x86dd; public static final int PPPoV6_TYPE = 0x8864; diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java index f67b85f5bb0..2fa945b39d0 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java @@ -168,11 +168,11 @@ public void addPacket(Packet p) { } // Add the start and ending time stamp. The packets are not necessarily received in order, so we have to check the timestamps this way - if (p.getTimestampMicro() < startTime) { + if (p.getTimestamp() < startTime) { startTime = p.getTimestamp(); } - if (p.getTimestampMicro() > endTime) { + if (p.getTimestamp() > endTime) { endTime = p.getTimestamp(); } @@ -247,8 +247,12 @@ public Period getSessionDuration() { return new Period(endTime - startTime); } + /** + * @return time from SYN to the final handshake ACK, or null if the capture + * did not see both, such as when it started mid-connection + */ public Period getConnectionTime() { - return new Period(connectTime); + return synTime == 0 || ackTime == 0 ? null : new Period(connectTime); } public Instant getSessionEndTime() { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java index fe2490756ca..c5798994f9e 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java @@ -44,7 +44,6 @@ public abstract class BasePcapFormatPlugin extends EasyFormatPlugin { static final Logger logger = LoggerFactory.getLogger(ManagedScanFramework.class); - private static PacketDecoder.FileFormat fileFormat = PacketDecoder.FileFormat.UNKNOWN; public BasePcapFormatPlugin(String name, DrillbitContext context, @@ -86,18 +85,14 @@ public PcapReaderFactory(PcapFormatConfig config, EasySubScan scan) { */ @Override public ManagedReader newReader(FileSchemaNegotiator negotiator) { - if (negotiator.file().fileSystem() != null) { // todo: can be simplified with java9 - // ifPresentOrElse - Path path = scan.getWorkUnits().stream() - .findFirst() - .orElseThrow(() -> UserException. - dataReadError() - .addContext("There are no files for scanning") - .build(logger)) - .getPath(); - fileFormat = getFileFormat(negotiator.file().fileSystem(), path); + // Local, and detected from this reader's own file: readers for + // different files can be created concurrently + PacketDecoder.FileFormat fileFormat = PacketDecoder.FileFormat.UNKNOWN; + if (negotiator.file().fileSystem() != null) { + fileFormat = getFileFormat(negotiator.file().fileSystem(), negotiator.file().split().getPath()); + String formatName = fileFormat.name().toLowerCase(); if (config.getExtensions().stream() - .noneMatch(f -> f.equals(fileFormat.name().toLowerCase()))) { + .noneMatch(f -> f.equals(formatName))) { logger.error("File format {} is not within plugin extensions: {}. Trying to use default PCAP format plugin to " + "read the file", fileFormat, config.getExtensions()); } @@ -105,12 +100,12 @@ public ManagedReader newReader(FileSchemaNegotiator negotiator) { logger.error("It is not possible to detect file format, because the File Framework is not initialized. " + "Trying to use default PCAP format plugin to read the file"); } - return createReader(scan, config, negotiator); + return createReader(fileFormat, scan, config, negotiator); } } - private static ManagedReader createReader(EasySubScan scan, PcapFormatConfig config, - FileSchemaNegotiator negotiator) { + private static ManagedReader createReader(PacketDecoder.FileFormat fileFormat, EasySubScan scan, + PcapFormatConfig config, FileSchemaNegotiator negotiator) { switch(fileFormat) { case PCAPNG: return new PcapngBatchReader(config, scan, negotiator); case PCAP: diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/schema/Schema.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/schema/Schema.java index 7655d26208f..0e85720c159 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/schema/Schema.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/schema/Schema.java @@ -67,6 +67,7 @@ private void setupStructure() { columns.add(new ColumnDto("connection_time", PcapTypes.DURATION)); columns.add(new ColumnDto("tcp_session", PcapTypes.LONG)); columns.add(new ColumnDto("is_corrupt", PcapTypes.BOOLEAN)); + columns.add(new ColumnDto("session_closed", PcapTypes.BOOLEAN)); columns.add(new ColumnDto("data_from_originator", PcapTypes.STRING)); columns.add(new ColumnDto("data_from_remote", PcapTypes.STRING)); } else { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java index e0cc5214d8d..e9ca47f18b2 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java @@ -17,6 +17,8 @@ */ package org.apache.drill.exec.store.pcapng; +import java.time.Instant; + import org.apache.drill.exec.store.pcap.decoder.Packet; import org.apache.drill.exec.store.pcap.decoder.PacketConstants; @@ -25,15 +27,47 @@ public class PacketDecoder extends Packet { - @SuppressWarnings("WeakerAccess") - public boolean readPcapng(final byte[] raw) { - this.raw = raw; - return decodeEtherPacket(); - } + // Link types from https://www.tcpdump.org/linktypes.html + static final int LINKTYPE_NULL = 0; + static final int LINKTYPE_ETHERNET = 1; + static final int LINKTYPE_PPP = 9; + static final int LINKTYPE_RAW = 101; + static final int LINKTYPE_IEEE802_11 = 105; + static final int LINKTYPE_LOOP = 108; + static final int LINKTYPE_LINUX_SLL = 113; + static final int LINKTYPE_IEEE802_11_RADIOTAP = 127; + static final int LINKTYPE_IPV4 = 228; + static final int LINKTYPE_IPV6 = 229; + static final int LINKTYPE_LINUX_SLL2 = 276; - private boolean decodeEtherPacket() { - etherProtocol = getShort(raw, PacketConstants.PACKET_PROTOCOL_OFFSET); - ipOffset = PacketConstants.IP_OFFSET; + private static final int VLAN_TYPE = 0x8100; + private static final int QINQ_TYPE = 0x88a8; + + // Offsets of the source and destination MAC addresses, or -1 if the link layer has none + private int srcMacOffset; + private int dstMacOffset; + + /** + * Decodes a packet captured on an interface with the given link type. + * + * @return false if the link type or the network protocol is not supported + */ + public boolean readPcapng(final byte[] raw, final int linkType) { + this.raw = raw; + srcMacOffset = -1; + dstMacOffset = -1; + int networkOffset = findNetworkLayer(linkType); + if (networkOffset >= 0 && isArpPacket()) { + // Typed, but has no IP fields + return true; + } + if (networkOffset < 0 || networkOffset + 20 > raw.length) { + return false; + } + // Packet addresses IP fields relative to a 14-byte Ethernet header, so + // place a virtual one just before the network layer + etherOffset = networkOffset - PacketConstants.IP_OFFSET; + ipOffset = networkOffset; if (isIpV4Packet()) { protocol = processIpV4Packet(); return true; @@ -44,12 +78,178 @@ private boolean decodeEtherPacket() { } return true; } else if (isPPPoV6Packet()) { - protocol = getByte(raw, 48); + protocol = getByte(raw, etherOffset + 48); return true; } return false; } + /** + * Sets {@link #etherProtocol} from the link-layer header. + * + * @return offset of the network layer, or -1 if the link type is not supported + */ + private int findNetworkLayer(int linkType) { + switch (linkType) { + case LINKTYPE_ETHERNET: + if (raw.length < PacketConstants.IP_OFFSET) { + return -1; + } + dstMacOffset = PacketConstants.ETHER_DST_OFFSET; + srcMacOffset = PacketConstants.ETHER_SRC_OFFSET; + // Skip 802.1Q and 802.1ad VLAN tags + int typeOffset = PacketConstants.PACKET_PROTOCOL_OFFSET; + etherProtocol = getShort(raw, typeOffset); + while ((etherProtocol == VLAN_TYPE || etherProtocol == QINQ_TYPE) && raw.length >= typeOffset + 6) { + typeOffset += 4; + etherProtocol = getShort(raw, typeOffset); + } + return typeOffset + 2; + case LINKTYPE_IEEE802_11: + return find80211NetworkLayer(0); + case LINKTYPE_IEEE802_11_RADIOTAP: + // Radiotap header length is a little-endian short at offset 2 + if (raw.length < 4) { + return -1; + } + return find80211NetworkLayer((raw[2] & 0xff) | (raw[3] & 0xff) << 8); + case LINKTYPE_RAW: + case LINKTYPE_IPV4: + case LINKTYPE_IPV6: + return ipVersionProtocol(0); + case LINKTYPE_NULL: + case LINKTYPE_LOOP: + // 4-byte address family, in host byte order for NULL. The protocol is + // read from the IP header instead of mapping per-OS family values. + return ipVersionProtocol(4); + case LINKTYPE_LINUX_SLL: + if (raw.length < 16) { + return -1; + } + etherProtocol = getShort(raw, 14); + return 16; + case LINKTYPE_LINUX_SLL2: + if (raw.length < 20) { + return -1; + } + etherProtocol = getShort(raw, 0); + return 20; + case LINKTYPE_PPP: + // Optional HDLC address/control bytes, then a 2-byte PPP protocol + int offset = raw.length > 1 && (raw[0] & 0xff) == 0xff && raw[1] == 0x03 ? 2 : 0; + if (raw.length < offset + 2) { + return -1; + } + int pppProtocol = getShort(raw, offset); + if (pppProtocol == 0x0021) { + etherProtocol = PacketConstants.IPv4_TYPE; + } else if (pppProtocol == 0x0057) { + etherProtocol = PacketConstants.IPv6_TYPE; + } else { + return -1; + } + return offset + 2; + default: + return -1; + } + } + + /** + * Finds the network layer of an unencrypted 802.11 data frame carrying + * LLC/SNAP, and the source and destination addresses for its DS bits. + */ + private int find80211NetworkLayer(int offset) { + if (raw.length < offset + 24) { + return -1; + } + int frameControl = raw[offset] & 0xff; + int flags = raw[offset + 1] & 0xff; + int type = (frameControl >> 2) & 0x3; + int subtype = (frameControl >> 4) & 0xf; + boolean toDs = (flags & 0x01) != 0; + boolean fromDs = (flags & 0x02) != 0; + boolean isProtected = (flags & 0x40) != 0; + // Only data frames carry packets; subtypes 4-7 and 12-15 carry no payload + if (type != 2 || (subtype & 0x4) != 0 || isProtected) { + return -1; + } + // Address fields 1-3 start at 4, 10 and 16; address 4 at 24 + if (!toDs && !fromDs) { + dstMacOffset = offset + 4; + srcMacOffset = offset + 10; + } else if (toDs && !fromDs) { + srcMacOffset = offset + 10; + dstMacOffset = offset + 16; + } else if (!toDs) { + dstMacOffset = offset + 4; + srcMacOffset = offset + 16; + } else { + dstMacOffset = offset + 16; + srcMacOffset = offset + 24; + } + int headerLength = toDs && fromDs ? 30 : 24; + if ((subtype & 0x8) != 0) { + // QoS control, plus HT control when the order bit is set + headerLength += (flags & 0x80) != 0 ? 6 : 2; + } + int llc = offset + headerLength; + // LLC/SNAP: AA AA 03 00 00 00, then the EtherType + if (raw.length < llc + 8 || (raw[llc] & 0xff) != 0xaa || (raw[llc + 1] & 0xff) != 0xaa + || raw[llc + 2] != 0x03) { + return -1; + } + etherProtocol = getShort(raw, llc + 6); + return llc + 8; + } + + private int ipVersionProtocol(int offset) { + if (raw.length <= offset) { + return -1; + } + int version = (raw[offset] & 0xff) >>> 4; + if (version == 4) { + etherProtocol = PacketConstants.IPv4_TYPE; + } else if (version == 6) { + etherProtocol = PacketConstants.IPv6_TYPE; + } else { + return -1; + } + return offset; + } + + @Override + public String getEthernetSource() { + return formatMac(srcMacOffset); + } + + @Override + public String getEthernetDestination() { + return formatMac(dstMacOffset); + } + + private String formatMac(int offset) { + if (offset < 0 || offset + 6 > raw.length) { + return null; + } + StringBuilder mac = new StringBuilder(17); + for (int i = 0; i < 6; i++) { + if (i > 0) { + mac.append(':'); + } + mac.append(String.format("%02X", raw[offset + i])); + } + return mac.toString(); + } + + void setTimestamp(Instant timestamp) { + setTimestampMicro(timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000); + } + + @Override + protected int getFrameEnd() { + return raw.length; + } + @Override protected int processIpV6Packet() { try { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java index e185e308465..e0fcf68b79e 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java @@ -28,13 +28,6 @@ import org.apache.drill.exec.store.pcap.PcapFormatUtils; import org.apache.drill.exec.vector.accessor.ScalarWriter; -import fr.bmartel.pcapdecoder.structure.options.inter.IOptionsStatisticsHeader; -import fr.bmartel.pcapdecoder.structure.types.IPcapngType; -import fr.bmartel.pcapdecoder.structure.types.inter.IDescriptionBlock; -import fr.bmartel.pcapdecoder.structure.types.inter.IEnhancedPacketBLock; -import fr.bmartel.pcapdecoder.structure.types.inter.INameResolutionBlock; -import fr.bmartel.pcapdecoder.structure.types.inter.ISectionHeaderBlock; -import fr.bmartel.pcapdecoder.structure.types.inter.IStatisticsBlock; public abstract class PcapColumn { @@ -70,43 +63,65 @@ public abstract class PcapColumn { columns.put("tcp_flags_fin", new PcapTcpFlagsFin()); columns.put("tcp_parsed_flags", new PcapTcpParsedFlags()); columns.put("packet_data", new PcapPacketData()); + // Interface and Enhanced Packet Block metadata + columns.put("captured_length", new PcapCapturedLength()); + columns.put("interface_id", new PcapInterfaceId()); + columns.put("interface_name", new PcapInterfaceName()); + columns.put("link_type", new PcapLinkType()); + columns.put("comment", new PcapComment()); + columns.put("direction", new PcapDirection()); + columns.put("reception_type", new PcapReceptionType()); + columns.put("fcs_length", new PcapFcsLength()); + columns.put("drop_count", new PcapDropCount()); + columns.put("packet_hash", new PcapPacketHash()); // Extensions summary_columns.put("path", new PcapStatPath()); + summary_columns.put("comment", new PcapComment()); // Section Header Block - summary_columns.put("shb_hardware", new PcapHardware()); - summary_columns.put("shb_os", new PcapOS()); - summary_columns.put("shb_userappl", new PcapUserAppl()); + addStat("shb_hardware", MinorType.VARCHAR); + addStat("shb_os", MinorType.VARCHAR); + addStat("shb_userappl", MinorType.VARCHAR); // Interface Description Block - summary_columns.put("if_name", new PcapIfName()); - summary_columns.put("if_description", new PcapIfDescription()); - summary_columns.put("if_ipv4addr", new PcapIfIPv4addr()); - summary_columns.put("if_ipv6addr", new PcapIfIPv6addr()); - summary_columns.put("if_macaddr", new PcapIfMACaddr()); - summary_columns.put("if_euiaddr", new PcapIfEUIaddr()); - summary_columns.put("if_speed", new PcapIfSpeed()); - summary_columns.put("if_tsresol", new PcapIfTsresol()); - summary_columns.put("if_tzone", new PcapIfTzone()); - summary_columns.put("if_os", new PcapIfOS()); - summary_columns.put("if_fcslen", new PcapIfFcslen()); - summary_columns.put("if_tsoffset", new PcapIfTsOffset()); + addStat("if_name", MinorType.VARCHAR); + addStat("if_description", MinorType.VARCHAR); + addStat("if_ipv4addr", MinorType.VARCHAR); + addStat("if_ipv6addr", MinorType.VARCHAR); + addStat("if_macaddr", MinorType.VARCHAR); + addStat("if_euiaddr", MinorType.VARCHAR); + addStat("if_speed", MinorType.BIGINT); + addStat("if_tsresol", MinorType.INT); + addStat("if_tzone", MinorType.INT); + addStat("if_os", MinorType.VARCHAR); + addStat("if_fcslen", MinorType.INT); + addStat("if_tsoffset", MinorType.BIGINT); // Name Resolution Block - summary_columns.put("ns_dnsname", new PcapDnsName()); - summary_columns.put("ns_dnsip4addr", new PcapDnsIP4addr()); - summary_columns.put("ns_dnsip6addr", new PcapDnsIP6addr()); + addStat("ns_dnsname", MinorType.VARCHAR); + addStat("ns_dnsip4addr", MinorType.VARCHAR); + addStat("ns_dnsip6addr", MinorType.VARCHAR); // Interface Statistics Block - summary_columns.put("isb_starttime", new PcapIsbStarttime()); - summary_columns.put("isb_endtime", new PcapIsbEndtime()); - summary_columns.put("isb_ifrecv", new PcapIsbIfrecv()); - summary_columns.put("isb_ifdrop", new PcapIsbIfdrop()); - summary_columns.put("isb_filteraccept", new PcapIsbFilterAccept()); - summary_columns.put("isb_osdrop", new PcapIsbOSdrop()); - summary_columns.put("isb_usrdeliv", new PcapIsbUsrdeliv()); + addStat("isb_starttime", MinorType.TIMESTAMP); + addStat("isb_endtime", MinorType.TIMESTAMP); + addStat("isb_ifrecv", MinorType.BIGINT); + addStat("isb_ifdrop", MinorType.BIGINT); + addStat("isb_filteraccept", MinorType.BIGINT); + addStat("isb_osdrop", MinorType.BIGINT); + addStat("isb_usrdeliv", MinorType.BIGINT); + } + + private static void addStat(String name, MinorType type) { + summary_columns.put(name, new PcapStat(name, type)); } abstract MajorType getType(); - abstract void process(IPcapngType block, ScalarWriter writer); + abstract void process(PcapngBlock block, ScalarWriter writer); + + private static void setString(ScalarWriter writer, String value) { + if (value != null) { + writer.setString(value); + } + } public static Map getColumns() { return Collections.unmodifiableMap(columns); @@ -124,7 +139,7 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { } + void process(PcapngBlock block, ScalarWriter writer) { } } static class PcapStatPath extends PcapColumn { @@ -135,7 +150,7 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { } + void process(PcapngBlock block, ScalarWriter writer) { } } static class PcapTimestamp extends PcapColumn { @@ -146,8 +161,8 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - writer.setTimestamp(Instant.ofEpochMilli(((IEnhancedPacketBLock) block).getTimeStamp() / 1000)); + void process(PcapngBlock block, ScalarWriter writer) { + writer.setTimestamp(block.timestamp); } } @@ -159,8 +174,8 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - writer.setInt(((IEnhancedPacketBLock) block).getPacketLength()); + void process(PcapngBlock block, ScalarWriter writer) { + writer.setInt(block.originalLength); } } @@ -172,10 +187,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setString(packet.getPacketType()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setString(block.packet.getPacketType()); } } } @@ -188,10 +202,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setString(packet.getSrc_ip().getHostAddress()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + setString(writer, block.packet.getSourceIpAddressString()); } } } @@ -204,10 +217,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setString(packet.getDst_ip().getHostAddress()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + setString(writer, block.packet.getDestinationIpAddressString()); } } } @@ -220,10 +232,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setInt(packet.getSrc_port()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null && (block.packet.isTcpPacket() || block.packet.isUdpPacket())) { + writer.setInt(block.packet.getSrc_port()); } } } @@ -236,10 +247,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setInt(packet.getDst_port()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null && (block.packet.isTcpPacket() || block.packet.isUdpPacket())) { + writer.setInt(block.packet.getDst_port()); } } } @@ -252,10 +262,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setString(packet.getEthernetSource()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + setString(writer, block.packet.getEthernetSource()); } } } @@ -268,10 +277,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setString(packet.getEthernetDestination()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + setString(writer, block.packet.getEthernetDestination()); } } } @@ -284,10 +292,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setLong(packet.getSessionHash()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setLong(block.packet.getSessionHash()); } } } @@ -300,10 +307,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setInt(packet.getAckNumber()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setInt(block.packet.getAckNumber()); } } } @@ -316,10 +322,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setInt(packet.getFlags()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setInt(block.packet.getFlags()); } } } @@ -332,10 +337,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x100) != 0); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x100) != 0); } } } @@ -348,10 +352,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x80) != 0); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x80) != 0); } } } @@ -364,10 +367,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x40) != 0); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x40) != 0); } } } @@ -380,10 +382,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x42) == 0x42); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x42) == 0x42); } } } @@ -396,10 +397,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x42) == 0x40); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x42) == 0x40); } } } @@ -412,10 +412,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x20) != 0); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x20) != 0); } } } @@ -428,10 +427,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x10) != 0); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x10) != 0); } } } @@ -444,10 +442,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x8) != 0); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x8) != 0); } } } @@ -460,10 +457,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x4) != 0); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x4) != 0); } } } @@ -476,10 +472,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x2) != 0); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x2) != 0); } } } @@ -492,10 +487,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setBoolean((packet.getFlags() & 0x1) != 0); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setBoolean((block.packet.getFlags() & 0x1) != 0); } } } @@ -508,10 +502,9 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setString(packet.getParsedFlags()); + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setString(block.packet.getParsedFlags()); } } } @@ -524,98 +517,42 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(((IEnhancedPacketBLock) block).getPacketData())) { - writer.setString(PcapFormatUtils.parseBytesToASCII(((IEnhancedPacketBLock) block).getPacketData())); - } - } - } - - /** - * shb_hardware: description of the hardware - */ - static class PcapHardware extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.VARCHAR); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof ISectionHeaderBlock)) { - return; + void process(PcapngBlock block, ScalarWriter writer) { + if (block.packet != null) { + writer.setString(PcapFormatUtils.parseBytesToASCII(block.data)); } - writer.setString(((ISectionHeaderBlock) block).getOptions().getHardware()); } } - // Section Header Block + // Interface and Enhanced Packet Block metadata - /** - * shb_os: name of the OS - */ - static class PcapOS extends PcapColumn { + static class PcapCapturedLength extends PcapColumn { @Override MajorType getType() { - return Types.optional(MinorType.VARCHAR); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof ISectionHeaderBlock)) { - return; - } - writer.setString(((ISectionHeaderBlock) block).getOptions().getOS()); - } - } - - /** - * shb_userappl: name of the user application - */ - static class PcapUserAppl extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.VARCHAR); + return Types.required(MinorType.INT); } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof ISectionHeaderBlock)) { - return; - } - writer.setString(((ISectionHeaderBlock) block).getOptions().getUserAppl()); + void process(PcapngBlock block, ScalarWriter writer) { + writer.setInt(block.capturedLength); } } - // Interface Description Block - - /** - * if_name: name of the device used to capture - */ - static class PcapIfName extends PcapColumn { + static class PcapInterfaceId extends PcapColumn { @Override MajorType getType() { - return Types.optional(MinorType.VARCHAR); + return Types.required(MinorType.INT); } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; - } - writer.setString(((IDescriptionBlock) block).getOptions().getInterfaceName()); + void process(PcapngBlock block, ScalarWriter writer) { + writer.setInt(block.interfaceId); } } - /** - * if_description: Description of the device used to capture the data - */ - static class PcapIfDescription extends PcapColumn { + static class PcapInterfaceName extends PcapColumn { @Override MajorType getType() { @@ -623,56 +560,31 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; - } - writer.setString(((IDescriptionBlock) block).getOptions().getInterfaceDescription()); + void process(PcapngBlock block, ScalarWriter writer) { + setString(writer, block.interfaceName); } } /** - * if_IPv4addr: IPV4 address + * link_type: LINKTYPE_ value of the packet's interface, see https://www.tcpdump.org/linktypes.html */ - static class PcapIfIPv4addr extends PcapColumn { + static class PcapLinkType extends PcapColumn { @Override MajorType getType() { - return Types.optional(MinorType.VARCHAR); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; - } - writer.setString(((IDescriptionBlock) block).getOptions().getInterfaceIpv4NetworkAddr()); - } - } - - /** - * if_IPv6addr: IPV6 address - */ - static class PcapIfIPv6addr extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.VARCHAR); + return Types.required(MinorType.INT); } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; - } - writer.setString(((IDescriptionBlock) block).getOptions().getIpv6NetworkAddr()); + void process(PcapngBlock block, ScalarWriter writer) { + writer.setInt(block.linkType); } } /** - * if_MACaddr: MAC address + * comment: opt_comment of the block; multiple comments are newline separated */ - static class PcapIfMACaddr extends PcapColumn { + static class PcapComment extends PcapColumn { @Override MajorType getType() { @@ -680,18 +592,16 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; - } - writer.setString(((IDescriptionBlock) block).getOptions().getInterfaceMacAddr()); + void process(PcapngBlock block, ScalarWriter writer) { + setString(writer, block.comment); } } /** - * if_EUIaddr: EUI address + * direction: inbound or outbound, from epb_flags bits 0-1 */ - static class PcapIfEUIaddr extends PcapColumn { + static class PcapDirection extends PcapColumn { + private static final String[] VALUES = {null, "inbound", "outbound", null}; @Override MajorType getType() { @@ -699,75 +609,18 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; - } - writer.setString(((IDescriptionBlock) block).getOptions().getInterfaceEuiAddr()); - } - } - - /** - * if_speed: interface speed in bps - */ - static class PcapIfSpeed extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.INT); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; - } - writer.setInt(((IDescriptionBlock) block).getOptions().getInterfaceSpeed()); - } - } - - /** - * if_tsresol: Resolution of timestamp (6 means microsecond resolution for instance) - */ - static class PcapIfTsresol extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.INT); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; + void process(PcapngBlock block, ScalarWriter writer) { + if (block.flags != null) { + setString(writer, VALUES[block.flags & 0x3]); } - writer.setInt(((IDescriptionBlock) block).getOptions().getTimeStampResolution()); } } /** - * if_tzone: indicate Time zone => offset from UTC time + * reception_type: unicast, multicast, broadcast or promiscuous, from epb_flags bits 2-4 */ - static class PcapIfTzone extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.INT); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; - } - writer.setInt(((IDescriptionBlock) block).getOptions().getTimeBias()); - } - } - - /** - * if_os: Name of the operating system - */ - static class PcapIfOS extends PcapColumn { + static class PcapReceptionType extends PcapColumn { + private static final String[] VALUES = {null, "unicast", "multicast", "broadcast", "promiscuous", null, null, null}; @Override MajorType getType() { @@ -775,18 +628,17 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; + void process(PcapngBlock block, ScalarWriter writer) { + if (block.flags != null) { + setString(writer, VALUES[(block.flags >> 2) & 0x7]); } - writer.setString(((IDescriptionBlock) block).getOptions().getInterfaceOperatingSystem()); } } /** - * if_fcslen: Length of the Frame Check Sequence (in bits) + * fcs_length: frame check sequence length in octets, from epb_flags bits 5-8 */ - static class PcapIfFcslen extends PcapColumn { + static class PcapFcsLength extends PcapColumn { @Override MajorType getType() { @@ -794,157 +646,17 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; + void process(PcapngBlock block, ScalarWriter writer) { + if (block.flags != null && ((block.flags >> 5) & 0xF) != 0) { + writer.setInt((block.flags >> 5) & 0xF); } - writer.setInt(((IDescriptionBlock) block).getOptions().getInterfaceFrameCheckSequenceLength()); } } /** - * if_tsoffset: Timestamp offset for each packet / if not present timestamp are absolute + * drop_count: packets lost between this packet and the preceding one (epb_dropcount) */ - static class PcapIfTsOffset extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.INT); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IDescriptionBlock)) { - return; - } - writer.setInt(((IDescriptionBlock) block).getOptions().getTimeStampOffset()); - } - } - - // Name Resolution Block - - /** - * ns_dnsname: Retrieve DNS server name - */ - static class PcapDnsName extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.VARCHAR); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof INameResolutionBlock)) { - return; - } - writer.setString(((INameResolutionBlock) block).getOptions().getDnsName()); - } - } - - /** - * ns_dnsIP4addr: Retrieve DNS IPV4 server address - */ - static class PcapDnsIP4addr extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.VARCHAR); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof INameResolutionBlock)) { - return; - } - writer.setString(((INameResolutionBlock) block).getOptions().getDnsIpv4Addr()); - } - } - - /** - * ns_dnsIP6addr: Retrieve DNS IPV6 server address - */ - static class PcapDnsIP6addr extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.VARCHAR); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof INameResolutionBlock)) { - return; - } - writer.setString(((INameResolutionBlock) block).getOptions().getDnsIpv6Addr()); - } - } - - // Interface Statistics Block - - /** - * isb_starttime: capture start time (timestamp resolution is defined in Interface description header check exemple) - */ - static class PcapIsbStarttime extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.TIMESTAMP); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IStatisticsBlock)) { - return; - } - IOptionsStatisticsHeader statisticsHeader = ((IStatisticsBlock) block).getOptions(); - writer.setTimestamp(Instant.ofEpochMilli(statisticsHeader.getCaptureStartTime() / 1000)); - } - } - - /** - * isb_endtime: capture end time (timestamp resolution is defined in Interface description header check example) - */ - static class PcapIsbEndtime extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.TIMESTAMP); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IStatisticsBlock)) { - return; - } - IOptionsStatisticsHeader statisticsHeader = ((IStatisticsBlock) block).getOptions(); - writer.setTimestamp(Instant.ofEpochMilli(statisticsHeader.getCaptureEndTime() / 1000)); - } - } - - /** - * isb_ifrecv: packet received count - */ - static class PcapIsbIfrecv extends PcapColumn { - - @Override - MajorType getType() { - return Types.optional(MinorType.BIGINT); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IStatisticsBlock)) { - return; - } - writer.setLong(((IStatisticsBlock) block).getOptions().getPacketReceivedCount()); - } - } - - /** - * isb_ifdrop: packet drop count - */ - static class PcapIsbIfdrop extends PcapColumn { + static class PcapDropCount extends PcapColumn { @Override MajorType getType() { @@ -952,68 +664,70 @@ MajorType getType() { } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IStatisticsBlock)) { - return; + void process(PcapngBlock block, ScalarWriter writer) { + if (block.dropCount != null) { + writer.setLong(block.dropCount); } - writer.setLong(((IStatisticsBlock) block).getOptions().getPacketDropCount()); } } /** - * isb_filteraccept: packet accepted by filter count + * packet_hash: epb_hash as algorithm:hex, such as md5:9e107d9d372bb6826bd81d3542a419d6 */ - static class PcapIsbFilterAccept extends PcapColumn { + static class PcapPacketHash extends PcapColumn { @Override MajorType getType() { - return Types.optional(MinorType.BIGINT); + return Types.optional(MinorType.VARCHAR); } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IStatisticsBlock)) { - return; - } - writer.setLong(((IStatisticsBlock) block).getOptions().getPacketAcceptedByFilterCount()); + void process(PcapngBlock block, ScalarWriter writer) { + setString(writer, block.hash); } } /** - * isb_osdrop: packet dropped by Operating system count + * A stat column, read from the options the reader decoded from a + * Section Header, Interface Description, Name Resolution or Interface + * Statistics Block. Null when the block does not have the option. */ - static class PcapIsbOSdrop extends PcapColumn { + static class PcapStat extends PcapColumn { + private final String name; + private final MinorType type; - @Override - MajorType getType() { - return Types.optional(MinorType.BIGINT); - } - - @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IStatisticsBlock)) { - return; - } - writer.setLong(((IStatisticsBlock) block).getOptions().getPacketDroppedByOS()); + PcapStat(String name, MinorType type) { + this.name = name; + this.type = type; } - } - - /** - * isb_usrdeliv: packet deliver to use count - */ - static class PcapIsbUsrdeliv extends PcapColumn { @Override MajorType getType() { - return Types.optional(MinorType.BIGINT); + return Types.optional(type); } @Override - void process(IPcapngType block, ScalarWriter writer) { - if (!(block instanceof IStatisticsBlock)) { + void process(PcapngBlock block, ScalarWriter writer) { + Object value = block.stats == null ? null : block.stats.get(name); + if (value == null) { return; } - writer.setLong(((IStatisticsBlock) block).getOptions().getPacketDeliveredToUser()); - } - } -} \ No newline at end of file + switch (type) { + case VARCHAR: + writer.setString((String) value); + break; + case INT: + writer.setInt((Integer) value); + break; + case BIGINT: + writer.setLong((Long) value); + break; + case TIMESTAMP: + writer.setTimestamp((Instant) value); + break; + default: + throw new IllegalStateException("Unsupported stat column type " + type); + } + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java index 2e8d0aa6efa..1fc0dcf6e0f 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java @@ -17,11 +17,19 @@ */ package org.apache.drill.exec.store.pcapng; +import java.io.BufferedInputStream; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -41,19 +49,35 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.drill.exec.store.dfs.easy.EasySubScan; +import org.apache.drill.exec.store.pcap.TcpSessionizer; import org.apache.drill.exec.store.pcap.plugin.PcapFormatConfig; +import org.apache.drill.exec.store.pcap.schema.Schema; import org.apache.drill.exec.util.Utilities; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import fr.bmartel.pcapdecoder.PcapDecoder; -import fr.bmartel.pcapdecoder.structure.types.IPcapngType; -import fr.bmartel.pcapdecoder.structure.types.inter.IEnhancedPacketBLock; public class PcapngBatchReader implements ManagedReader { private static final Logger logger = LoggerFactory.getLogger(PcapngBatchReader.class); + private static final int SECTION_HEADER_TYPE = 0x0A0D0D0A; + private static final int INTERFACE_DESCRIPTION_TYPE = 1; + private static final int NAME_RESOLUTION_TYPE = 4; + private static final int INTERFACE_STATISTICS_TYPE = 5; + private static final int ENHANCED_PACKET_TYPE = 6; + private static final int BYTE_ORDER_MAGIC = 0x1A2B3C4D; + // Option codes + private static final int OPT_ENDOFOPT = 0; + private static final int OPT_COMMENT = 1; + private static final int IF_NAME = 2; + private static final int IF_TSRESOL = 9; + private static final int IF_TSOFFSET = 14; + private static final int EPB_FLAGS = 2; + private static final int EPB_HASH = 3; + private static final int EPB_DROPCOUNT = 4; + private static final String[] HASH_ALGORITHMS = {"2s-complement", "xor", "crc32", "md5", "sha1", "toeplitz"}; + private final PcapFormatConfig config; private final EasySubScan scan; private final FileDescrip file; @@ -61,9 +85,13 @@ public class PcapngBatchReader implements ManagedReader { private CustomErrorContext errorContext; private List columns; private List projectedColumns; - private Iterator pcapIterator; - private IPcapngType block; + private final byte[] header = new byte[12]; + private ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; + // Interfaces of the current section, indexed by interface ID + private final List interfaces = new ArrayList<>(); private RowSetLoader loader; + // Set when TCP packets are grouped into sessions instead of returned as rows + private TcpSessionizer sessionizer; private InputStream in; private Path path; @@ -78,11 +106,9 @@ public PcapngBatchReader(final PcapFormatConfig config, final EasySubScan scan, errorContext = negotiator.parentErrorContext(); DrillFileSystem dfs = file.fileSystem(); path = dfs.makeQualified(file.split().getPath()); - in = dfs.openPossiblyCompressedStream(path); - // decode the pcap file - PcapDecoder decoder = new PcapDecoder(IOUtils.toByteArray(in)); - decoder.decode(); - pcapIterator = decoder.getSectionList().iterator(); + // Blocks are decoded one at a time as next() pulls them, so memory use + // does not grow with file size. + in = new BufferedInputStream(dfs.openPossiblyCompressedStream(path)); logger.debug("The config is {}, root is {}, columns has {}", config, scan.getSelectionRoot(), columns); } catch (IOException e) { throw UserException @@ -90,12 +116,12 @@ public PcapngBatchReader(final PcapFormatConfig config, final EasySubScan scan, .message("Failure in initial pcapng inputstream. " + e.getMessage()) .addContext(errorContext) .build(logger); - } catch (Exception e) { - throw UserException - .dataReadError(e) - .message("Failed to decode the pcapng file. " + e.getMessage()) - .addContext(errorContext) - .build(logger); + } + if (isSessionQuery()) { + negotiator.tableSchema(new Schema(true).buildSchema(new SchemaBuilder()), false); + loader = negotiator.build().writer(); + sessionizer = new TcpSessionizer(loader); + return; } // define the schema negotiator.tableSchema(defineMetadata(), true); @@ -123,29 +149,379 @@ public PcapngBatchReader(final PcapFormatConfig config, final EasySubScan scan, @Override public boolean next() { while (!loader.isFull()) { - if (!pcapIterator.hasNext()) { - return false; - } else if (config.getStat() && isIEnhancedPacketBlock()) { - continue; - } else if (!config.getStat() && !isIEnhancedPacketBlock()) { - continue; + PcapngBlock block; + try { + block = nextRow(); + } catch (IOException e) { + throw UserException + .dataReadError(e) + .message("Failed to decode the pcapng file. " + e.getMessage()) + .addContext(errorContext) + .build(logger); + } + if (block == null) { + // At end of file, more batches are needed only for open sessions that did not fit + return sessionizer != null && !sessionizer.writeOpenSessions(); + } + if (sessionizer != null) { + if (block.packet != null) { + sessionizer.addPacket(block.packet); + } + } else { + processBlock(block); } - processBlock(); } return true; } + /** + * Reads blocks until one produces a row: an Enhanced Packet Block for + * packet queries, a descriptive block for stat queries. Section and + * interface blocks are always tracked, since packets depend on them. + * + * @return the next row, or null at end of file + */ + private PcapngBlock nextRow() throws IOException { + while (true) { + int n = IOUtils.read(in, header, 0, 8); + if (n == 0) { + return null; + } + if (n < 8) { + logger.warn("Ignoring truncated pcapng block header at end of {}", path); + return null; + } + ByteBuffer buf = ByteBuffer.wrap(header).order(byteOrder); + int type = buf.getInt(0); + int consumed = 8; + if (type == SECTION_HEADER_TYPE) { + // Each section declares its own byte order; the type itself is a palindrome + IOUtils.readFully(in, header, 8, 4); + consumed = 12; + byteOrder = ByteBuffer.wrap(header, 8, 4).order(ByteOrder.BIG_ENDIAN).getInt() == BYTE_ORDER_MAGIC + ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; + buf.order(byteOrder); + } + int totalLength = buf.getInt(4); + if (totalLength < consumed + 4 || totalLength % 4 != 0) { + throw new IOException("Invalid pcapng block length " + totalLength + " for block type " + type); + } + // Body excludes the header and the trailing copy of the block length + byte[] body = new byte[totalLength - consumed - 4]; + try { + IOUtils.readFully(in, body); + IOUtils.skipFully(in, 4); + } catch (EOFException e) { + logger.warn("Ignoring truncated pcapng block at end of {}", path); + return null; + } + ByteBuffer block = ByteBuffer.wrap(body).order(byteOrder); + + if (type == SECTION_HEADER_TYPE) { + // Interface IDs are scoped to their section + interfaces.clear(); + } else if (type == INTERFACE_DESCRIPTION_TYPE) { + interfaces.add(new Interface(block)); + } else if (type == ENHANCED_PACKET_TYPE && !config.getStat()) { + return readPacket(block); + } + + if (config.getStat() && (type == SECTION_HEADER_TYPE || type == INTERFACE_DESCRIPTION_TYPE + || type == NAME_RESOLUTION_TYPE || type == INTERFACE_STATISTICS_TYPE)) { + PcapngBlock row = new PcapngBlock(); + row.comment = readComment(block, optionsOffset(type, block)); + row.stats = readStats(type, block); + return row; + } + } + } + + private PcapngBlock readPacket(ByteBuffer block) throws IOException { + PcapngBlock row = new PcapngBlock(); + row.interfaceId = block.getInt(0); + if (row.interfaceId < 0 || row.interfaceId >= interfaces.size()) { + throw new IOException("Packet references undefined interface " + row.interfaceId); + } + Interface iface = interfaces.get(row.interfaceId); + row.interfaceName = iface.name; + row.linkType = iface.linkType; + // 64-bit timestamp stored as high then low 32-bit words + row.timestamp = iface.toInstant((long) block.getInt(4) << 32 | (block.getInt(8) & 0xFFFFFFFFL)); + row.capturedLength = block.getInt(12); + row.originalLength = block.getInt(16); + if (row.capturedLength < 0 || 20 + row.capturedLength > block.capacity()) { + throw new IOException("Invalid captured length " + row.capturedLength); + } + row.data = Arrays.copyOfRange(block.array(), 20, 20 + row.capturedLength); + + StringBuilder comment = new StringBuilder(); + forEachOption(block, 20 + pad4(row.capturedLength), (code, start, length) -> { + switch (code) { + case OPT_COMMENT: + appendComment(comment, block, start, length); + break; + case EPB_FLAGS: + row.flags = block.getInt(start); + break; + case EPB_HASH: + row.hash = formatHash(block, start, length); + break; + case EPB_DROPCOUNT: + row.dropCount = block.getLong(start); + break; + default: + break; + } + }); + row.comment = comment.length() == 0 ? null : comment.toString(); + + // Decode once here instead of once per projected column + if (!isSkipQuery() || sessionizer != null) { + PacketDecoder packet = new PacketDecoder(); + if (packet.readPcapng(row.data, row.linkType)) { + packet.setTimestamp(row.timestamp); + row.packet = packet; + } + } + return row; + } + + /** + * Offset of the options in a descriptive block body (which, for a section + * header, starts after the byte-order magic). + */ + private static int optionsOffset(int type, ByteBuffer block) { + switch (type) { + case SECTION_HEADER_TYPE: + return 12; + case INTERFACE_DESCRIPTION_TYPE: + return 8; + case INTERFACE_STATISTICS_TYPE: + return 12; + case NAME_RESOLUTION_TYPE: + // Options follow the name records, which end with a zero record type + int offset = 0; + while (offset + 4 <= block.capacity()) { + int recordType = block.getShort(offset) & 0xFFFF; + int length = block.getShort(offset + 2) & 0xFFFF; + offset += 4 + pad4(length); + if (recordType == 0) { + break; + } + } + return offset; + default: + return block.capacity(); + } + } + + /** + * Decodes the options of a descriptive block into stat column values. + * Repeated options, such as several interface addresses, are comma separated. + */ + private Map readStats(int type, ByteBuffer block) { + Map stats = new HashMap<>(); + // Statistics timestamps use the resolution of the interface they describe + Interface iface = type == INTERFACE_STATISTICS_TYPE && block.getInt(0) >= 0 && block.getInt(0) < interfaces.size() + ? interfaces.get(block.getInt(0)) : null; + forEachOption(block, optionsOffset(type, block), (code, start, length) -> { + String name = statName(type, code); + if (name == null) { + return; + } + Object value; + switch (name) { + case "if_speed": + case "if_tsoffset": + case "isb_ifrecv": + case "isb_ifdrop": + case "isb_filteraccept": + case "isb_osdrop": + case "isb_usrdeliv": + value = block.getLong(start); + break; + case "if_tsresol": + case "if_fcslen": + value = block.get(start) & 0xFF; + break; + case "if_tzone": + value = block.getInt(start); + break; + case "isb_starttime": + case "isb_endtime": + long timestamp = (long) block.getInt(start) << 32 | (block.getInt(start + 4) & 0xFFFFFFFFL); + value = iface == null ? Interface.DEFAULT.toInstant(timestamp) : iface.toInstant(timestamp); + break; + case "if_ipv4addr": + // Address followed by netmask + value = formatAddress(block, start, 4) + "/" + Integer.bitCount(ByteBuffer.wrap(block.array(), start + 4, 4).getInt()); + break; + case "ns_dnsip4addr": + value = formatAddress(block, start, 4); + break; + case "if_ipv6addr": + // Address followed by prefix length + value = formatAddress(block, start, 16) + "/" + (block.get(start + 16) & 0xFF); + break; + case "ns_dnsip6addr": + value = formatAddress(block, start, 16); + break; + case "if_macaddr": + case "if_euiaddr": + StringBuilder mac = new StringBuilder(); + for (int i = start; i < start + length; i++) { + mac.append(mac.length() == 0 ? "" : ":").append(String.format("%02X", block.get(i))); + } + value = mac.toString(); + break; + default: + value = new String(block.array(), start, length, StandardCharsets.UTF_8); + break; + } + stats.merge(name, value, (a, b) -> a instanceof String ? a + ", " + b : b); + }); + return stats; + } + + private static String statName(int type, int code) { + switch (type) { + case SECTION_HEADER_TYPE: + return code >= 2 && code <= 4 ? new String[] {"shb_hardware", "shb_os", "shb_userappl"}[code - 2] : null; + case INTERFACE_DESCRIPTION_TYPE: + String[] names = {"if_name", "if_description", "if_ipv4addr", "if_ipv6addr", "if_macaddr", "if_euiaddr", + "if_speed", "if_tsresol", "if_tzone", null, "if_os", "if_fcslen", "if_tsoffset"}; + return code >= 2 && code <= 14 ? names[code - 2] : null; + case NAME_RESOLUTION_TYPE: + return code >= 2 && code <= 4 ? new String[] {"ns_dnsname", "ns_dnsip4addr", "ns_dnsip6addr"}[code - 2] : null; + case INTERFACE_STATISTICS_TYPE: + String[] isb = {"isb_starttime", "isb_endtime", "isb_ifrecv", "isb_ifdrop", "isb_filteraccept", + "isb_osdrop", "isb_usrdeliv"}; + return code >= 2 && code <= 8 ? isb[code - 2] : null; + default: + return null; + } + } + + private static String formatAddress(ByteBuffer block, int start, int length) { + try { + return InetAddress.getByAddress(Arrays.copyOfRange(block.array(), start, start + length)).getHostAddress(); + } catch (IOException e) { + return null; + } + } + + private static String readComment(ByteBuffer block, int offset) { + StringBuilder comment = new StringBuilder(); + forEachOption(block, offset, (code, start, length) -> { + if (code == OPT_COMMENT) { + appendComment(comment, block, start, length); + } + }); + return comment.length() == 0 ? null : comment.toString(); + } + + private static void appendComment(StringBuilder comment, ByteBuffer block, int start, int length) { + if (comment.length() > 0) { + comment.append('\n'); + } + comment.append(new String(block.array(), start, length, StandardCharsets.UTF_8)); + } + + private static String formatHash(ByteBuffer block, int start, int length) { + if (length < 1) { + return null; + } + int algorithm = block.get(start) & 0xFF; + StringBuilder hash = new StringBuilder(algorithm < HASH_ALGORITHMS.length + ? HASH_ALGORITHMS[algorithm] : String.valueOf(algorithm)).append(':'); + for (int i = start + 1; i < start + length; i++) { + hash.append(String.format("%02x", block.get(i))); + } + return hash.toString(); + } + + private interface OptionVisitor { + void visit(int code, int start, int length); + } + + /** Calls the visitor for each option from offset up to opt_endofopt or the end of the block. */ + private static void forEachOption(ByteBuffer block, int offset, OptionVisitor visitor) { + while (offset + 4 <= block.capacity()) { + int code = block.getShort(offset) & 0xFFFF; + int length = block.getShort(offset + 2) & 0xFFFF; + if (code == OPT_ENDOFOPT || offset + 4 + length > block.capacity()) { + return; + } + visitor.visit(code, offset + 4, length); + offset += 4 + pad4(length); + } + } + + private static int pad4(int length) { + return (length + 3) & ~3; + } + + /** What packets need from their Interface Description Block. */ + private static class Interface { + static final Interface DEFAULT = new Interface(); + final int linkType; + String name; + // Timestamps are counted in units of 1 / unitsPerSecond, plus an offset in seconds + long unitsPerSecond = 1_000_000; + long offsetSeconds; + + private Interface() { + linkType = PacketDecoder.LINKTYPE_ETHERNET; + } + + Interface(ByteBuffer block) throws IOException { + linkType = block.getShort(0) & 0xFFFF; + IOException[] error = new IOException[1]; + forEachOption(block, 8, (code, start, length) -> { + switch (code) { + case IF_NAME: + name = new String(block.array(), start, length, StandardCharsets.UTF_8); + break; + case IF_TSRESOL: + int resolution = block.get(start) & 0xFF; + int exponent = resolution & 0x7F; + if ((resolution & 0x80) == 0 && exponent <= 18) { + unitsPerSecond = (long) Math.pow(10, exponent); + } else if ((resolution & 0x80) != 0 && exponent <= 62) { + unitsPerSecond = 1L << exponent; + } else { + error[0] = new IOException("Unsupported if_tsresol " + resolution); + } + break; + case IF_TSOFFSET: + offsetSeconds = block.getLong(start); + break; + default: + break; + } + }); + if (error[0] != null) { + throw error[0]; + } + } + + Instant toInstant(long timestamp) { + long seconds = Long.divideUnsigned(timestamp, unitsPerSecond); + long remainder = Long.remainderUnsigned(timestamp, unitsPerSecond); + // remainder * 1e9 overflows only for resolutions finer than ~2^-33 s + long nanos = remainder <= Long.MAX_VALUE / 1_000_000_000L + ? remainder * 1_000_000_000L / unitsPerSecond + : (long) (remainder / (double) unitsPerSecond * 1_000_000_000L); + return Instant.ofEpochSecond(seconds + offsetSeconds, nanos); + } + } + @Override public void close() { AutoCloseables.closeSilently(in); } - private boolean isIEnhancedPacketBlock() { - block = pcapIterator.next(); - return block instanceof IEnhancedPacketBLock; - } - - private void processBlock() { + private void processBlock(PcapngBlock block) { loader.start(); for (ColumnDefn columnDefn : projectedColumns) { // pcapng file name @@ -159,6 +535,13 @@ private void processBlock() { loader.save(); } + /** + * Stat queries take precedence: they describe the capture, not its packets. + */ + private boolean isSessionQuery() { + return config.getSessionizeTCPStreams() && !config.getStat(); + } + private boolean isSkipQuery() { return columns.isEmpty(); } @@ -258,7 +641,7 @@ public void define(SchemaBuilder builder) { } } - public void load(IPcapngType block) { + public void load(PcapngBlock block) { getProcessor().process(block, writer); } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBlock.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBlock.java new file mode 100644 index 00000000000..d872fd13577 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBlock.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcapng; + +import java.time.Instant; +import java.util.Map; + +/** + * One pcapng block being written as a row, with everything the reader + * decoded from it. Fields that do not apply to the block are null. + */ +class PcapngBlock { + + /** Options of a descriptive block by stat column name (stat queries only). */ + Map stats; + /** All opt_comment options of the block, newline separated. */ + String comment; + + // Enhanced Packet Block fields, resolved against the block's interface + int interfaceId; + String interfaceName; + int linkType; + Instant timestamp; + int capturedLength; + int originalLength; + byte[] data; + /** epb_flags, or null if absent. */ + Integer flags; + Long dropCount; + String hash; + /** Decoded packet, or null if the link type or protocol is not supported. */ + PacketDecoder packet; +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapEVFReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapEVFReader.java index ed465402627..aa48a360c7f 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapEVFReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapEVFReader.java @@ -79,8 +79,7 @@ public void testAggregateQuery() throws Exception { .sqlQuery(sql) .ordered() .baselineColumns("is_corrupt", "packet_count") - .baselineValues(false, 6984L) - .baselineValues(true, 16L) + .baselineValues(false, 7000L) .go(); } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java index 43fc3ff3b40..adc4d2dd88b 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java @@ -50,12 +50,12 @@ public void testCorruptPCAPQuery() throws Exception { @Test public void testTrueCorruptPCAPQuery() throws Exception { - runSQLVerifyCount("select * from dfs.`pcap/testv1.pcap` WHERE is_corrupt=true", 16); + runSQLVerifyCount("select * from dfs.`pcap/testv1.pcap` WHERE is_corrupt=true", 0); } @Test public void testNotCorruptPCAPQuery() throws Exception { - runSQLVerifyCount("select * from dfs.`pcap/testv1.pcap` WHERE is_corrupt=false", 6984); + runSQLVerifyCount("select * from dfs.`pcap/testv1.pcap` WHERE is_corrupt=false", 7000); } @Test diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java index 1a306de3abe..aa88a7bdfbb 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java @@ -52,7 +52,7 @@ public static void setup() throws Exception { @Test public void testSessionizedStarQuery() throws Exception { - String sql = "SELECT * FROM cp.`/pcap/http.pcap`"; + String sql = "SELECT * FROM cp.`/pcap/http.pcap` WHERE session_closed = true"; String dataFromRemote = readAFileIntoString(dirTestWatcher.getRootDir().getAbsolutePath() + "/pcap/dataFromRemote.txt"); QueryBuilder q = client.queryBuilder().sql(sql); @@ -76,6 +76,7 @@ public void testSessionizedStarQuery() throws Exception { .addNullable("connection_time", TypeProtos.MinorType.INTERVAL) .addNullable("tcp_session", TypeProtos.MinorType.BIGINT) .addNullable("is_corrupt", TypeProtos.MinorType.BIT) + .addNullable("session_closed", TypeProtos.MinorType.BIT) .addNullable("data_from_originator", TypeProtos.MinorType.VARCHAR) .addNullable("data_from_remote", TypeProtos.MinorType.VARCHAR) .buildSchema(); @@ -90,10 +91,10 @@ public void testSessionizedStarQuery() throws Exception { 1084443427311L, 1084443445216L, Period.parse("PT17.905S"), 31, - 437,18000,14, 17, + 479, 18364, 14, 17, Period.parse("PT0.911S"), - -789689725566200012L, false, - "r-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113..Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1..Accept-Language: en-us,en;q=0.5..Accept-Encoding: gzip,deflate..Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7..Keep-Alive: 300..Connection: keep-alive..Referer: http://www.ethereal.com/development.html....$K.@....6...6", + -789689725566200012L, false, true, + "GET /download.html HTTP/1.1..Host: www.ethereal.com..User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113..Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1..Accept-Language: en-us,en;q=0.5..Accept-Encoding: gzip,deflate..Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7..Keep-Alive: 300..Connection: keep-alive..Referer: http://www.ethereal.com/development.html", dataFromRemote ) .build(); @@ -106,7 +107,7 @@ public void testSessionizedSpecificQuery() throws Exception { String sql = "SELECT src_ip, dst_ip, src_port, dst_port, src_mac_address, dst_mac_address," + "session_start_time, session_end_time, session_duration, total_packet_count, data_volume_from_origin, data_volume_from_remote," + "packet_count_from_origin, packet_count_from_remote, connection_time, tcp_session, is_corrupt, data_from_originator, data_from_remote " + - "FROM cp.`/pcap/http.pcap`"; + "FROM cp.`/pcap/http.pcap` WHERE session_closed = true"; String dataFromRemote = readAFileIntoString(dirTestWatcher.getRootDir().getAbsolutePath() + "/pcap/dataFromRemote.txt"); @@ -145,10 +146,10 @@ public void testSessionizedSpecificQuery() throws Exception { 1084443427311L, 1084443445216L, Period.parse("PT17.905S"), 31, - 437,18000,14, 17, + 479, 18364, 14, 17, Period.parse("PT0.911S"), -789689725566200012L, false, - "r-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113..Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1..Accept-Language: en-us,en;q=0.5..Accept-Encoding: gzip,deflate..Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7..Keep-Alive: 300..Connection: keep-alive..Referer: http://www.ethereal.com/development.html....$K.@....6...6", + "GET /download.html HTTP/1.1..Host: www.ethereal.com..User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113..Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1..Accept-Language: en-us,en;q=0.5..Accept-Encoding: gzip,deflate..Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7..Keep-Alive: 300..Connection: keep-alive..Referer: http://www.ethereal.com/development.html", dataFromRemote ) .build(); @@ -161,7 +162,35 @@ public void testSerDe() throws Exception { String sql = "SELECT COUNT(*) FROM cp.`/pcap/http.pcap`"; String plan = queryBuilder().sql(sql).explainJson(); long cnt = queryBuilder().physical(plan).singletonLong(); - assertEquals("Counts should match", 1L, cnt); + // The closed HTTP session, and one to 216.239.59.99 still open when the capture ended + assertEquals("Counts should match", 2L, cnt); + } + + @Test + public void testUnclosedSession() throws Exception { + // The capture starts after this connection's handshake and ends before its FIN + String sql = "SELECT src_ip, dst_ip, src_port, dst_port, total_packet_count, data_volume_from_origin, " + + "data_volume_from_remote, connection_time, substr(data_from_originator, 1, 27) AS request " + + "FROM cp.`/pcap/http.pcap` WHERE session_closed = false"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + + TupleMetadata expectedSchema = new SchemaBuilder() + .addNullable("src_ip", TypeProtos.MinorType.VARCHAR) + .addNullable("dst_ip", TypeProtos.MinorType.VARCHAR) + .addNullable("src_port", TypeProtos.MinorType.INT) + .addNullable("dst_port", TypeProtos.MinorType.INT) + .addNullable("total_packet_count", TypeProtos.MinorType.INT) + .addNullable("data_volume_from_origin", TypeProtos.MinorType.INT) + .addNullable("data_volume_from_remote", TypeProtos.MinorType.INT) + .addNullable("connection_time", TypeProtos.MinorType.INTERVAL) + .addNullable("request", TypeProtos.MinorType.VARCHAR) + .buildSchema(); + + RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema) + .addRow("145.254.160.237", "216.239.59.99", 3371, 80, 7, 721, 3020, null, "GET /pagead/ads?client=ca-p") + .build(); + + new RowSetComparison(expected).verifyAndClearAll(results); } /** diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java index 7c9cd400446..cd6bf82cf6d 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java @@ -35,7 +35,6 @@ import org.apache.drill.test.QueryTestUtil; import org.apache.drill.test.rowSet.RowSetComparison; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -46,7 +45,6 @@ public class TestPcapngRecordReader extends ClusterTest { public static void setup() throws Exception { ClusterTest.startCluster(ClusterFixture.builder(dirTestWatcher)); dirTestWatcher.copyResourceToRoot(Paths.get("pcapng/")); - dirTestWatcher.copyResourceToRoot(Paths.get("todo/")); } @Test @@ -61,7 +59,7 @@ public void testStarQuery() throws Exception { @Test public void testExplicitQuery() throws Exception { - String sql = "select type, packet_length, `timestamp` from dfs.`pcapng/sniff.pcapng` where type = 'ARP'"; + String sql = "select type, packet_length, `timestamp` from dfs.`pcapng/sniff.pcapng` where type = 'ARP' limit 2"; QueryBuilder builder = client.queryBuilder().sql(sql); RowSet sets = builder.rowSet(); @@ -72,8 +70,8 @@ public void testExplicitQuery() throws Exception { .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) - .addRow("ARP", 90, Instant.ofEpochMilli(1518010669927L)) - .addRow("ARP", 90, Instant.ofEpochMilli(1518010671874L)) + .addRow("ARP", 60, Instant.ofEpochMilli(1518010666140L)) + .addRow("ARP", 60, Instant.ofEpochMilli(1518010666140L)) .build(); assertEquals(2, sets.rowCount()); @@ -99,10 +97,23 @@ public void testSerDe() throws Exception { assertEquals("Counts should match", 1, cnt); } + @Test + public void testMixedPcapAndPcapngDirectory() throws Exception { + // Each file must be read with the reader for its own format + dirTestWatcher.copyResourceToRoot(Paths.get("pcapng/sniff.pcapng"), Paths.get("mixed/sniff.pcapng")); + dirTestWatcher.copyResourceToRoot(Paths.get("pcap/tcp-1.pcap"), Paths.get("mixed/tcp-1.pcap")); + long pcapng = queryBuilder().sql("select count(*) from dfs.`mixed/sniff.pcapng`").singletonLong(); + long pcap = queryBuilder().sql("select count(*) from dfs.`mixed/tcp-1.pcap`").singletonLong(); + long both = queryBuilder().sql("select count(*) from dfs.`mixed`").singletonLong(); + + assertEquals(123, pcapng); + assertEquals(pcapng + pcap, both); + } + @Test public void testExplicitQueryWithCompressedFile() throws Exception { QueryTestUtil.generateCompressedFile("pcapng/sniff.pcapng", "zip", "pcapng/sniff.pcapng.zip"); - String sql = "select type, packet_length, `timestamp` from dfs.`pcapng/sniff.pcapng.zip` where type = 'ARP'"; + String sql = "select type, packet_length, `timestamp` from dfs.`pcapng/sniff.pcapng.zip` where type = 'ARP' limit 2"; QueryBuilder builder = client.queryBuilder().sql(sql); RowSet sets = builder.rowSet(); @@ -113,8 +124,8 @@ public void testExplicitQueryWithCompressedFile() throws Exception { .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) - .addRow("ARP", 90, Instant.ofEpochMilli(1518010669927L)) - .addRow("ARP", 90, Instant.ofEpochMilli(1518010671874L)) + .addRow("ARP", 60, Instant.ofEpochMilli(1518010666140L)) + .addRow("ARP", 60, Instant.ofEpochMilli(1518010666140L)) .build(); assertEquals(2, sets.rowCount()); @@ -185,6 +196,16 @@ public void testValidHeaders() throws Exception { .addNullable("tcp_flags_fin", MinorType.INT) .addNullable("tcp_parsed_flags", MinorType.VARCHAR) .addNullable("packet_data", MinorType.VARCHAR) + .add("captured_length", MinorType.INT) + .add("interface_id", MinorType.INT) + .addNullable("interface_name", MinorType.VARCHAR) + .add("link_type", MinorType.INT) + .addNullable("comment", MinorType.VARCHAR) + .addNullable("direction", MinorType.VARCHAR) + .addNullable("reception_type", MinorType.VARCHAR) + .addNullable("fcs_length", MinorType.INT) + .addNullable("drop_count", MinorType.BIGINT) + .addNullable("packet_hash", MinorType.VARCHAR) .build(); RowSet expected = new RowSetBuilder(client.allocator(), schema).build(); @@ -192,17 +213,146 @@ public void testValidHeaders() throws Exception { } @Test - @Ignore // todo: infinite loop with current PcapNGReader - public void testPcapNG() throws Exception { -// String sql = "select * from dfs.`todo/dhcp_big_endian.pcapng` limit 1"; // Bad magic number = 000a0a0a -// String sql = "select * from dfs.`todo/dhcp_little_endian.pcapng` limit 1"; // Bad magic number = 1c0a0a0a -// String sql = "select * from dfs.`todo/many_interfaces.pcapng` limit 1"; // Bad magic number = ef0a0a0a - String sql = "select * from dfs.`todo/mac2.pcap` limit 1"; // Bad magic number = 1c0a0a0a - QueryBuilder builder = client.queryBuilder().sql(sql); - RowSet sets = builder.rowSet(); + public void testBigEndian() throws Exception { + // The same capture written in both byte orders must decode identically. + // (The upstream big-endian copy has zeroed timestamps, so they are not compared.) + String sql = "select packet_length, src_ip, dst_ip, src_port, dst_port, src_mac_address from dfs.`pcapng/%s`"; + RowSet little = client.queryBuilder().sql(String.format(sql, "dhcp.pcapng")).rowSet(); + RowSet big = client.queryBuilder().sql(String.format(sql, "dhcp_big_endian.pcapng")).rowSet(); + + assertEquals(4, little.rowCount()); + new RowSetComparison(little).verifyAndClearAll(big); + } - assertEquals(1, sets.rowCount()); - sets.clear(); + @Test + public void testManyInterfaces() throws Exception { + // 11 interfaces plus statistics and name resolution blocks between the packets + String sql = "select count(*) from dfs.`pcapng/many_interfaces.pcapng`"; + assertEquals(64, queryBuilder().sql(sql).singletonLong()); + } + + /** + * metadata.pcapng has one UDP packet from 10.0.0.N:100N per interface, each + * on a different link type, plus a second big-endian section in which + * interface 0 is raw IP. All timestamps are 2024-01-02T03:04:05.678Z except + * eth1, which uses 2^-10 s units and if_tsoffset to give 03:04:05.500Z. + */ + @Test + public void testLinkTypesAndInterfaces() throws Exception { + String sql = "select interface_id, link_type, interface_name, `timestamp`, src_ip, src_port, " + + "src_mac_address, captured_length from dfs.`pcapng/metadata.pcapng`"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + + TupleMetadata schema = new SchemaBuilder() + .add("interface_id", MinorType.INT) + .add("link_type", MinorType.INT) + .addNullable("interface_name", MinorType.VARCHAR) + .add("timestamp", MinorType.TIMESTAMP) + .addNullable("src_ip", MinorType.VARCHAR) + .addNullable("src_port", MinorType.INT) + .addNullable("src_mac_address", MinorType.VARCHAR) + .add("captured_length", MinorType.INT) + .buildSchema(); + + Instant ts = Instant.parse("2024-01-02T03:04:05.678Z"); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow(0, 1, "eth0", ts, "10.0.0.0", 1000, "02:00:00:00:00:01", 44) + .addRow(1, 101, "tun0", ts, "10.0.0.1", 1001, null, 30) // nanosecond resolution + .addRow(2, 113, "any", ts, "10.0.0.2", 1002, null, 46) + .addRow(3, 0, "lo0", ts, "10.0.0.3", 1003, null, 34) + .addRow(4, 9, "ppp0", ts, "10.0.0.4", 1004, null, 34) + .addRow(5, 276, "any2", ts, "10.0.0.5", 1005, null, 50) + .addRow(6, 105, "wlan0", ts, null, null, null, 54) // 802.11 is not decoded + .addRow(7, 1, "eth1", Instant.parse("2024-01-02T03:04:05.500Z"), "10.0.0.7", 1007, "02:00:00:00:00:01", 44) + .addRow(0, 101, "be0", ts, "10.0.0.9", 1009, null, 30) + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + /** + * encapsulations.pcapng: 802.11 (FromDS), radiotap + 802.11 QoS (ToDS) with TCP options, + * an 802.1Q tag, QinQ + IPv6 with hop-by-hop and routing headers, an encrypted 802.11 + * frame, and an Ethernet frame with trailing padding. + */ + @Test + public void testEncapsulations() throws Exception { + String sql = "select interface_name, type, src_ip, dst_ip, src_port, dst_port, src_mac_address, " + + "dst_mac_address, tcp_flags from dfs.`pcapng/encapsulations.pcapng`"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + + TupleMetadata schema = new SchemaBuilder() + .addNullable("interface_name", MinorType.VARCHAR) + .addNullable("type", MinorType.VARCHAR) + .addNullable("src_ip", MinorType.VARCHAR) + .addNullable("dst_ip", MinorType.VARCHAR) + .addNullable("src_port", MinorType.INT) + .addNullable("dst_port", MinorType.INT) + .addNullable("src_mac_address", MinorType.VARCHAR) + .addNullable("dst_mac_address", MinorType.VARCHAR) + .addNullable("tcp_flags", MinorType.INT) + .buildSchema(); + + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("wlan-fromds", "UDP", "192.168.1.3", "192.168.1.1", 5353, 53, "02:00:00:00:00:03", "02:00:00:00:00:01", 0) + .addRow("radiotap-qos", "TCP", "10.0.0.4", "10.0.0.5", 40000, 443, "02:00:00:00:00:04", "02:00:00:00:00:05", 24) + .addRow("vlan", "UDP", "10.1.0.6", "10.1.0.7", 1111, 2222, "02:00:00:00:00:06", "02:00:00:00:00:07", 0) + .addRow("qinq-ipv6", "TCP", "2001:db8:0:0:0:0:0:8", "2001:db8:0:0:0:0:0:9", 50000, 80, "02:00:00:00:00:08", "02:00:00:00:00:09", 24) + .addRow("wlan-protected", null, null, null, null, null, null, null, null) + .addRow("eth-padded", "UDP", "10.2.0.10", "10.2.0.11", 7, 9, "02:00:00:00:00:0A", "02:00:00:00:00:0B", 0) + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testSessionizedTcpStreams() throws Exception { + // http.pcapng holds the same packets as pcap/http.pcap, so both readers must build the same sessions + dirTestWatcher.copyResourceToRoot(Paths.get("pcap/http.pcap"), Paths.get("pcap/http.pcap")); + String sql = "select * from table(dfs.`%s` (type => '%s', sessionizeTCPStreams => true))"; + RowSet pcap = client.queryBuilder().sql(String.format(sql, "pcap/http.pcap", "pcap")).rowSet(); + RowSet pcapng = client.queryBuilder().sql(String.format(sql, "pcapng/http.pcapng", "pcapng")).rowSet(); + + assertEquals(2, pcapng.rowCount()); + new RowSetComparison(pcap).verifyAndClearAll(pcapng); + + sql = "select session_closed, data_volume_from_origin, data_volume_from_remote, substr(data_from_originator, 1, 27) as request " + + "from table(dfs.`pcapng/http.pcapng` (type => 'pcapng', sessionizeTCPStreams => true))"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("session_closed", MinorType.BIT) + .addNullable("data_volume_from_origin", MinorType.INT) + .addNullable("data_volume_from_remote", MinorType.INT) + .addNullable("request", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow(true, 479, 18364, "GET /download.html HTTP/1.1") + // Still open when the capture ended + .addRow(false, 721, 3020, "GET /pagead/ads?client=ca-p") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testPacketOptions() throws Exception { + String sql = "select interface_name, comment, direction, reception_type, fcs_length, drop_count, packet_hash " + + "from dfs.`pcapng/metadata.pcapng` where interface_name in ('eth0', 'eth1', 'tun0')"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + + TupleMetadata schema = new SchemaBuilder() + .addNullable("interface_name", MinorType.VARCHAR) + .addNullable("comment", MinorType.VARCHAR) + .addNullable("direction", MinorType.VARCHAR) + .addNullable("reception_type", MinorType.VARCHAR) + .addNullable("fcs_length", MinorType.INT) + .addNullable("drop_count", MinorType.BIGINT) + .addNullable("packet_hash", MinorType.VARCHAR) + .buildSchema(); + + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("eth0", "first packet", "inbound", "unicast", 4, 7L, "md5:000102030405060708090a0b0c0d0e0f") + .addRow("tun0", null, null, null, null, null, null) + .addRow("eth1", null, "outbound", null, null, null, null) + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); } @Test @@ -243,7 +393,7 @@ public void testPcapNGFileWithPcapExt() throws Exception { @Test public void testInlineSchema() throws Exception { String sql = "SELECT type, packet_length, `timestamp` FROM table(dfs.`pcapng/sniff.pcapng` " + - "(type => 'pcapng', stat => false, sessionizeTCPStreams => true )) where type = 'ARP'"; + "(type => 'pcapng', stat => false, sessionizeTCPStreams => false )) where type = 'ARP' limit 2"; RowSet sets = client.queryBuilder().sql(sql).rowSet(); TupleMetadata schema = new SchemaBuilder() @@ -253,8 +403,8 @@ public void testInlineSchema() throws Exception { .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) - .addRow("ARP", 90, Instant.ofEpochMilli(1518010669927L)) - .addRow("ARP", 90, Instant.ofEpochMilli(1518010671874L)) + .addRow("ARP", 60, Instant.ofEpochMilli(1518010666140L)) + .addRow("ARP", 60, Instant.ofEpochMilli(1518010666140L)) .build(); assertEquals(2, sets.rowCount()); diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java index 185a7b6d089..b49f8ec47e2 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java @@ -92,13 +92,29 @@ public void testLimitPushdown() throws Exception { sets.clear(); } + @Test + public void testComments() throws Exception { + String sql = "select comment from dfs.`pcapng/metadata.pcapng` where comment is not null"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + + TupleMetadata schema = new SchemaBuilder() + .addNullable("comment", MinorType.VARCHAR) + .buildSchema(); + + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("capture comment") + .addRow("stats comment") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + @Test public void testSerDe() throws Exception { String sql = "select count(*) from dfs.`pcapng/*.pcapng`"; String plan = queryBuilder().sql(sql).explainJson(); long cnt = queryBuilder().physical(plan).singletonLong(); - assertEquals("Counts should match", 6, cnt); + assertEquals("Counts should match", 55, cnt); } @Test @@ -108,6 +124,7 @@ public void testValidHeaders() throws Exception { TupleMetadata schema = new SchemaBuilder() .addNullable("path", MinorType.VARCHAR) + .addNullable("comment", MinorType.VARCHAR) .addNullable("shb_hardware", MinorType.VARCHAR) .addNullable("shb_os", MinorType.VARCHAR) .addNullable("shb_userappl", MinorType.VARCHAR) @@ -117,12 +134,12 @@ public void testValidHeaders() throws Exception { .addNullable("if_ipv6addr", MinorType.VARCHAR) .addNullable("if_macaddr", MinorType.VARCHAR) .addNullable("if_euiaddr", MinorType.VARCHAR) - .addNullable("if_speed", MinorType.INT) + .addNullable("if_speed", MinorType.BIGINT) .addNullable("if_tsresol", MinorType.INT) .addNullable("if_tzone", MinorType.INT) .addNullable("if_os", MinorType.VARCHAR) .addNullable("if_fcslen", MinorType.INT) - .addNullable("if_tsoffset", MinorType.INT) + .addNullable("if_tsoffset", MinorType.BIGINT) .addNullable("ns_dnsname", MinorType.VARCHAR) .addNullable("ns_dnsip4addr", MinorType.VARCHAR) .addNullable("ns_dnsip6addr", MinorType.VARCHAR) diff --git a/contrib/format-pcapng/src/test/resources/pcap/dataFromRemote.txt b/contrib/format-pcapng/src/test/resources/pcap/dataFromRemote.txt index cde5c694d81..9e8b63eaaa7 100644 --- a/contrib/format-pcapng/src/test/resources/pcap/dataFromRemote.txt +++ b/contrib/format-pcapng/src/test/resources/pcap/dataFromRemote.txt @@ -1 +1 @@ -10:17:12 GMT..Server: Apache..Last-Modified: Tue, 20 Apr 2004 13:17:00 GMT..ETag: "9a01a-4696-7e354b00"..Accept-Ranges: bytes..Content-Length: 18070..Keep-Alive: timeout=15, max=100..Connection: Keep-Alive..Content-Type: text/html; charset=ISO-8859-1....... . Ethereal: Download. .. .
. . . . . . .
.. . .

Ethereal

.
Download
.
.. . . .. .. .... . .
..
. %K.@....6...6./a>...
..
..
. ... ...
..
... ...
.. .
.
.
.

. Home. |. Introduction. |. Download. |. Documentation. |. Lists. |. FAQ. |. Development.

.
...
...
..
.

Official Releases

.

. The Official source code release and installers for Windows, Red Hat. Linux/Fedora, and Solaris can be found on the main Ethereal web site and. its mirrors..

.

Source Code

.

.HTTP:.Main site.Australia.Australia.Germany.Japan.Korea.Malaysia.Sweden.SourceForge.

.

.FTP:.Main site.Australia.Australia.Austria.Germany.Japan.Mexico.Sweden.

.

.The latest development sources are available via.anonymous CVS..

.

Windows 98/ME/2000/XP/2003 Installers

.

.HTTP:.Main site.Australia.Australia.Germany.Japan.Korea.Malaysia.Sweden.SourceForge.

.

.FTP:.Australia.Australia.Austria.Germany.Japan.Mexico.Sweden.

.

Red Hat Linux / Fedora Packages

.

.HTTP:.Main site.Australia.Australia.Germany.Japan.Korea.Malaysia.Sweden.

.

.FTP:.Main site.Australia.Australia.Austria.Germany.Japan.Mexico.Sweden.

.

Solaris Packages

.

.HTTP:.Main site.Australia.Australia.Germany.Japan.Korea.Malaysia.Sweden.SourceForge.

.

.FTP:.Main site.Australia.Austria.Germany.Japan.Mexico.Sweden.

.
.
.

Other Platforms

.

. Binary distributions and ready-to-compile packages are available for. most platforms. Please note these packages may depend on external. libraries including GLib/GTK+, libpcap, Net-SNMP, PCRE, and GNU ADNS.. You may have to download and install them before installing Ethereal..

... . ... . ... . ... . ... . ... . ... . ... . ... . .. .. . ... . ... . ... . . ... . ... . ... . ... . ... The Written Word (6.5)1
. SGI Freeware.... . ... . ... . ... . ..
PlatformLocation(s)
Apple Computer:
Mac OS X
. Fink Project. DarwinPorts.
Be (Palm?):
BeOS
. BeBits.
Debian:
Debian GNU/Linux
. &K.@iM........table/net/ethereal">stable,. testing,. unstable.
The FreeBSD Project:
FreeBSD
. ports.
Gentoo Technologies:
Gentoo Linux
. portage.
HP:
Tru64 Unix
. The Written Word (4.0d, 5.1)1.
HP:
HP-UX
. UK,. Germany,. Netherlands,. US,. Canada,. South Afr&K.@iM..6...6. on each site's home page),
.
The Written Word (10.20, 11.00, 11.11)1.
IBM:
AIX
. Bull archive
. Vienna University mirror
. UCLA mirror.
IBM:
S/390 Linux (Red Hat 7.2)
. Ashley Chaloner
.
MandrakeSoft:
Mandrake Linux
. Cooker. (in the contrib section).
Microsoft:
Windows (Intel, 32-bit)
. local archive
. OPENEXTRA.
NetBSD Foundation:
NetBSD
. OpenBSD:
OpenBSD
. ports.
PLD Team:
PLD Linux
. FTP site.
Red Hat:
Red Hat Linux
. RPMFind (requires glibc)
. Henri Gomez
. The Written Word (7.1)1
. local archive
.
ROCK Linux:
ROCK Linux
. package
.
SCO (formerly Caldera):
UnixWare/OpenUnix
. Skunkware:. UnixWare 7. Open UNIX 8.
Slackware Linux:
Slackware Linux
. Linux Packages.
Sun Microsystems:
Solaris/Intel
. Solaris 01/01 update (unsupported).
Sun Microsystems:
Solaris/SPARC
. local archive (8, 9)
. The Written Word (2.5.1 - 9)1
. Sunfreeware.com (7, 8)
. Solaris 8 and 9 Companion Software CDs (unsupported).
SuSE:
SuSE Linux
. SuSE FTP site.. Mirrors are also available..
.

. If you know of any binary distribution not listed here, please send mail. to. ethereal-web[AT]ethereal.com...

.

. [1] Each Ethereal package produced by. The Written Word depends on the. zlib,. Glib,. GTK+,. Perl, and. Net-SNMP. packages.. Please refer to The Written Word's. documentation. for installation instructions.. Please do not call The Written Word for support. Email. free-support[AT]thewrittenword.com. with questions..

.
.
.

Other Downloads

.

Sample Captures

.

. A menagerie of capture files is available on our. sample captures pag'K.@`/..6...6.DF version of the Ethereal User's Guide is available in the. documentation page..

.
. .

.AIX is a registered trademark of International Business Machines, Inc..Tru64 is a registered trademark of Compaq Computer Corporation..Debian is a registered trademark of Software In The Public Interest, Inc..FreeBSD is a registered trademark of Walnut Creek CDROM, Inc..HP-UX is a registered trademark of Hewlett-Packard Company..Irix is a registered trademark of Silicon Graphics, Inc..LinuxPPC is a trademark of Jeff Carr..Mac OS is a registered trademark of Apple Computer, Inc..NetBSD is a registered trademark of the NetBSD Foundation..Red Hat is a registered trademark of Red Hat, Inc..Linux is a registered trademark of Linus Torvalds..SCO and Unixware are registered trademarks of Santa Cruz Operation, Inc..Slackware is a registered trademark of Patrick Volkerding..Solaris is a registered trademark of Sun Microsystems, Inc..SuSE is a registered trademark of SuSE AG..Microsoft, Windows, Windows 95, Windows 98, Windows ME, Windows NT,.Windows 2000, and Windows XP are registered trademarks of Microsoft,.Inc..All other trademarks on this site are property of their respective owners..

.
....(K.@....6...6 \ No newline at end of file +HTTP/1.1 200 OK..Date: Thu, 13 May 2004 10:17:12 GMT..Server: Apache..Last-Modified: Tue, 20 Apr 2004 13:17:00 GMT..ETag: "9a01a-4696-7e354b00"..Accept-Ranges: bytes..Content-Length: 18070..Keep-Alive: timeout=15, max=100..Connection: Keep-Alive..Content-Type: text/html; charset=ISO-8859-1....... . Ethereal: Download. .. .
. . . . . . .
.. . .

Ethereal

.
Download
.
.. . . .. .. .... . .
..
. Search:...
..
..
. ... ...
..
... ...
.. .
.
.
.

. Home. |. Introduction. |. Download. |. Documentation. |. Lists. |. FAQ. |. Development.

.
...
...
..
.

Official Releases

.

. The Official source code release and installers for Windows, Red Hat. Linux/Fedora, and Solaris can be found on the main Ethereal web site and. its mirrors..

.

Source Code

.

.HTTP:.Main site.Australia.Australia.Germany.Japan.Korea.Malaysia.Sweden.SourceForge.

.

.FTP:.Main site.Australia.Australia.Austria.Germany.Japan.Mexico.Sweden.

.

.The latest development sources are available via.anonymous CVS..

.

Windows 98/ME/2000/XP/2003 Installers

.

.HTTP:.Main site.Australia.Australia.Germany.Japan.Korea.Malaysia.Sweden.SourceForge.

.

.FTP:.Main site.Australia.Australia.Austria.Germany.Japan.Mexico.Sweden.

.

Red Hat Linux / Fedora Packages

.

.HTTP:.Main site.Australia.Australia.Germany.Japan.Korea.Malaysia.Sweden.

.

.FTP:.Main site.Australia.Australia.Austria.Germany.Japan.Mexico.Sweden.

.

Solaris Packages

.

.HTTP:.Main site.Australia.Australia.Germany.Japan.Korea.Malaysia.Sweden.SourceForge.

.

.FTP:.Main site.Australia.Australia.Austria.Germany.Japan.Mexico.Sweden.

.
.
.

Other Platforms

.

. Binary distributions and ready-to-compile packages are available for. most platforms. Please note these packages may depend on external. libraries including GLib/GTK+, libpcap, Net-SNMP, PCRE, and GNU ADNS.. You may have to download and install them before installing Ethereal..

... . ... . ... . ... . ... . ... . ... . ... . ... . .. .. . ... . ... . ... . ... . ... . ... . ... . ... . ... . ... . ... . ... . ... . ..
PlatformLocation(s)
Apple Computer:
Mac OS X
. Fink Project. DarwinPorts.
Be (Palm?):
BeOS
. BeBits.
Debian:
Debian GNU/Linux
. stable,. testing,. unstable.
The FreeBSD Project:
FreeBSD
. ports.
Gentoo Technologies:
Gentoo Linux
. portage.
HP:
Tru64 Unix
. The Written Word (4.0d, 5.1)1.
HP:
HP-UX
. UK,. Germany,. Netherlands,. US,. Canada,. South Africa.
(more mirrors are listed on each site's home page),
. The Written Word (10.20, 11.00, 11.11)1.
IBM:
AIX
. Bull archive
. Vienna University mirror
. UCLA mirror.
IBM:
S/390 Linux (Red Hat 7.2)
. Ashley Chaloner
.
MandrakeSoft:
Mandrake Linux
. Cooker. (in the contrib section).
Microsoft:
Windows (Intel, 32-bit)
. local archive
. OPENEXTRA.
NetBSD Foundation:
NetBSD
. packages.
OpenBSD:
OpenBSD
. ports.
PLD Team:
PLD Linux
. FTP site.
Red Hat:
Red Hat Linux
. RPMFind (requires glibc)
. Henri Gomez
. The Written Word (7.1)1
. local archive
.
ROCK Linux:
ROCK Linux
. package
.
SCO (formerly Caldera):
UnixWare/OpenUnix
. Skunkware:. UnixWare 7. Open UNIX 8.
SGI:
Irix
. The Written Word (6.5)1
. SGI Freeware.
Slackware Linux:
Slackware Linux
. Linux Packages.
Sun Microsystems:
Solaris/Intel
. Solaris 01/01 update (unsupported).
Sun Microsystems:
Solaris/SPARC
. local archive (8, 9)
. The Written Word (2.5.1 - 9)1
. Sunfreeware.com (7, 8)
. Solaris 8 and 9 Companion Software CDs (unsupported).
SuSE:
SuSE Linux
. SuSE FTP site.. Mirrors are also available..
.

. If you know of any binary distribution not listed here, please send mail. to. ethereal-web[AT]ethereal.com...

.

. [1] Each Ethereal package produced by. The Written Word depends on the. zlib,. Glib,. GTK+,. Perl, and. Net-SNMP. packages.. Please refer to The Written Word's. documentation. for installation instructions.. Please do not call The Written Word for support. Email. free-support[AT]thewrittenword.com. with questions..

.
.
.

Other Downloads

.

Sample Captures

.

. A menagerie of capture files is available on our. sample captures page..

.

Documentation

.

. A PDF version of the Ethereal User's Guide is available in the. documentation page..

.
. .

.AIX is a registered trademark of International Business Machines, Inc..Tru64 is a registered trademark of Compaq Computer Corporation..Debian is a registered trademark of Software In The Public Interest, Inc..FreeBSD is a registered trademark of Walnut Creek CDROM, Inc..HP-UX is a registered trademark of Hewlett-Packard Company..Irix is a registered trademark of Silicon Graphics, Inc..LinuxPPC is a trademark of Jeff Carr..Mac OS is a registered trademark of Apple Computer, Inc..NetBSD is a registered trademark of the NetBSD Foundation..Red Hat is a registered trademark of Red Hat, Inc..Linux is a registered trademark of Linus Torvalds..SCO and Unixware are registered trademarks of Santa Cruz Operation, Inc..Slackware is a registered trademark of Patrick Volkerding..Solaris is a registered trademark of Sun Microsystems, Inc..SuSE is a registered trademark of SuSE AG..Microsoft, Windows, Windows 95, Windows 98, Windows ME, Windows NT,.Windows 2000, and Windows XP are registered trademarks of Microsoft,.Inc..All other trademarks on this site are property of their respective owners..

.
... \ No newline at end of file diff --git a/contrib/format-pcapng/src/test/resources/pcapng/dhcp.pcapng b/contrib/format-pcapng/src/test/resources/pcapng/dhcp.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..530c64cee167ccbef4fe4f3d89f23e6dbe45040d GIT binary patch literal 1508 zcmd<$<>iuLU|{gI(UxKa(*L0VBnILuKGDx zj7_b8d=Q2h!@%9d_{WKZ!IgngXN4I9!!JScZjfq_EQ1S!GowVLJR>8>99vnCI0gXg z#}FN|0?AFm$)=2qw(N|MkTGHbdBvQCk(o^k7{E{f3R6&+R3L?|Nw=|S7?2Ob5WNf> zC2K#<28XQ&iz@>|gCGOLffWplK+MJ9%;3UkqGZg37PcTckQ%T>!!m4{7+FAR%L*8< z>>SoWlE=>&Nb(7ov4A{|2xMqjBU7O8Mo-goW}Dzk)8^nb-32lQk*4SGg{EmaWYgj7 zVHxhsNNF0B${@jwNZHuZ^tCxA_|mi+wlsbJHZ)Dc!xm)Q@D5v6T&Wq7)(NI*0H?ct A?f?J) literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/pcapng/dhcp_big_endian.pcapng b/contrib/format-pcapng/src/test/resources/pcapng/dhcp_big_endian.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..464a206ce4f4623f314734cbede6df6de571a72b GIT binary patch literal 1596 zcmd<$<>g{vU`UYCw((_PWMKFY1q@6KoQ?$rIjIW1P60p#49pBX{=vEt3Wf%HKxE3m z!oXjUoLCTFUX)pqTBMg#n#cgu&t8&RTw-8g2+{!p2|x@qmB9o|0vSMaC5kh1aw-*4 zOEOZ6@={9_GV_22q$MV&f~0}K1c=#y+G4;wIKgNI6tjXjkAb_1@sAS+gDV50&I&UI zhF^l>-5`Y^Sq2vdXGV!gc}7N{Aq=*%a08GTVExDfLz|u46r60z$Y{&X2niV@7Lb3; zSs0nwwEiP`4IHMRu&qE61Tq=Jz!(xL93^W%&IX6A2a78MLxUg#!+{kHj6lrA;LPB{ zXrg4ygci0SIglE#slzgCnHX6>Y0C;2ue zZ4OS;T_DYfG(CSWG)>E4v4Q~-^H^nuq#z_sGb5#GP%482H*(4*IZeA^OVjsnL(?=o WY(cic18!KRX;xgR8IslsrfC2*0*U7U literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/pcapng/encapsulations.pcapng b/contrib/format-pcapng/src/test/resources/pcapng/encapsulations.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..dee9737df62c2313c8b1af397d5d1d7592edee3b GIT binary patch literal 896 zcmZuw&1w@-7(I7pGBdetghCfZ6tWazup1SjLJ&$ow@x9esI3~-S0b`d+r^^)@t?vfY%$1`-bbh4E3~s zMD$%B2(W~ML3h9TZk)f*KCoNNJ@4~1ZyDom*3T#1VRMu_qIATom+O2ky$Dn4v31?| zykcI(sJ}mI_J`AFA~F2@G3y&&_X=ObaX#toPI{T6LLBc+YoTe;o9s1*-7NDwX~sl{ zP+ZkwoHpK^y<{zYLS%a8JqMh_ax0*T~>3t{k3SRFcYr!d1T~3dVVX3+L z?Qe$@JV&c;YQI#LkuL13jdBn>0Z+SG|FmVGxO2O_^Va4z3G(Cb8L{@h)>#YbtJ2%T qx8&%*q4$#BTSA}0>usg$gYHbwXZB;HRJ}#|KwDp0HQnlc11r(KPxmCH}cV=d1_DO==-|Zi+E<3OD z&3DfE-rsjl^78ZZCfIDYj_T>-kAmNW`UkJ!`N{h4!GqfI7&!kieDO2;osqVIwGD%{ zaE#w?4-(o2+Z|(Tvf11x*3GhAK4#tSb>H5$x9-)Qf5~^)ycG|8dlEcgi;l2YJbS#& zHe%H1-yC(+s1X|6RQSz(=O*B57-?_O;M;7oY|NO?9S9rWVe{rY#E%gcCNp8UL4Nl~jv ze}BJ=Q^TChv4SfgMe_5y6;5{6g*XxJc1TNjL139umzxyM<3)g_%plz}h<;4wbdge* zX9nR!XSZ(#k^8DVuCi&Qg$wjbOp)7N;`Vro^YiKg0WJ#gRBk|J1|oukjYb7Nz^c3? zrk^9C)A1?#IG)W|T;q0CYUfdbrZ_kaUTNZiCNUrdc`;N)LQD9lBgpj#tjZ;BH-uSP;h;ZlzD{RF zMQNqeTFhQ=56QpPariD&qeqMea&QDdNbjCs zrqKJEiU$sU;Vn3gEI$M3&!ET6qw!3m$KI+72iLFCPO^Ko(?4ozK@4TNE=+F3<) zcEJ4f@LUjD$2bQxgWIy+x)@u-8))mME*7TIUdg&Wth1!7vdme|6_=Ly-EMngKIZ~G za7q>N$5l?Dlcd5pJKH%iq=u`Ar@~zhb7!4Qm5*>zOs(;hI!J^asPUvuuKBPDuPYvf z-kL;B0va9Fm}G?0T0IkJpFo!tAd?!q$K|#IooWPGWAE;2c2?MD*5*~8(b(D0wZPj1 zG!qf>2=R8;x3@JAyOUw&6*n+UV^<@Y-`dsDP5=b)sWKl>8K!BD9iFwrO=bWadYlTq z;|i)lJ3z&21P~CF8oPxU`-@$Y9AZ3`m6cjMJ3dq;z{u6uInhoMN3{UHI?S;_Jb*)$ zR|T%NNtZEIq*0eg)r@vWdrDE`0!LIBp&C0Kp@5>;NrVgXY>k~21VHWBS$rfa$tsD- zLZLkpVc-$QkX_DVr`v^Xrp?d*G87_oh%s=f_DjLIj#QBEBLRU`pp&W;1>hR0tRIl* z=Ywh(#@+3nVkZGk5ELLh5C%1Nm%<2Zw!^HOLzD zAn5~k55b&9_+U`r?1@)l@8EHkgk`R$#?IJltNBRCz)+AvlAX}es<9gbNy1VDASiMG zz(p8M5`u^bNzSqorz*3e;sP0HC#;|XO2(_oO6(-e@u4tWgyY(3Xc>ZkSqq|!s?VK) zwvM&H^{DpLKA7!r+KUns4dY&ve6N(yDXFpdhk2EA;-~^Uh*H1IMrYV-O}Yq|nhk+U zG9x2nAGC49fu=L3!-rW2@uHJH0o?134$SzD6Nv+XpXwe-ju4TJ0F^2nD+j_zqiR?R z*4U$XeNXVCk^86c2MxQGsNOM>DE zvDr{&r|N_+%N8bdLaHrcD?G88+9;nS+Y$%A7~9^n4{a1Vt4=_;IdsBpR42HTbwbZR zL)SceuFYoj6#8b6zcw;d&Dm>xbeO9sg&vDc!oUz%#zZ5K#1K^@xg^epGGG=2crhAN zEkX#`i(dj$?J2VBlim;Pt{~MRCf3Kpz|klMHQESBVhtesXB^OpO|kDcda1IbRkp?f|%Wcn(NLz8~~{%CUnX)Iw` zND(X@2yKKHvWf{UAPJJZP&3$}1nNr(VNipc?v zfJeg-y**(82VPWEF!jOlAuxqTjKKjSBS>0V6}sQfi?IQwnF~rX>!59XlEBKmLIR*O z{2a840U${vARfRm13pXG%l(`m)Va!GGXTY_3W@NtEXj%s#svjixTMynq1FJXfdN~h zp^t8BRbE0;Tf8DevC|O`g5;KY(7G7Zu-A6r+wp~(=roKQy>+4`h%=E>HB2&rsfz(r zFu>W2U@%T#^rD}aITZpcT+js&8HEePWL}LkU}yJosx!bw)tJmBJSIRfH*6Cv^!&89 zFg{=8Bx=R|Sb6C*nx!Yn-`uQc9MrXDJfveoD z-6HQ6sq>oh<_=b1<4Tr+#eOO=z}lrifGOr@;1Gc%5arAsUf>j0G#s57lBHO5A#hYtX(<@Jlnk4(jRup_C(vkf5+wqX z5GzWX5=7S+_^gCz>I7?yXUmF3xn%bSD!X0DvU~AkRCbR$jeJmQv)N6g(nvc6u`J|g z0?0zy2O)4PIR#W7ppS7*dr-%#e{1aK#FH7ooHZyDs}o80Z7Vga`466jMmUc|OG3B9e_z zb7ck7(S$K&H#6Uhzs002p)kIium68!$3ZY^t`)D#{HQhcD0#2uDt=8xa5Pgu5xbt#N;p-8u45t<;|1 zlWx!VPe6N~{fE__zZg&8gLx=~y8w>yn-)8oePWx!bb&hA(l)fX!xb zEo01P7Q4)pk7?|kB$y2_S=ieIlZCs7q`ZdolA)`xByfkN$P5S>#e~`Ctm@2YpFu=e zTIWz|27n`^+6-`3);4?q-QCcf4Z;x3-5_2tqru|>r~bS61#&ckQJySBm?|#QNHIy% z=&-U-;t^0ZLwdo`^o$NFBmJcI8X#u!6?708NGKIRRxP80ID93e12}RzMhBQihlgeB zC$P)td76xtMpj7@PKT0(tV}4TfXl3p-n>-IrH{^``sm|yeRT2VsE;mK2dW@reChc| zrCd7UPO1}9>NzgH+|UU(jI`O3{Ny1CsKj=%08514c3}^UFts%1Af66ar*$#}o;~(fppkA<*f;X|=J;FA5>E+-ejKVfU z@Flh!ZySvs#pb7NV~xkhXs*YWsas}fH*HI7=XJbg`iFFgMEAVhl>Y~hI9O;`bi@WoY;Z-?sf(H1rP)I({0|eXC@Z9 zdVR~i!uVn!~dyeXK)D< zWQp>@se1M#RHLOv^m9yW)Mz;k{R-wtYP1B2eg)s97w57I=TN)wqjbA)=}xo@=Q)wzGUXjF&hO7<7v4qf z!s*Gef~7kRyKvGaXcwAs^C61Hp&NZE01wOVGht*@;HI?BYWkTFGAM6L`yAGv3E~JD zfF?kz%BAkOo~5*8$A^@a%vC60CMD~65$g4b)u)E|HRKN=rxc7kRmlm_sVJmq2&T}` z_fLs#gK&jRVgw4m28^)gft`^CYncBcpVqzv>YW!O->eBtlboN zzYKXycmpp6gcz0=l9qPf&$M*Sp6wO)O&pkgR&Tx*xKa_~ZBns!+$EYeFTt))@4 zV6zSi$T`vg^Am$Xtg80Q5K{?|P9K>M4Qa_>S{rz~rRqqxRC_>Exzg;ppr58qVa_o- zmyinf+Gb4td$klEO|WEw8DwP$Fk!8#;*xN-={Y8Opr(c%d-0y0+Cyv;oi+08PX z99QpaH0YI1m3}S93}qNeB$*xn=2wDr6rqZh(a0N)W-g9#Q1)yD+NVMlE>v;Tnr-?_ zCj2uJR11_ZNfPPe0$~xVbV48wDa!iwLs3>1ym2LqBxxB|U5w+TIa!dr@gn z4siy)&xG}%N4?ccG47IMWhJCr0pT8?XQYYfUMK8sx|L<4&wEO<)Wq z@0Bi383Z(Nw&!Wh$UJCA3QJt>paU1@fH-AqtCd)kli?=<^HS@9Iv{!gzw|oFAFC9~ zI6ZpbTD{KhVanKWG&TS|p-W$IB3!XvCLE0h;U6gHX;FI_6z6*N5{AdSXCu;?nUMlx zm0p-bVC>v*ELKtuV5$hU06|XZQC*^t8{iyDI=?0W=2Gl&(DPe(&lNJJN`2>=PrFBFGO- z6CEJSWVV;{Nj)kh4&#E>kKtnup`X}*6Dk5JSV}3J$OI&*mxHBJu!<2a>SB9s1HF>q z&qByqf$oE^0IjY9ehBR6r%_8w1ujq5S9XPf`b$n!IB&GFebq; zX=o>`ZlYF9V6TNbZK(Dh3J2hvqAM@}#herzlmX(p1+#8zA-x50IVeZ>HqB|8-!-?6 zj?S>B6qFQNea%=4OV=aNdDX5CaXw8U2Wr-^`Y=og^6a?2@g3hO6Y`?IB$Wi za7Bg9RzBv@HjIPPIIlNtfEo*@Uk1xt30S!2}+R$cSEt(PR=rp8!d9g~;pnw{yo z=9&W3H7!_Xl87Dc8I5OJ?DnIM%LZS9bNr^_JNn5LuOPk;{YOT8$1T~p@$C$2TfN2J zDiW2ztclY_eYz+_LeOH-5aETDt+-A=BM8tpY>fqZ*+RlUD^BPHKmzQ*Q&xPN)q-_qKH^6dMSY1pPt`2FTsqt`f7|~pA+fdis1S`dH z(K4>Pg;)qKtP!A3GT#UFe-1L>#MRMGsTbB4nlBfORkKCqMTc79TcWMe&Ci>3)iF5m zwh;$kp0`_hr=sBCrI)RECDL~NnA1nbmp$%@zxtbwC428Gf1d}WUT0QZzOPSlR-+$3Lot1mPx>PK=^x z4?ZtE$?-;4-;3NbpS&{CzP4ia1Ft#rMz4(Q->{%M+B6vR_@*5JhT3{7e2LZPbAX)_kq*jlGdGi!^=&pUcTk#*75PiYcTNwYy$# zKl)F~i1$W6vBAFeUH{5uosRlo-QvjNMV=+5ew28XsTw z{#h%|nDXo&pRMnI`2LG-U%+&YTlQu3nkO%7Sx|Y-xZQ8;n>k~BfBzRduUpXHbK41{ zZrDI-%YOUe0`9)~tNzrwaMtznKVH!B;+YRTJpPnF9Xq*daN|iYeAsy7PcQBK>*c;K zS^e?^N6Fov9CcZ1<-F5QIHT#7U9Y@(+SEJekE|)U##?#gR`&02PpsbiyVsl_biOI< zaYpt`+_*h_|MqD4_QKAYkFo`$s@{Lq*(-*5?wjjx&iicf(Ld@w`qYmKCf{<`XU|2I zjn{@&JpWAe-q>SZe_49ck6-aeUk<)d|5thR>W8mx-FDl!SrfKwKIOPSOkCe?yQypK zj$#CFH7>mS+o&Lg&sRqUdbYi}R5 zJG!a$&he+z{Qi-bHs8Lze96vDZ+HCgl-lJ1TV zZe8(~`?+t=f9HQHsw)os^ZtisR&1XXd-9tP{B~_I>lwzGqtw{`2*} ze6w@p>ff||8*hE?(>t!dHUEmQH=OdfrI)>ZWoP)sgI`X0{Nj-p{I+CLT+ILZ1v2wm z)eQNjf3!Zi=+>o=biu#R)jxOBrqX?$%SQP-JHHse z^_CkVBOmVc?voyDKC6A{q`XhQJUA-y>Q#kRku~nWhA;m`-G#Ss^^1P*C5?cpugNx#2$O8&H~ z$KG}37Vlfroi~2G{G2P}=U%X{??!p)>d2F;79IOlIPa7LZ=e6^oey5J%lQuX^!{1X zPQIXE$6N6Ki~HVO)jBxuzt}B{mppOW=bhU&z46uR`lnCMzv}eRy1K6iPW|DGk2+r4 zT(+Kh=BcADdT(UqxD&8f*x%H=3-&vQpXNi|d1o!^PBTAA4!C+3Z1NV^D;B^`6Q=I8 z#_kGBm*w))eog(f!el?~ngSzscf|pltv3Ygnvx=E%-p8B#*QW{dRsfLpGfC3zj0D)TaX9VvWqXri0j*)Urp zG;N*}r^9Cz8P=N;R>5AYyQjE(njgEP5yt3u9vw=hZ0v$XLFtE z5`i=cV<5y%ZVA;8ZH(J-saM(8Td;UGz7n%vB+B7nBU78c(2C0`ppgHz@<@(}v+ zQVe@6gpgi8UBKv%*I;g9D%}9X5Mfgxu2+M58bC(jecEFp(ZW1&2-!xo z%#cs7=%j(3;r9ww!fAZ3gGtN^R!89~Os)FV?P1&=5{p7Gg-!!0gw$Ltiu#@l4lO|C z%Z0tqTQ|31_}CkROuLX2oiYn5l2ipLtZbpwBYlu>2F)&)_)7Fhjt+pc8huzAl3x1IZ|VfAJx`AQNiNxa z50%|hlV$f!+YQ-0X1vWtr%XpH1~Eb{V3gR25ZB4Vh^q^Yno^D3I<(dJDCo308$prG zrKGqb)RLn#HmA|H7ijAtaj1Ohh0Uah)C2PtC$PySdLf9kVT>5^Q~tOWDGNcTo2}`&U^<{$)sdoajbMc~JjP!F zhPe~=k93s)o#6>W$6RA#$c6_G4Sj%D`$vtIXvA z1if%V;h}6e{9Z#)-KmI?D8PjjKEQ>rI<4zM*Z?-6Y2jEeCxaL=B&ndJt1Bhdf0iCGkJ|HpPPgYbKp=U_=CU`k+4Ikzy(*VZxR>gLDakrv zL$#q3Dg{`FYs?0WP|9Xm)-m%oHXB=C;-~>Nm|_ukriJW$ON@t&W0FQt%N?eSAl;b@ zwxH78(F}x6bNq-G-BQlF2h)%UB=eR}MtvzH*(s7xO&UU-BZC}=oePz0C>;t039MiDF1Nkij;kfL(cdXd^7=uS053jdDYn(Lt=Wsne5J-?^SDs!t znzLWY^`?5Cv}9;*%#ech=gfqx3pV_vdm80#t3RZ_P4l91Vu_C^q)U=vqfyxNDG$RY zpr~ji8Z}(eEzCrZU>Nie9}D}}!PLc;5Qr*h1f;*)bRcuS-^=1&vc2RyNx*DB)JFmi zx;dS*VXA**ycX@^Lo|dZ>RMnDjS>Kdxp9sp3~*XGmdeS z5v{h`xH8{MP13AGXTCj;a%~;xLt2Dxvb=74L2tBOs9+|*zJqawvKLL#@iNSf62-y? zsNROP6tEMPAvM!Q6xxs+?xvVf#!6PMA-1!h7jD;qKc|CwLZHbpqy7@Pq4} zCdg}Y~!Q?!Op5y;MXa0T8{QI2w_c`=AuWs3heg*S>vR`rcZAR|6<~r~zz`W7jN7M5o ztRD)OP0s<`X0XGo{y}PB2LflZk|VT}@xJbL!V)kD6X6mn3@*9!P-Zi@&;d!7wK_(H z!A;|-TM9MzAy`XBkExHWhQTPn?J2S3m=vPHR_(8a_)s)2bUOv>9zRBDq!AIYZAVkaCI)a zx`5gy+tY26!B%gdU=;r=lr+_5kYQZ~5n`%X&0O&G literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/pcapng/many_interfaces.pcapng b/contrib/format-pcapng/src/test/resources/pcapng/many_interfaces.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..960e35d913679a2c04c68f89c954c76258eabfc2 GIT binary patch literal 20888 zcmeHv2Ur!!wr+27Qpuo*Bqhg9mYfmENs?qF2MJ0>Ns5vrDWZ~*Bp{MeK#-(>B8VUc z6afJdiAqpj6~O>AbMAfj&3xzG@6^n)x_Wo_zy7sWt*Y+ohG5q&0w)AP)WkXVp+LR1 zSuhbUM9spAUPFi8fS!++8~iW8NpI<8?_^ESC!io8M9(U3;eOiQ1+=&YxOv&YxmXA} zqG{pb;c4gY>Sb$3?`h-VNzcSb4}L-*2E4xKW(|+VKrj&-7hVJ({DK4a6Cz{?V&@7@ z;uA%4^9hM?^KtX>iy}n-7rKDU{lN164HFx&IBo633->7x3=iZ|9vc#rL=Qb_?rIY|Qs}+jtBA zXa7O}?OY)y5p?T%pT$Dl&~ATQEN$`o6Z@k73Hz>7-ONu;u76*Dg)ilE+-b3q zF=xy6QqYF7k&3fS&1VAQ@el$->9Q!r2DwW@H4r++5w^ znS8>6yu6?ZfAVm%u(I)h=b?E8(E?~eVNQA>eo;YwQC?y6)+RsykG+C|g2w|kxNPV2 z#^tX#nU_IM2{n)tkWEfsjQ{;OwE>?H0D*pL0Dj5^+fXoo{Quw`s6X`0uRQ92Z?^OJ zclXE^511ohf&P#hwQtu4+fWjSC9#Eie(5*LgJ{s3NK6tpV+_@(iwGhDShjPRU`_(3 zC7@t}sRoJ<6HGfW!Sn;y-i6D;+S(nd5yR^ii-3@F** zkiFK9%}lJ@VIBKdOt`%hcgGP&)VXBj<+ZV7v^8a5SOAM^ok6G?xS>{{xSc_e{qzX_ zVPI_V1JrlBz4psE{zc3lu!Hv6YrbW#hpaGWFO%zs{cf-Se#|o2I{t{+ZT4bCRH3~D zNNiMLT&~B>ddyB}97N%Om|ZVQN}4(zAU025`gX5vD@I|hT-WTsUe_0cK@Ikk?VMQv z_EtaLhK(S~o4#uM=_|lmfb^1nSXjufG4fyI?768fjFDUcmv&vyhO!Np6;XWDf*@=p z5!kmHJMKZwsM?Np1U(c;JdLvHufDGA%fMNax9iTBY5nupm4RYDEIwDl7^Zb}j|QR{ zxY-oCgvBA&cCM8pCx39=(I1p;1)zVQ#BcexnF{)s$H?{8Z~o=S;6sDp1C{X3sQj6e zAU-URD;|~mUvTYkhg{Da{?64kmH0C%%gDOAX7mZub>Sl=$;*lQJSWfOPT?O>eiYvr zhDny@rT9s;NyeDtkGMVtm|K3`$u+?Yc$OqD4swOK9qp)EMBqXy(OhCoVu4`e#JNB# zQaR5DYmaW;Fqh12njU=3I{GsC=&j((g|Y?*IETkW;(v1eweQzQd;GWhKCrUaw3X|v zvoHt3fg9$WzqIjI4kU;~_V4!caJIML<8twIv4P_It6s3> zqew^*6c`u9g&ytpU>nM|K7b&`n|-ewU~Y~V098p3%@iL7Z3Tcqs{BjLEen7H@o^o2 zX)GLV;8{?I?R}%H&Eqe|ir{|8zieuYe+LfWFVwE_5&e@tCZC8XCYo0OjMhMe`FHjX z?@iomPzg|=4J1*X?Y>|e$_^ViZQ3B~rwstIK?3ea^m@Fk+u&d0ZrTAwl8I0foV4Tu zQ;8EmS!_NVK3-l-a3>D1p-#{yaO{Q!>UM=Vv|R&iL)j4%GKtVbBo>}qHc55@HsQkb zO&j=W6Fe&?dm9%|4dP@4gCZuwjM&& za&&QDfc7fESX)zg5AGPSu**=GnAj*39>G!eOAPQgto3ZPjlO|$Y5x@3TQ#u z<|kG}T>2e?2;%4*SU2bT*wuPSPB(-XU|vik&EdQNrbPfR2@3;{;QcW_9f$|8aW|gH zL2lOTNE_i9Li9I?Sk3}S6u8)De4v)jOM zei_=X1lmxxV_=9zYHUWd+A;D*c$^Q%E(FBMn)z-^VmJCpj@0L2wtLW(^-sqMx%)^ zHgGiXAU8V5JIFdH8VDN55zz+x1~P;nbXY-z2i*Rx0jNuW5H&)HAY@zpjeDcNal?Ut z3jE51l#P#Q?3;Jy^)Wq^cY>2F@y=ZG!0Sp~kMkF^=Av+lQTIj!-X3ZboVqM{_W)M9 zCk|Cl`n+O`)Ux3~+KS@6LMi@0Yk9#;!uos^Vab7Nhd{2})sRyWRna*jyGO&b=6I;0 zOD`~Hz4*d%=;XO?6&B?L5t+`nx%UN{x80&Vdk-6<1Ix0}BTvmMcAvHS>*`CrMl`!z z)D*_=-#+q~PQ;jsH%8ei?&7RROqF7j?%UULLN5mjT*i@IfE76o3I~KHHYOGd3lkHB z3I+-T6Le?n_0+)+vll54VuTi*SY7O)95LV*~9)BdIc<7;~?yGe)rh~*vqMnHOMz2DGm>o=qhXk6S#yN zTpopjAgo(%AWnN5)*h4@Y>QjG@5NpIsMGW;HbSL-ixLj^JBx{wJt5cac1-}BWx;+Z z+xssfFyIUOuPdBK{F488V?mB+0y%D9E(9q7gG_(QK_kUJG{1uwV~EGVrj?4%{8 z+=6U1TzyjI`8B62uo~YRGvc%aVIM+KhmPK(qkh{(5`z0fuY`>g*XIsFKrc_whO$Gi z6T?+dFB^qzdVSr}YqoZ$UR;}3$M}4A+3!1J6bT6pq5GiCbcGpbBv#7Uxf-K;mQ<`S zMfukjO;tFcF3?tRt_KVB;TfP49LJ#S(8(0@4Ae;>{CA!B*tc||O53T^A#f#FFfYxA z*BP^Gq)I+TpX43R zA*GkE?W|hSl6yM0*=wz@c=pdcCFO?C)`}Fxk*%2zVmR~6xI-LqGFts;=*`-=@u13K zo~GG-)JLz@_v800={-9znLU3_q=h0oCxs{gXHq|$P7No9!6E+6uIaUhSwz|K$pwzf zxIWJbKhF4M9czigtPOWJvsM;M=RSQnR!6_AWnzp_UVV>}skA|L-B5F<`2*+q+s0xR zlNY)6EUHe(L=7D$PJd}T#??;WbNvR_q;IWU5foB{iRlvddY<~s zIbJ@dyZkbHM>VJD987*$X$KxnEfr0^mXfe$F1yEmcjuBujp%+ zG30$qZ#geh5va$v-_4xkCkiLKQFKDb{)1b@Wt_NB@2P=-=SZ-UBj&f-L!Dwt*KLJ` z^}m)Le~>u%dcrXe@A&;Wcdv1sbQ!O4!-d#(*`f+o-^!dAj#HdMg<0SD2!{tk@m)lT zeJ%2eJ?8E@eoi0swrQ18b&0qqOn={6##VE@xnZWfvQ|M_dWP3HrdSwNi6gr2)B{a- zxvQ_p-68^{-Dg>o2F^Y2F7Z~x{B$Bs`>c3*PQ!U6+*9uy-euf?Iy^salpaKOHK9F9OoLb0{q6IResynrY1Rkic*&@mFW*fK?tXYLPf8Pv zd;NHCI`u3=eegHID{pQLli#hYJe_sq9p&n-eLF6h@##ya2o=5KNJ(UE-S_!vy@psye zHrZbhx}x*A4>_LiovC=cML-r+>rL1XRHL90f zbx+C3C!R~Fn$XfEw`{Ak>6dym(KA_Nl5_n_v2qosZpCd2{k}f;odU3i915XliZ7rzZ%i<7%0Eg!8rZc|v3$}f z>==U0-BSHb)R(|-gouso0H#$i1?^<$X(Jj7=W{eS$rcp5R`ROw`-?KOms<_|RGypJ zjPB3KN1H}H-pF^`$7=o6pJVlo`H6t^>iQgrXxsdxiK9#K1DFHZ=@o6{Jup^9f;o`h z&$0Tqduk;7&+e&N5hv+y>vN!9JbG5i3oSPE`nvU0|`tug5QCKEUw z$^%{OKpV;qU5upH)^!>FU6;^9Te_fL|Ij6R%T5fNSI4|Zd^1ZufVE6TFH>~T`i^gr z#Cz`5$>7t!R-i*Dh>NJ7G4bnt zZCmqUz9-<=qthX4kNBA9+{&Jq6tHI1>*s+OGWp#!-Mltf=-IM)Xe}zxvD+ z_0wnD=M9JvjS|G*YWhPAYGUI1^OmYfzA7j*d z)?-?Y;oz&dp*I#?ovzK#GB%U+h*!v-s8Y2Hi+}d?y$aR*eMUTg>qkR^rmq8@OfHSj zMXFyL87-FNIi^OvO5KGW6pwz_-FfudqP(} znNViNmCqp(dvtqWBG==brk@xun8|ieQN6>!P;Jg7^fSD7cP#E5M*r!skIXKVC8p+# z`?^xeuiw`Xou0gXi`n)gL)f0!9yxdHg5#>r6l41dM5lx0$D%IjWpvTKc$;feZ(4d= zX)_ayz_=6UtrHmJJg&fx0+b~tWO;XTJk=?H1zim*i zn-`XQ_xUvK+*;g42G+AoDd+9*pT*u@>t=R&!>mPdL-jyto7$D$A!gg+v>-!?v0HkF zDRQEhB#b?DuSe9TD`8upZ1IfcZx>oFp-f96ojWvLByTG;3U!No-8U#b9(OE5ijG`S zU^r#>RlP~|MvvEtHSbBe-ks{>VmEsfxdh_~=73-0S^&h89oPqDdpsdVGN|=j^!E37 z3K!jqC-$G?!d5)}jO%A7S2E9~8id4Bv~t;KzmX>U^eCv0w@viyR$Rw!Mjni7L7)eG z<^^Sk9+omdb8!FMzfBLbEj^Z~cf>Wqwt01&o692|OwW#I5roK4snG3;LzxKav}&Hg zRHe&2Vu6by=7oA}&y8+CA6c*;%8vdnP)Q2=dmQcW{at9coNl-%$SH`t#EnG%+Gy_!EoKta?5wI*z4-7>+_Wg=mXkNw(Em!`Bn=7&(AZmxAa+# zMb%=fA2_y4Sw%wcfoLb07B5;5MYte5a;nJehP0q7KPnA{V96<~YQQIYnlkc83g{L( z>PL_wSnD1|K7d+ZK^yr9YB@O_WFFM&8tQW7AZ*oiH07v4Eu$bKzaP|kGDqZL^45_% zA}*^$|H8RoG!Q^<;J2;C?w(9ZygnRWQ);2DO}qrtW$z zr>z?c+DEi?braxz1zmj+@Mu&{QSO*HsC7Us4Qj6clz&$X{K3QHGTXEce)^_m)lc;X z;t^}_3C@~{6GUcrz0>a$tdV3$$=Mjoo#po z!27%0XaQdE^p>C3kl|BnLqgY=$_tmZKef2x!R}=s$SVq;A|3$SANCSm=2T(pOsM!M zzSf9y{Y6mVBlFT*FXlruji{uxm_ZjncbmrSC(i=j|wsTVn<9IWJnBA|A?at5SG#$aim4V)CnG#0Dx;!Rso;i%O zUvxS;Q)b!c#d4UbWaerRBn>cu@}KfwumH@%aGZ17JQj`*mek>Yu}Jpq^6woVoSfBSu%?ALn^rof+L_l~)QfDG&UTw>e3UU(EomykHT z*VX3!F_)-^_clMrZg~F)3*=PxG2$=Q;$*-)8_ZXp{W4!+4#O~80K5$0THGR7ANyr3 z?g7Aw`17^64**LR;6m9xo*@E8)zE&QevYpj_A>-{i(oFYw+5{6ZQMh}g@Ez(!B0No zu#bWQ`M7+FXooNUL)ngjB68aUG5VFa`DgSafE4z>qHN7KejWd{rz6_s06Qq#$1+6F2;5vslL%YEvFui(k5DTYcn6Uo zIAQ%t{TbzDo~1EkAB>T` z`*hMyWxRK zjf$c~o<4W}7RNS97WQ3|K69F>dC%S#%=5$v5$+Gf^^H&T>E!NGkk4MaNLt`7GdYSK zTTRrKPUV~v=&0hFJ9_OSzE6+jZ5^RD7ZE*foa0xpIt!(_N*hS-qbBsuX=yXHe^YRy zqMF|Gj8cz&P&@b4tM20eioZ?EKSXmkB@kA#N`oooVHl8_kC|4w6t|O zjnT(ft=`?wJgaJgw72&bIh|bS7to@5izlx}RVU?c=3T*DXM^hSlK)0QNNU~VVW=}uieu%i zq6=Pe%Ve}c8a{Giqj0QSUnrX9p0klIc@%T81gKYUO5W z1-H~`>#Qg$onZ6(>94{HCLa^xa9G7ho8|G=8sw&Hubd-gI(PO#*GxB}#YUc*@~rdZ z8Xw*BM4jZ4nSEjHuS4@+#A9yZP%l~12~D#)l&=!(;fi_^q{b*=bo0`-Mz^!e}E z#Y|^7%`@v%E0q^IO+yvqF>i#KTS^hR2&F}K;#V$5 z(F-?8HA*Nv_a7$5rM#5QAowVgtz8!8pH`5(%U0~Alw^+Ti zkM^Bw+55(n$&aL%V=cs&>pQ=gYiiuCdNsO&n7|5e&Hy8Qoo^=5GmmvhTwKf81}f2Wq;%uMn!@a5QNaSUhf+%?`b%mE|s|ug?2T8y&}cDM6UK{n!PJ&fm%cuvy|8?Xfm2PnFKed9bE5Ggi5dd-Tg z)st*Ig`y@V;ZX=O7H_iuup!q0>)O{V4+p=5clX5IXFYTHIo;4(aPz=CDwVTZ3VChjpAEUTl*5mFsz6ZkB5ezOtSX8+&oZ5O>wHCk@3d6 zJ!W`e5!F_X`$uZ%R0>=P_Te_x>V~W)D~V}S+2bW&GD+OZ`M>7lyK{e;kA>Is@#Ww1 zac32R@g#Ac%lE*LWun~ccCrJ7(Vo{dXgDL5;z`K{=S&>L5@^0*OvxYAYAP3Vd@g5h zR6(Yhf85VRAY?dtsgTU^Hjapjs_PyM?GtOuUTH*g-nf^STN)kdOvwa^zZ5j6jN9Ou z-o(ve>S@4YYmgBP;xI@ZD^_0>Zp!elwKiByBRXSxwlmb+vzo)Ggr)3$`N$)oJICKX zb@hp+IH}Ue{<>Wu3O#iAgJyP987eMY-)0XByNu1i+SgZScl#!eOPul#zzgWBut?=lpso{g_3W8De|AxMcUO(xOnU zo`sepy|I>QUJXW+&xNm5Tvuy7=)Nn``c!k{`={Su;FNUu5{z`2?LE47b%umE;q8mo zU|iep#(9m47biNkOy(65aJgy^7ny{*jWyKF?Xy6o=vehPr=-yEsw*U=AP>_&S~srk z@2Nc6I_Vf8m=d-$ThxbLf|YAkD-iVlvCdLM-2$!RFRyq+Sum@gUI z5%ZC4meh@QB^~?KiLR!$j5{Uh`^l{&8B2lM0-Y+!SkuUe@Uft>T`R+?wJJK&#+2W# zBG-J6vbIDj^WJa4dxeAckigfu-RM^zzM^o?v!+N-g5gOleopv4W{J)|XMsMtm_5W* zxUUv*Eov9V>O6}q(T4`q#8cUB@Nv^i zGE@akSQmo!rghR$OL-{MKSMQk-n+J|^Twp4uLz%t)A8Y#`(&BAXK*rtrY{C5GPvJ} z?OZfz!QI=Be(-kci|FA={sqs>PbN&qqD#hm1iFPB1@~mEyqJ9;8eB)a|8ekKk(0=% zJ@?9#zLh!`mViH-{9zZlK<6pUu~*4-w;gK|W)q&ySh9ExpOW{GYUMKuGpzHoJ~+QH zgySU?`zB5ax9?Md5zS1j_BmqpqR>6Nmy_MurYPTdbXZ)myjE@CA>F}nZ^q}*%opK> z`}cGAV~O(^&}Ej<2rEc+$u0IyHtbef9xo?}7L*!mnKE1@zP0w$&unB+u{%dkx$H~+ zzRSlhw_Gq1e_nnf^kLkxiLsQWFOlA(hPufkQ@Hf*hvGe`nD6hwXVfI&E}u-&Sj5m6 zJ%7mIgZ|q$A8r#?oeK6++P7f!>G)Eq*HRI#)-!&3oDTh$Wf+eo5;5B2nk(E>&lppx znyG4~N3M=yViN`RxzTkyFja?|?iR(S^b@BL=RXoMWyZbQW_|b5O5v$wZJqOIFPb=M zr^MTPl<_sH;`uq;IVGu{G>_tlnX(u7^BZQWXT%hX*qTaGh8OsMOvY=a3@4=QzaptX zm@&xgS5itQH?KJP%!anW%SpEd(LI*_gWw0_kMQ#dL>3_?bF?xpGfa3M6bMbu&fb1rSaI8Mzw-cf59bN z&L><+@5OOnPxH~77I~`tnX!I4wAfOSY~TfaJG;7CpEVAJPpYeViLr+l7l?U!6b|H_ zq;5Tp*M>c^#{2l}!`o*SweRrn7r)17rzbsowU=}&AH#PJVS&$+wC5w*wZS%&9dmH` zWbiacnq)e5>vN=&9NUW_!n3(D>?GUoWKjBa>7{6!JNVOEqu z?DNg{I$fx*vZr(3FIA63;hU8-R8rL6#I@>+J(8)7cXacwj2o14)DQm+nAM7BtYqGoAB&4u|2t*krpS6|I>ba>;n)7mKc$Z*q; z-$+?GuX_Mj5?^Kyk&bo2+o#0DwG>RZT3#}pcX&SeVAAuJiDIp!{H#GX;o%+*hwP%K z&9j^w3-V=&2i)-wS`XEn&`B4U8f_zr@K+ww*EZ6-Af$g!-^dT46S1Tn^m&elQBdXb zf~vC0=$Q_?Jf&JP9@Cng8!Js-?XzJSPg2?L_86zSbfKEEwUbMndk=S8ofo~&Vp6$e zXFJoSw9*>k+d$G?jbaX;NV3L#jo&<$E8M;JysiE$nSz@;=O^!gS2=vuzPh$gh_!>J zc6W%V=N`#30|mONjmV&l=)-ius{Sp^)5%MSKFV|XDZ_=nn`AMw`t zTE(35RGp@CdAPDr)P~t|raW4NlQtAzgD_}W(wHGhI(;#VUns)Jq^$ID7b&b$*$yKq@j zIXlU)f57pZ=xpIwe^|)3#W-C$>i0>J{z1!knVB?6uZ_2=rnf#gr(}LErvJgfy~+6& zmdjVeSg>U84czM%8{?QEJE4<+#dH4+t?QHM8z$)(EKVQRo-UwhresGR@5c5qIGNS8 z{3bG6DZZffW>kKr9JdH%j!!RiFSf1GAy^gio5I&NLzM1TP-f9g1oUD&QDN4=pHc1G_qd|M9#@-WV$h39d>Es!%Bs^w#sbikEHJUkIp7#XZ1f`IfTX26;;!IlCE#rjim(7Ec zGwj1XLp+TsNs467v-Y==M-F)0d74{rkkslz%7Z&fBJYl1Yc$rZ9Qt7Y9L-7tCVwMu zMKoWYfM-c);Q444N? z(6GQU;48?-y`T+cdp^c!!7PONIQX}Gj8V#G@FO2%WAI<~pu$YZN-uo)R$}VSkv(6o zn&hip)KaFnW$g7dWHN5LyIqi4mD;Sea6i2-?W;M;C%d00-Oxx1;drl5_F?{0JUXHU zPf9q>#_N1(#R4@x36TjAD|-XQORwWEbB!O^s`ydSbc! zFUPIA_AEIn1a=;!rjZ|LwJNobCmf;SW|huG;fM*uWE)+KJ)po!O-b)2k|4hDmi^kX z_BWq9ted=Kq6iY_^n0!M=R6j9Fx<&9bU9q4j8;CR9%W0vH%csE)g5zEpoO)xW})Kh z?guOjR{h0aUaUmV=)W&u` z{oGZeIN83;CcHXN;;nA3D-vsTQ-0hd;!ys}z`6%aj=Rr!z9^iY=iEI#Nh^=*JM)bEeoX?@)Q%sZ=SKf*YrEPSu)4G|r3A$D?t_aZQiMwJ^0zEEOIT z$-y_nv`)e!FdX(_VU%jpOTmA!w2Mc$-~9DnuBkNJW&5j1%}I9;4D8v{SmD%~7-c!4 z#`9{gUypd3@kS2 zPsyCc?)|D(E+1SSOTod7*qUIEwu>aay(xY_^Q}-@aQRTRwqnyIC!JfF$_x=wt@b^6 zSd_RF2;s4F#>V7*-!G2D5DR45!H|7s*gX&Io_U#4Ztvb zCT3|WE7(1YCtsFYds>*@L$Iap4!5*g#r{V@k%~+MdSrQRPKBRwiND>59mJlpu{x4w z{9u>G2`fH6^SNc>JjO(Uf(goIapH8rcWFv#dZZr=MEAFsiJFgQYRy{Jf3Qj|mfx$g zt59vXn@F_pi@8*O=DMe>7b94lPposp0$lomppd-?yo26cKj3!y^=r4y-{Du?KR)+df>l4B2qqaUdMuG3s;9tkFba%C|wz2?!j|}{A9H1vGkPld@ zUjHTqwqd>!n2K&kA^Q|&b>OX)5{RXkL{sAnQOCXm1KY#^Z3RuSq{RgmMZ-Bfa{sAoTKEXOx^gn<_u!$A_ OKg5FZ49g$&<^Kb8n%-0Z literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/pcapng/metadata.pcapng b/contrib/format-pcapng/src/test/resources/pcapng/metadata.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..13c54efebe5f5c61400f2701b518004f2cce0402 GIT binary patch literal 1212 zcma)+zi-n(6vv-^wl8+;R+W&DDui6bfE0uZLQEYpz!-^vA*+du652YUaa6Ifupnms z1elPzvGz~kH)3ZfKLTQ54d1Wr*EW`rIO&_ad+*)n-aB6|isR_Ih;+Bw>kj&P-5gn! zemX9uds)&S?e1oI0j~&S8JE=kd@kaOCs{Grgz4CR3F8b>e=*HBL#zxLghsz|9$;L> z8At`sYedoxvs{FRQ04b!e<#hW9V#DV zv}R?X)~l@L34ANkk|Tpb-547{PwV+C4&PKZ*a`~IuL7nHF$pZ(Tn>%ra?6_a*&Z@zS=icM%6m5zYVbpt?vrP6q+MJRi7ZA z2G0f0HQ(%8<&i_)O^C;+ybBmpBsCZY3}e3Acgo;eWq24j8FM2W4TcAXXMWiCO~xIp z_fIi+S*R9-=A5ps!SuoO&CGsKCRxm@SNEFN4a}*3wFWl;H!wf#N98JyJ-_!ee#k#x zy8m^@ENIsJvY#p%?^OA;2b!9f@U9q{U?6C8**eydPL{ZoR$L<%QDsgEokn rCFMiFf0!%gZl-kDPp%?H@$9hIK7g&fij6Y}k#%T(+s}Fii0k|XO(>MS literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/todo/dhcp.pcapng b/contrib/format-pcapng/src/test/resources/todo/dhcp.pcapng deleted file mode 100644 index d7d2e3388d6f546577cfb88802624895982bf367..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1733 zcmds0%}&BV5T2nG>S~OhBu0dI6i>E=1m(hCyvT)w#Kaq~;K7ISRfuolYxoL&v$JhU zTTqU4vp=&lli8i`%V&(aCZd<8?iJ(w`C6Aw$&q-+oti{+3~wDi867OK-y{)3b38II z!gv6$tYUdOfse690ZobTVfAd@#U;$F5-xI|(3qRAZDx6E*phf*q!kO1gLSOX`(hO{ z4;!2z_TWoHeV@Jhode5uC>#hrA2}`}B!%%3URhZk3f>XHv=lu1Qzamwi-aCbGBl diff --git a/contrib/format-pcapng/src/test/resources/todo/dhcp_big_endian.pcapng b/contrib/format-pcapng/src/test/resources/todo/dhcp_big_endian.pcapng deleted file mode 100644 index 894b361911fefd59e86c9aacd5d7040021531b11..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1757 zcmds0!A`Pk*dAwm`W(U%Hu7%zae&Y6;#?1novNSfQ+~csZV?k)HFNXi2CcUXgU9P?y zF=H|6)TExdv}2~SvWWx?43aD6PMJMp7e=2}z{S$3Bg>u8=pcGbB{8eV_!!w|6P~AV zVi`6PJOq6!KuSG{Jkmd+F7^2Gq*0ioKdrl2ejxJOG*4?Wq%ssZutybE+EAe!USr&P zYGABvH(k{bxA@>jpp0?iDjL6wVI*`Z1)f_qO>3EH?8nYHhq=4*KSXYM6_crA#W~|x wHYoB^{kKuRf#TW#A1*i0{bB>z#hr;%igZg#^BT*svB)&D<}!3EBRPPle}(2c)&Kwi diff --git a/contrib/format-pcapng/src/test/resources/todo/dhcp_little_endian.pcapng b/contrib/format-pcapng/src/test/resources/todo/dhcp_little_endian.pcapng deleted file mode 100644 index 3378440a19a9f200593028ac315f0be8119549c1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1757 zcmds0!Ab)$5S?jbyF@8kge_uK#FKimwS{UA+E%>ir633*%3eGadT0;c{4(*A{Dr^IPBJ?)nI!M=wATAX^l*K)$5?-VzB50mk^HV(iInjb{!dJ-LSHBujidL` z$9OVDOorr2uou~kf#$EU8-GZ(gN&Guc!xIOGVC^>0r5G!mZ{qb;ks_{>_>g-aqan# zSrC(dF!!(xw74I zSp%G5WXU81WeMjEeQV*LMAKd;8hc?UoP*2@<(RQzN;nn`(g{`bGFm1e>>Ad?^#ZzE gEg*Zi6OpGPzLKXV$8>D063x800E)*W!w~_^0Ji5k)&Kwi diff --git a/contrib/format-pcapng/src/test/resources/todo/many_interfaces.pcapng b/contrib/format-pcapng/src/test/resources/todo/many_interfaces.pcapng deleted file mode 100644 index 6a8b3973d1d1cfe53b6fb1c980d7a4a3335d8317..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30743 zcmeHwhj(39neVx}>e8}$x4Nxfue#N)x~$$UtG8?QVij9m1PBa6fDi~wfT0cTG4zr^ zV8U7naT2GHOajCSN!EJIdOVo5X5M4|f%p5JZ|mGE>&kX!k_JS;|YP2PidKSDfv+XlBi}l%si}#2xuQT9BsAEU&G{<(62oatn*I&_G^^ z=KJr^7R?!bZ64{Jty?x_7?C+3EznTA|j zTd}OXZN79kz7yhIKdlDVl>uR(cQ;QQ%{lLg~8o)=!ki7@= z)ao3&Fk2@JHOvrFp<=i>5nO7aVG^Ye@IGU5`h*jCqO zmzR(|n41w-JdCWV*|n90@vWutMH{jc5|FjByvi$ciwbgbkYv%?%J`&hjn`Rn3M~1S zf})JL!n~4#ypo(E%b}Q;XXF+X6b$bjj^3#qmfpM_XcuJ#W88K-P;#)s;9ss4ypMRp zHw<4A5-kn3bcIJX-H$BYA$Dc6j{lNpC7kJ+Wn@u{g**YVYraRzbJ zqBdM@yR426wX&iSYBhi~5#n*Jkg2sSC2@77K^CAQ3^~3)Z3q8~Qw|bcL{xVp}M2%C~WRYIWAOu$I|c-9ep@N$nlIK-#D7 z{U{+qJZ^DKYwB7SFF<*pT&2`TLLWgqynw^pQl)lJuoM)S9q_m>9C`w-ga-Vm{pZ~L zQ=djk@V}3J2nTWtL9N~yLA{dBleFNLuJ5C)=%LYM0PMI2g3sJ zL*3u}DomWFMhQTo2v{V|@O;XP&GjYc4v#wc?-1ldm?d6|>HEzdefB_il03*U)e`vpS zD_O0|c;t2WH1AvDvr_9I7nL2J{T!_+7hVQ)u&A@B&^VnhUithdR4QTQh3-2l5a9Y>(A zbJX$uxBEYKKRCAx!2V=l&ANzBe;s_RLJ|f|f%LlvnvFDs>i!t|%pc$P3A@pe#xVF% z&Wnt-w=xWe>0x-1#uvcy;%Sn>>Xwdy+G=~Fhij)#ZLBTMj^}1Bt*>O}$8hCxAB#g1sjUmS&kk#S&mMITma*fgQGcf{Q41GhH9LyULe3e zS%(`soZs@wm+4A&=QrivDjXj=5H6uX@Nt*}WX>Xo2+_IwYuex)1UNR|&}*S)0^Gvb z+pV=jUHkFE%VQqq9-p`IIyUIk^-(};@?b%P=**f;$Ojk}l_cm*Hg#7=jCK@R>^3#$ z7MGYUIr%tf0*msFhZF|4;|I426W7Z&%7_q8vgyJd*wkI?M++X*3Fu>cWwEP? z?xcl!U!KUsdHE8C<(6y9&B-z2p_yo?#=%1-Kgv58!vO8bAwqO!^zEp+{lC=^y3auf8AEnvg?;=-i`Hv@$CSHLbwCJH|mL z$vL*&)RZ|7Y~kPo0P zA_=`CZNmK{Ld?gt2{oHsxUkTY+5CS#{__v=-0{rcG|kZkDu<$_2ZIjrU>{a)KU+mb z#4YvG5B8-_a6qb28AA5V^#?;GgHr-vs#`?Rs17XwjPsezQ3Ha3K|ro~UK6Zw*j~UO zEsu2&=(06$N}@L`8LowDTF_yr%aTx+#gjxBN)uJfxT2kNxR|pwau)nj2KzDK^@n;? z_>(N!Sh-z3L|(ze8ZpKdWZgaN#V>*L2+8L$7jJw~rI?YoKUK@BK0ttpJO((opD8(@ zR}2Y4@u~myfvpukt3`BV^wQ?m3a28{W2g7o}8z+ryWdsh3iw#gy2y)}^c^d@7hibv@CU=AYH?ynB z)ojM{axuA>5xU*BH4_^ZH-0$^fg;ClRTWqEkMr>d`6cHqj}U-b|YU{7t=s z`gM!~5H9R2=`pF)9tHGbeawtV!}jSBeQvE4E1SMjIGGuq6ATSa zChD1@TMk5>qEPcdI-lmbB=VJ?=Yd^~OSaL6cYx({AkTi+w{aaD7vAg>$W?RT;1y1y zv3?`B%~))|7hKV_-qcKK=FQ0;<+TS7l!m zK;nl!zTYbKuOdxE%GsD^kso!Zd!2)YF{-qAhcWSyAuJpQZ^@ww_~;Mx(s*L!mnSI~(F9T*VRE zl*irioH-v}CcDRnn*e~`&MhP$Aj)SI+y)%wS&3u&!v`OxqfCVH!9oca31Fm&u?sC6 z9})7^n0SyPBFEX7M$9PSpzM+Cq+#WzsYQV)bH?|p!)y!~`vqnKWfu3AH>lHP> zzw2kb2Ll8F$AIj1ZI4C*>|5CdO^`Wk1!X+i^IBv?FV{&%^Fms}5|Xv*ik*xbpGI;% z8U(QGHe?#~6glnHzlG2TeDx8KB%nH*Lz?rA1r<)_ ztJ<24ED;WbWebZ8x0icKRxcP&a{wAQL_|>EJ;n=KAhwlKJa-!aH-EtOulR}}bRNuO z3xsn3pm^chN0g_pj!rdlcg?zb4z!z%8sJmg=GQP{?Kd(+;PftvI9N(^WpcDFKn@)+ z%rOJ#HJQLT-~y1bV2T2ujk7t%?v^D3lyyTYm73+Vz3Y9P+>NafZ$=q`v-CKCT3fbI zb!Me=aUQtY0F+fF0Xo|Sk~(%Knxh9?4Txq2=LEbfZQABn9{_2R9FADv6qpMg zDK6_@STGS*?(A1|_qIrgtxi zyaLv-s%#m|5${z&I3b%T%O*_eYyCnS`b$WJPF1IIDEh! z@M;tk@Iuc-aTsZ3{4fA3;E!E_N)&c=(1AqF(Cq!%pqb`NOZpqSy!Rcl7lCA<{ zF(6-SKlC8Olp(qC8MRM;sriWZmpKa;G`Fm4KZQ`T zt%p(H)SwtM`wEE$xjGOvP7k6_QxP@8kN{pUe@6tN{W~<-DCh39w(t!L=N@8ofN4LpmME<>qgi=>?NR5QGbyQT^_>_oTa>3Jp~Nw|%Og zqAhU7c|T52L$ZrYh^j!|J15s;k5^ih^i+IYv4@`>+GR#VH!C>Ck;w}D2;B- zsH1{g+g=J1P4vhTAV;_jEC*%*0hb>CD_nB(IzfRNi^Do`a-X2(UtBwKpFsT_{L*+9 zz@?$zCV&I{{R3%Q8Hv6+Bwt@G_H}a8Q|miaNt#k>akzln>U9>C{QU#!aSLxerI1VC zVx6A++#PDf4bk7dwWfowMXNQC;N#yZ-2XN<7Gk`NO%7dO8uvUfe2BY}dPeR<8QVK?~ zc>_2BX>+xpm(@nffrqI**Qf@;^1Cm9dh{~rn6n~4`%~4PQWb4Q62P+nAPTr4VltRI z*qNUH_&_9Wp!SoYP^ltREB8~UF%=TjlnkPHv|8%W@GdSbkAK5w=c42YWyrrOV$u>@ z9T`iAeC`Xh2tbYNpTGa@q-iAMfxMRl)U48A!!3&;V5hw~vf4j<GH21Wuo>2$ry*x>qna4vk>j-J2v`u(7QK!&3$z|6GK zr|$u0G&p{I$X9W}A;g^!7qo#YeAjYE$E6a7$dHDvPlyfn2!}eCfd+7905n^sjps9~ znl?^v4v;PbKx8>SqyjZ0ew$20y_1GCQ3a#i(U3+RhJ*&gPa2Z`Gc&3s3JRtF3kn_d7W^@p;kQ6 z-_BNg70i*DA-HYsG_vqfgS`0RF=M_{C3- zdWZpQCSn+F7|?zZz( zgi=qllKSeplvGpGt)#a0wwho>g4XVinsD%{x~kgK;JsDnYk64et~p<81@8f`1@EYB zt!-p}S9N_0`1zXd7KYc3-s=TA*R*%^=)9(j!5=y2yLyyVQ`gg1 zjQ1*P>T52Pfp>#j!870e{H0pJ$!j8*Rkh{k#X$zN0Q-49v4mfg9HGHrJf0Z9b4@aE z57-itwZwFMB}m~aH@%mBTm|~*y&y}McR;xe^e0D<0Ylt4xkdn5!nMyo_a2eC9u#u-!EdE)+4=Z^b(SSR2X9#91KBO$jgJTyN_!=%0R+yLEz*pWv@f8ZS8$9keGa6YhyR}Jp%vCn00MZjc=od( zX~0roQW(8RpCFkg(04y~U;$VSyBT*p?%w?HZhAs_$?aa3q`z`MfA($&0Pa){J6#gf z0Pl)^whYNdGCHSXAmJeQOM-G z;-}PUl=>&Sm!mp(k5g(pR|xNOvOnq7%;%qgJYg|lyCRm>gZuQ%jq^aveQE)?91H?D z7wU?)yBUk;KgfK$A8%4 z5C+f+XU>iH?|@^6UKd93y)lq-sR6nax5{hYJ@)>k9uoii>GkfhS61J<$6iLMcaOc? ztpB@yFv05B>v*_S_kA@OH4&AZ980u<0X%TB1{6)oa#3eoB)0evTYs}ciB;> z`wzf*%Nx+YYYo~A`}OyEqBu|u=s&o*vmN+^qptqg2`v<;uMLjxy>=zt+NhNNdzR-P zhE$ae0*HL^nq zk1p!YqmNw)41(pgirfOm`S%6#2oK;rE?|$7O-r<|fpEA5Rd(YdX~0u=K~xVl3NJ}^ z4p_1>DMV}UD2F}6o8f3q%wtM`KN3#(aRPT?yb20I57&^SOQcc-&yOh9il$R*kY&>l5jy$4Q9 zai#^Hd6+W+DDe>94c5KFehv85!$Y)XS$qd*?KlWmQ8CaFeD{!w1@3VH(;D>9gpqR4 zva|tEfVznO0N`M$HJ~C+cp{h|Zyng9zvOWpl=TgADyW@zo%$XNav*S3jm8Z!!1u{7 zg|Yqp`6j!Mcx@r40@zLOSSs$#avz4^`s}x?ncx5}S`(1-*V#oi0Cg*2d}QMX4DF?c z9GW9UA~F5HKl2%|*FGQxRP(~F$U-Y(y_IemQD$UqCPg>8OVRn5iSv*auKei-`csq` zk6bAz$>H1y@X^Y=k+Ki|Ja6FGau;iC61OZ?K5dFp=2qEY^i;t1hHy~tB#X$G5-YF& zHN>5>AW4#^^jCLUl9!ga?d?9%Ii$berj|~3!#<58;0ZRn&gDoQY1gQDkm5P22Iv8* zFq8w4P2{n^#NU3s4~&L$ghJX#wf@p~7A{L!ekW(dv$mD2a&?q#vk@C7Z-X6&_yp1h zYZtq3{J?#3vx(iBX!iu03NV>&CU;f=)N>sJY*vt`KPHLmpH@ymyH>hurTyS+nqvdZ z`Ljnr-HLG4SQy=NOtN0PTR5nXd-|j#7!^5mwg*t&6sWvSGKFfLsYi^@(dW`PR97kt z($N7nVEFa#+3XE9Q(b|Agmwtaw5A71va93^4Hu?-K@w~xLyd#cgNqqJQ`G~_k{p|` z_P52yA<+~gIb2(`<1IivIz)}p0;#qGHkLVYRZ`V%+zkX6gE`z&vC+B#Y3y++x>7A1 zbZe8&1BhYw0$zp`M~7p$Dz5{fTL*iDf&$_Tp8mT4@URjUgJ_(rf+xNp#h}~36-Ew# z3X_Swhp)*-Mk9Aw3LpKf@IYl6e@O}*>!AG|fuXkWxkr>Y86}NZqabrD2b{b1wS)Em z`&4nYEcvQ+9iDfM?iDFjciCRGnghx5X>m`7)B#i$`}0@dukR$Q1)U302UJb$1jMO7 zedR8&YhMIe>~JZ>Q^DL3UCit3Je5J(vs^W_ATt`_i0EE zoj!LlGlsM26kfa!*(v8eI@(tny~~X%9B$3oL$|Ny<>84NYGLR3mloR=vil z`^_goTqddcFc84CGqo=AjUR=FToExrnBOSlmcZmZcj(wAkY(>kk4JyjV-z)%c7@HV znnRMS&aOy6;^)YNNQXx@w>mHu-RXhx*`H`G&)<*EGPgO^oUyl)1>+l6`s*{4` z&c}`f70}oMG@Z?J1eCQHNC2+f=&K42(pB8!Ij-ojFLO0csVp|ktDb%T7>AKg&60tR z!1X;W8OvOiA(z=(n&fbKGl-o7azu~)h#LZ+&*qn$R%Md}=xcYK+LlE5mEAqhz27`& z_ev@z5M2hIUk*~-k_1{f2C@ePWM9B{5zQii2H1`N#Ctuc!JtuQ2%U6Un#dy) zgf?Qv3ebSaaT=_966EpEf<)!C@dAK${Y#+AN~d3ZNcBw@DUcclb%W#zWGndxlHDy# zsO^k!)%9*hU~m)-AwVwxQ~;mhUQE9UT!{t+ux;gvPkvdlIN<^ea-x~+q%N0x6lr(C)YM3R_>*A-jQrjnu3lA=q- z<|6xLeDvdV(D%vc6@XG8FP^ZR1#P8kab&rfQ2}aqX_!M=;51uVx=4*(DVm@BiT)#+ zY+Fu3?|?<^CH-vEfJg6F?WC+p9^LQ84>=q&a3Mc6kv3_J90k41Rk;bg2n6g3GpO3r zkiPL%$z1Z7g}`XS%eM1oB)c4}9>~$A0F|5fK@>OuV~;+-EXj%&#@#-iF1WM`^6Q_0 z1U6Y*TA%(tq?=&wjlXNz9F|Ngn8(OVk4P%9#<6Nun4~!K2N5F2hF7jN#zCfUKg*WA zpZz>&#ddLS(}HR=vYHkFaFtQ#gZg-?WdIfRV2gHtL!yMsC=aOoVkq$f5(dyA3g-*_ zj0_P%L?PJ)^lt+J_|ZoIt8_&!2)Yg(mH`UTzeZ!L`Pz~^ zP1LhnIltjszJ)QIty4TKwY^^1a?M+*T3+!-bN!g5)$os?XL&Q ze*0st`|J0C3=LLJZ2R^?GT=NUkGNGXt+lWuP(QDPjsRs-E*BS)vs__z*Q85c0d6#~ zqe^oYLS{m2EZ9O&}YeUCq=Tw^e_Wwb@PJcV0sp84d=(>+~Nxu{i^{Z~^|=Aaxwe zWh#1=_HqLPa(;E!_3wk%bH(JH=J*n3dbuC~)LNbW_@jA#c=BUaKo^okh|Zo{)=I4- zU)C`=9e-M=aZ@Vsb$mI^?7_8?W?$ubU4^uf0UeizEyF={bpT_N+Sugh{kM3$iK-3< zRws(PO<4V@w0(U$*V?U(j($XI*X5SQ4rB_H1%VgR&$vV~J99y6=@Q(b;wuURPk{q2 zr4Nb#8l4PcJy)cB(L0Ihf@<%9!E^|X77zpKT{!%bc;ydVRn)Zls!>A(FBNu>3XvR- z$=}>d<-0taPlKt2nkt1i(GjE<*se_q*1$LtwO%1H3Ka9S^@qE^8Kgx+r~n$EdS}!>9)nIB-?%4{J zzZ_Wus6e9w*dbT!I>*E@=)W^By$lJ^3CszmExJvr#ml<`bUKgBI8cFC{+{t@Jff)G z51dZNwWU@nBmh@0hdDMHwa?{dKxGN-7|o(0c0W1w%r}if&hI7{X0)wgt12rh*eHR} z6XQ|gRpsZ~Z4ajU&s{=P@I@6$^algDSr0MpRd6M8+&&Bk=|?IdBZED&3Ls|H>kPZN zL}x0iluTc+T~0z@*8s<+FT}oq9WAzaf`g&2dUE3cFox00fB|54T5TxL`<~QpW+f#G z%z&!(K+fJ%A6t;DLZgH;k`X~|CxIov1`T-OrzQq*z!V+EOXMQ3 z2;ej)FsN>2H3NKe@3pVd@Y_9X#=q+Tl#8&q4de_EkQaD0Y>Zt4t^g-EziLR9sZAs@ z)n<}TJ~yimAJUMQ^4;^Bv>rgI)uu-M`-i?&VoPG^FV2Vt^=kK_0u_VtpiMwC5YWrP zIYJ~7S^A&a@@1sZZRtD1FBt{~<-36ba-%_-fod@_fE>L3UC?TE4a5SVK!pku7bQ6lRwCld zcMBKV6C3_dpSow4mIovyc5ssuXc6|y4>lZG&o@*tN|TMc$*%NWy@*z)6@v6t<4S)4 zL@z)8Q845;x2=A9yTBXwfUCKB;_G{onPL>@BwcK(Gg7`G(;~-e z973E?@ah8)=Y2t*y4zHfIxYo~TnK>tw!YJkeU*)>K~lG6djOVN`awy=j`b=CbUMJJ z_b{@}NaB>RfOgI^;J`CmO=|M5K3V$cPmtxO)M}F&pfjAZDrLyBrPFGZ^IUREKs6mT zTxls$?3(u%0&|aZDro}VG~5Kk*;LCLIL{~rNK0TNUejysN_)vp8Gq@+6I7t|@%uq7 zEx&wN3VguW;j=#k;WYYOP~HSN=ZTMUJ=f-3iUkWT99;o>`BNZ(=e|8C0_d1>ty##7 z!Rv#e0bp!cO=U1d0C!UT>#j3_YU%y*hYd94dAIgQsnS^>Td^FCo5(5dcMu7;bV@X{9uy7h&DW;+WJ%#ZI5K;)z$+g+*Vt(QM>ok!-tE!QQ zq}{ynM)^D>w`GuBAme9DdYtN}W^W5=fOm(Alx50hJ6%PcG;jUlw$xxHk^JZ&*FFu> zssBCVG|BD}CUm3=-2SiO_~;YgIvuwiEg7L;Ty_i+?UI!a z#wB5ZMI_J4A_xn=xrbU$s%{}F*XVfZ14x`nn;*~IfI$B}V9Wr}T@6SL5|Lp7&~AG1 z-b_xZ04EqQU$+n193G__pG1nDOLvLNO0j~O+)MMp00Y8e>Qcf(<5uYKWs<`wRW2wQ zQ(zl(aODGZ%&F5-#_I%BEJ1>K(Mko=bX)|IrF$T&oCo%<91xNKP07nAU;eIdz`6bdB08jo9WXuyJdqi@szX#m>n=eQzOIZV{ z*%&ojQUCnm&q^t16R^gvt{aj=8xx@13;{DhWddqbxg^(rsAI7jNed*Fod(id0grzQBnIf`_vnKQPjG*r zlC9Yw5j!BtRiAqZGC=>rlnYFNhV9A%8RVJ@tm8JtQB{|M{P}?oKNw86u-u^4IMe+v zD+|#c07e4kzmP)Gv$W|URm5QdRR53)pM0-mb=5!i@r7=Y8{@$_+SwQXhij-uh~~q| z9~?Xq`Te;RW<6)J3&A@AmAh0U0XUaq*$V;Rq@58V)vdIIQfILgC8#OXNi1byCVmcU z4H#6q(_F`+DC<$nBXQIn8Gr-enDRu3Es|TqEe5dX!dWhz7^yCh3{SfWuNK9v6<95fBP2V;;Nbn(5`1 z+Fx?|sUMZ_3hthhKs&3AXyO`aJ*7uKwbkiw2q0I*k>tfoGzskLYFOB$MQDeXQ?HL^ zEja#lUDRY9TnBjOAMAhmJWKz}=UMWX&rv5p)!|&s9R9~Pixc{fGEDgGk+iT?I6FRN z$y>7&>VHAqg8#jde?VO|W71+Z1{m%dNB{4DEcJf}=zsASz4;#YW;*;U_a^*7TIoaR z#Om_+)Fl3UN&fd`7zIBl0p&N2j=z6UMmK4vwVLIPCH#MsYjG>9%U5QnY*TR)TX9qJ ze_6U{p1QN1xg;H!57Ux#Me^`}O2Yq!!S0L?dpqnEFRSa#aQy<_f?f~1xmB+`xbW0s zeMfu6uO#(e)rR#Bjg{Wbv8qM=4)x0UWv^VD@7%mHma^BrceGdhR7USrI(_Fx%T>kE bzSI5Uw=(}J{W|=QcW>GJa&_?IEL!|O7*_ui From 2d2f3563992ecbd56c3e8338b62d9a03b61b7844 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 14:21:49 -0400 Subject: [PATCH 02/50] DRILL-8556: Add design for PCAP protocol decoders --- docs/dev/DevDocs.md | 4 + docs/dev/PcapProtocolDecoders.md | 284 +++++++++++++++++++++++++++++++ 2 files changed, 288 insertions(+) create mode 100644 docs/dev/PcapProtocolDecoders.md diff --git a/docs/dev/DevDocs.md b/docs/dev/DevDocs.md index eef8105b4d6..96f3ed94a46 100644 --- a/docs/dev/DevDocs.md +++ b/docs/dev/DevDocs.md @@ -27,3 +27,7 @@ For information about the Jetty 12 upgrade, known limitations, and developer gui ## Materialized Views For information about materialized view support, including SQL syntax, query rewriting, and metastore integration, see [MaterializedViews.md](MaterializedViews.md) + +## PCAP Protocol Decoders + +For the design of protocol decoding in the PCAP and PCAP-NG format plugin, and how to write a new decoder, see [PcapProtocolDecoders.md](PcapProtocolDecoders.md) diff --git a/docs/dev/PcapProtocolDecoders.md b/docs/dev/PcapProtocolDecoders.md new file mode 100644 index 00000000000..f377f256edf --- /dev/null +++ b/docs/dev/PcapProtocolDecoders.md @@ -0,0 +1,284 @@ +# PCAP Protocol Decoders + +Design for decoding application protocols (DNS, HTTP, SMTP, and so on) in the PCAP and PCAP-NG format +plugin (`contrib/format-pcapng`), and the guide for writing new decoders. Tracked under +[DRILL-8556](https://issues.apache.org/jira/browse/DRILL-8556). + +## Goals + +- Parse the payload of packets and TCP sessions whose protocol is known, and return the fields as Drill + data that can be queried directly. +- Make it easy to add decoders, including from outside Drill, through a small Java interface. +- A decoder only ever handles its own protocol. Traffic that merely uses the protocol's port is left alone. +- Errors never stop a file from being analyzed, and every error is visible in the query results. +- Queries that do not ask for decoded data pay nothing for it. + +## Columns + +Three columns are added to the packet schema of both readers and to the session schema +(`sessionizeTCPStreams`): + +Column | Type | Contents +-------|------|--------- +`parsed_protocol` | VARCHAR | Name of the decoder that handled the row, such as `dns`. Null if none did. +`parsed_data` | MAP | One sub-map per registered decoder. Only the sub-map named by `parsed_protocol` is filled. +`decode_error` | VARCHAR | Why the row could not be fully decoded. Null when there was no problem. + +```sql +SELECT parsed_data.dns.questions[0].name, parsed_data.dns.answers +FROM dfs.`capture.pcapng` +WHERE parsed_protocol = 'dns'; + +SELECT parsed_data.smtp.mail_from, parsed_data.smtp.rcpt_to +FROM table(dfs.`mail.pcapng` (type => 'pcapng', sessionizeTCPStreams => true)) +WHERE parsed_protocol = 'smtp'; + +SELECT decode_error, count(*) FROM dfs.`capture.pcapng` +WHERE decode_error IS NOT NULL GROUP BY decode_error; +``` + +Each protocol has its own sub-map so that field names cannot collide (HTTP, SIP and RTSP all have a +`method`) and so that fields keep real types: integers, timestamps, arrays and repeated maps. + +Decoding runs only when `parsed_protocol` or `parsed_data` is projected. + +## Interfaces + +Package `org.apache.drill.exec.store.pcap.protocol`, used by both readers. + +```java +public interface ProtocolDecoder { + /** Value of parsed_protocol and the name of this decoder's sub-map. Lower case, unique. */ + String protocol(); + + /** When several decoders accept the same data, higher priority is tried first. */ + default int priority() { return 0; } +} + +public interface PacketProtocolDecoder extends ProtocolDecoder { + /** Fields of parsed_data. for packet rows. */ + void defineSchema(SchemaBuilder fields); + + /** Cheap pre-check, typically transport and port. */ + boolean accepts(Packet packet); + + /** + * Fully validates and parses the payload. + * @return null if the payload is not this protocol; throw if it is this protocol but malformed + */ + T parse(Packet packet, byte[] payload, DecoderContext context); + + /** Writes a successful parse. Must not fail on a value returned by parse. */ + void write(T parsed, TupleWriter fields); +} + +public interface SessionProtocolDecoder extends ProtocolDecoder { + /** Fields of parsed_data. for session rows. */ + void defineSchema(SchemaBuilder fields); + + boolean accepts(TcpSession session); + + T parse(TcpStream fromClient, TcpStream fromServer, DecoderContext context); + + void write(T parsed, TupleWriter fields); +} +``` + +`parse` and `write` are separate so a decoder can never leave a half-written sub-map: everything is +validated before anything is written. + +A protocol that is useful both per packet and per session (HTTP) has one class of each kind, sharing +its parsing code. Within a query the mode is fixed, so `parsed_data` holds the packet decoders' sub-maps +in packet mode and the session decoders' sub-maps in session mode. + +`DecoderContext` gives decoders the format options they need, currently `exposeCredentials`. + +### Registration + +`ProtocolDecoders` loads every `PacketProtocolDecoder` and `SessionProtocolDecoder` once through +`java.util.ServiceLoader`. Built-in decoders are listed in +`META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder` (and the +`SessionProtocolDecoder` equivalent) in the plugin jar. A third-party decoder is a jar on Drill's +classpath with its own `META-INF/services` entries; no change to Drill is needed. + +If two decoders of the same kind declare the same `protocol()` name, the one loaded first is kept and +the other is ignored with a logged warning naming both classes. + +### Matching + +For each packet or session, the registry tries the decoders whose `accepts` returns true, highest +priority first: + +- `parse` returns a value: that decoder handles the row. Stop. +- `parse` returns null: not this protocol. Try the next decoder. +- `parse` throws: the decoder recognized its protocol but could not parse it. `parsed_protocol` is set to + that decoder's protocol, its sub-map stays empty, `decode_error` explains, and no other decoder is tried. + +## Data Flow + +### Packet mode + +After the reader decodes a packet, and if a decoder column is projected, it passes the packet and the +bytes for the decoder's layer to the registry: + +- TCP and UDP: the transport payload. +- ICMP and ICMPv6: the bytes after the IP header. +- ARP: the bytes after the link-layer header. + +`Packet` gains two accessors for the last two. + +### Session mode + +When `TcpSessionizer` writes a session, at its close or at end of file, it passes the session to the +registry. A `TcpStream` for each direction is built only when a decoder column is projected. + +**TcpStream** is the reassembled byte stream of one direction: + +- Segments are ordered by sequence number relative to the first segment seen, so sequence wraparound + is handled. +- Retransmitted and overlapping bytes are kept once. +- Missing ranges are recorded. `hasGaps()` and `firstGap()` let decoders stop at a gap rather than parse + across it. + +**Client and server:** the client is the side that sent the SYN without ACK. If the handshake was not +captured, the endpoint with the lower port is taken as the server. + +### Repeated values + +Lists use Drill repeated types: DNS answers, SMTP recipients, and the requests of a keep-alive HTTP +connection are arrays or repeated maps, for example `parsed_data.http.exchanges[0].uri`. + +## Credentials + +Decoders for protocols that carry cleartext credentials (FTP, POP3, IMAP, SMTP AUTH, HTTP Basic) write +`username` and `password_present` by default. The password itself is written to a `password` field only +when the format option `exposeCredentials` is true. Credentials sent after a protocol switches to TLS are +not visible to any decoder. + +## Error Handling + +The rule: **an error never stops a file from being analyzed, and every error is visible in the results.** + +Errors are reported in the `decode_error` column of the row they affect, prefixed with where they +happened (`dns: ...`, `packet: ...`, `file: ...`). A row with several problems lists them separated by +`; `. Each reader also logs one warning per file with error counts by kind. + +Source | Result +-------|------- +Decoder `parse` throws | Row kept; `parsed_protocol` set, sub-map empty, `decode_error` = `: ` +Decoder `write` throws | Row kept with whatever was written; `decode_error` = `: write failed: `. This is a decoder bug. +Malformed packet headers (for example an invalid IPv4 header length, which used to fail the query) | Row kept with the fields that could be read; `is_corrupt` true where the schema has it; `decode_error` = `packet: ` +PCAP-NG packet referencing an undefined interface | Row kept, link type unknown so no IP decoding; `decode_error` explains +PCAP-NG interface with an unsupported `if_tsresol` | Its packets use microseconds; `decode_error` explains +PCAP-NG block with an invalid captured length | Block skipped; an error row is written +Structural damage that makes the rest of the file unreadable (invalid block length, truncated block or header, unreadable file, not a PCAP or PCAP-NG file) | Rows read so far are kept; reading the file stops; an error row is written +Session stream gap that stops a decoder early | Fields parsed before the gap are kept; `decode_error` = `: stopped at missing data in stream at byte ` + +An **error row** has `decode_error` set and every other column null. To allow this, the PCAP-NG packet +columns that are currently `REQUIRED` (`timestamp`, `packet_length`, `captured_length`, `interface_id`, +`link_type`) become nullable. Queries that count packets can exclude error rows with +`WHERE decode_error IS NULL` or count them with `WHERE decode_error IS NOT NULL`. + +Snapshot-length truncation is normal capture behavior, not an error: payloads are bounded by the +captured length without reporting anything. + +### Bounded work + +Capture files are untrusted input. Every decoder must bound its work so a crafted packet cannot hang a +reader or exhaust memory: + +- Follow at most 16 DNS compression pointers per name, and never a pointer to its own position or later. +- Cap repeated items: 64 headers or records per message, 64 list entries per field. +- Cap strings at 4 KB, truncating longer values. +- Never allocate based on a length field without checking it against the bytes available. +- Session decoders stop after 1,000 messages per session. + +Values over a cap are truncated and noted in `decode_error`; they are never a reason to fail. + +## Testing + +Each decoder has: + +- Unit tests of `parse` on byte arrays, without a Drill cluster. +- A query test on a fixture covering a valid exchange, a different protocol on the decoder's port (must + be left undecoded), and a truncated or malformed message (must produce `decode_error`, not a failed + query). + +Fixtures are generated by scripts or taken from public sample captures, and their expected values are +cross-checked with an independent decoder (scapy or Wireshark). + +The framework is tested with a test-only decoder registered under `src/test/resources/META-INF/services`, +which proves the third-party path end to end, and with fixtures that exercise each error-handling row +above. + +## Rollout + +All phases are part of DRILL-8556. + +Phase | Contents +------|--------- +1. Framework | Interfaces, registry, `TcpStream`, the three columns, `exposeCredentials`, the error handling above for both readers. Decoders: DNS (including mDNS and LLMNR) per packet, HTTP/1.x per packet and per session. +2. Packet decoders | DHCP, DHCPv6, NTP, TLS ClientHello (SNI, ALPN, JA3), SSDP, SIP, Syslog, NetBIOS name service, TFTP, STUN, RADIUS, SNMP, ICMP and ICMPv6, ARP +3. Session decoders | SMTP, POP3, IMAP, FTP control channel, SSH banners (HASSH if practical), TLS server side (certificate subject, issuer, validity; JA3S), DNS over TCP +Later | Kerberos, SMB, QUIC, HTTP/2. These need much more parsing per protocol and are encrypted or deeply nested. + +## Phase 1 Decoder Fields + +### `dns` (packet) + +Ports 53, 5353 (mDNS) and 5355 (LLMNR), over UDP and single-segment TCP (DNS over TCP across segments is +a phase 3 session decoder). A payload is DNS only if the header counts are consistent with its length and +every question and record parses. + +Field | Type +------|----- +`transaction_id` | INT +`is_response` | BIT +`opcode` | INT +`rcode` | INT +`authoritative`, `truncated`, `recursion_desired`, `recursion_available` | BIT +`questions` | repeated map: `name` VARCHAR, `type` VARCHAR (`A`, `AAAA`, ... or the number), `class` INT +`answers`, `authorities`, `additionals` | repeated map: `name` VARCHAR, `type` VARCHAR, `class` INT, `ttl` BIGINT, `data` VARCHAR (address, name, or text rendering of the record) + +### `http` (packet) + +TCP. A payload is HTTP only if it starts with a request line (` HTTP/1.x`) or a status +line (`HTTP/1.x `). The message's start line and the headers present in that segment are +parsed. + +Field | Type +------|----- +`is_request` | BIT +`method`, `uri`, `version` | VARCHAR +`status_code` | INT +`reason` | VARCHAR +`host`, `user_agent`, `content_type`, `referer` | VARCHAR +`content_length` | BIGINT +`headers` | repeated map: `name`, `value` VARCHAR +`username` | VARCHAR (HTTP Basic) +`password_present` | BIT +`password` | VARCHAR (only with `exposeCredentials`) + +### `http` (session) + +Every request in the client stream and response in the server stream, paired in order. + +Field | Type +------|----- +`exchanges` | repeated map: the request fields (`method`, `uri`, `version`, `host`, `user_agent`, `referer`, `username`, `password_present`, `password`, `request_headers`) and the response fields (`status_code`, `reason`, `content_type`, `content_length`, `response_headers`) + +Bodies are skipped using `Content-Length` or chunked encoding, so later messages on a keep-alive +connection are found. Bodies themselves are not returned; `data_from_originator` and `data_from_remote` +already carry the raw bytes. + +## Writing a Decoder + +1. Implement `PacketProtocolDecoder` or `SessionProtocolDecoder`, where `T` is a small class + holding the parsed fields. +2. In `accepts`, check transport and ports only. +3. In `parse`, validate everything before returning a value. Return null for data that is not your + protocol; throw for data that is your protocol but malformed. Follow the bounded-work rules. +4. In `write`, write only fields declared in `defineSchema`, and skip null values (the sub-map's + fields are nullable). +5. List the class in the matching `META-INF/services` file. +6. Add the tests described above. From be84c935b1eb32fc43e90694a4ebaac0cff364fe Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 14:39:01 -0400 Subject: [PATCH 03/50] DRILL-8556: Record malformed packet headers instead of failing --- .../drill/exec/store/pcap/decoder/Packet.java | 32 ++++++-- .../exec/store/pcapng/PacketDecoder.java | 30 ++++--- .../store/pcap/TestPacketDecodeErrors.java | 69 ++++++++++++++++ .../exec/store/pcap/protocol/TestPackets.java | 78 +++++++++++++++++++ 4 files changed, 190 insertions(+), 19 deletions(-) create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketDecodeErrors.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestPackets.java diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java index 0e7f0df9a8e..f263e844868 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java @@ -59,6 +59,8 @@ public class Packet implements Comparable { protected int etherProtocol; protected int protocol; protected boolean isCorrupt = false; + // Why the packet could not be fully decoded, or null + private String decodeError; private static final Logger logger = LoggerFactory.getLogger(Packet.class); @@ -378,6 +380,14 @@ public boolean isCorrupt(){ return isCorrupt; } + public String getDecodeError() { + return decodeError; + } + + protected void setDecodeError(String decodeError) { + this.decodeError = decodeError; + } + public byte[] getData() { int payloadStart = ipOffset + getIPHeaderLength(); if (isTcpPacket()) { @@ -499,12 +509,20 @@ private long getTimestampMicro(final byte[] header, final boolean byteOrder, fin private void decodeEtherPacket() { etherProtocol = getShort(raw, etherOffset + PacketConstants.PACKET_PROTOCOL_OFFSET); ipOffset = etherOffset + PacketConstants.IP_OFFSET; - if (isIpV4Packet()) { - protocol = processIpV4Packet(); - } else if (isIpV6Packet()) { - protocol = processIpV6Packet(); - } else if (isPPPoV6Packet()) { - protocol = getByte(raw, etherOffset + 48); + try { + if (isIpV4Packet()) { + protocol = processIpV4Packet(); + } else if (isIpV6Packet()) { + protocol = processIpV6Packet(); + } else if (isPPPoV6Packet()) { + protocol = getByte(raw, etherOffset + 48); + } + } catch (RuntimeException e) { + // Keep the link-layer fields; treat the rest as undecodable + isCorrupt = true; + decodeError = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + etherProtocol = 0; + transportOffset = -1; } // everything is decoded lazily } @@ -552,7 +570,7 @@ protected int processIpV6Packet() { private void validateIpV4Packet() { Preconditions.checkState(ipVersion() == 4, "Should have seen IP version 4, got %d", ipVersion()); int n = ipV4HeaderLength(); - Preconditions.checkState(n >= 20 && n < 200, "Invalid header length: ", n); + Preconditions.checkState(n >= 20 && n < 200, "Invalid IPv4 header length %s", n); } private String getEthernetAddress(int offset) { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java index e9ca47f18b2..96ca36dfbde 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java @@ -68,18 +68,24 @@ public boolean readPcapng(final byte[] raw, final int linkType) { // place a virtual one just before the network layer etherOffset = networkOffset - PacketConstants.IP_OFFSET; ipOffset = networkOffset; - if (isIpV4Packet()) { - protocol = processIpV4Packet(); - return true; - } else if (isIpV6Packet()) { - int tmp = processIpV6Packet(); - if (tmp != -1) { - protocol = tmp; + try { + if (isIpV4Packet()) { + protocol = processIpV4Packet(); + return true; + } else if (isIpV6Packet()) { + int tmp = processIpV6Packet(); + if (tmp != -1) { + protocol = tmp; + } + return true; + } else if (isPPPoV6Packet()) { + protocol = getByte(raw, etherOffset + 48); + return true; } - return true; - } else if (isPPPoV6Packet()) { - protocol = getByte(raw, etherOffset + 48); - return true; + } catch (RuntimeException e) { + setDecodeError(e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName()); + etherProtocol = 0; + transportOffset = -1; } return false; } @@ -241,7 +247,7 @@ private String formatMac(int offset) { return mac.toString(); } - void setTimestamp(Instant timestamp) { + public void setTimestamp(Instant timestamp) { setTimestampMicro(timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000); } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketDecodeErrors.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketDecodeErrors.java new file mode 100644 index 00000000000..13ae67f9f63 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketDecodeErrors.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.TestPackets; +import org.apache.drill.exec.store.pcapng.PacketDecoder; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestPacketDecodeErrors extends BaseTest { + + @Test + public void testMalformedIpv4HeaderInPcapng() { + byte[] raw = TestPackets.ipv4("10.0.0.1", "10.0.0.2", 17, new byte[8]); + raw[0] = 0x43; // IHL 3: shorter than the minimum header + PacketDecoder packet = new PacketDecoder(); + assertFalse(packet.readPcapng(raw, 101)); + assertNotNull(packet.getDecodeError()); + assertTrue(packet.getDecodeError(), packet.getDecodeError().contains("header length")); + } + + @Test + public void testMalformedIpv4HeaderInPcap() { + // PCAP record header (16 bytes, little endian) followed by an Ethernet frame with IHL 3 + byte[] ip = TestPackets.ipv4("10.0.0.1", "10.0.0.2", 17, new byte[8]); + ip[0] = 0x43; + byte[] record = new byte[16 + 14 + ip.length]; + ByteBuffer.wrap(record).order(ByteOrder.LITTLE_ENDIAN) + .putInt(0).putInt(0).putInt(14 + ip.length).putInt(14 + ip.length); + record[16 + 12] = 0x08; // EtherType IPv4 + System.arraycopy(ip, 0, record, 16 + 14, ip.length); + Packet packet = new Packet(); + packet.decodePcap(record, 0, false, 65535); + assertTrue(packet.isCorrupt()); + assertNotNull(packet.getDecodeError()); + } + + @Test + public void testValidPacketHasNoError() { + PacketDecoder packet = TestPackets.udp("10.0.0.1", 1000, "10.0.0.2", 53, new byte[] {1, 2}); + assertNull(packet.getDecodeError()); + assertEquals(53, packet.getDst_port()); + } +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestPackets.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestPackets.java new file mode 100644 index 00000000000..f0c353c72cc --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestPackets.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.time.Instant; + +import org.apache.drill.exec.store.pcapng.PacketDecoder; + +/** Builds decoded raw-IP packets for unit tests. */ +public final class TestPackets { + public static final int FIN = 0x01; + public static final int SYN = 0x02; + public static final int RST = 0x04; + public static final int PSH = 0x08; + public static final int ACK = 0x10; + private static final int LINKTYPE_RAW = 101; + + private TestPackets() { } + + public static PacketDecoder udp(String src, int srcPort, String dst, int dstPort, byte[] payload) { + ByteBuffer udp = ByteBuffer.allocate(8 + payload.length); + udp.putShort((short) srcPort).putShort((short) dstPort).putShort((short) (8 + payload.length)).putShort((short) 0).put(payload); + return decode(ipv4(src, dst, 17, udp.array()), Instant.EPOCH); + } + + public static PacketDecoder tcp(String src, int srcPort, String dst, int dstPort, long seq, int flags, byte[] payload) { + return tcp(src, srcPort, dst, dstPort, seq, flags, payload, Instant.EPOCH); + } + + public static PacketDecoder tcp(String src, int srcPort, String dst, int dstPort, long seq, int flags, byte[] payload, + Instant timestamp) { + ByteBuffer tcp = ByteBuffer.allocate(20 + payload.length); + tcp.putShort((short) srcPort).putShort((short) dstPort).putInt((int) seq).putInt(0) + .put((byte) (5 << 4)).put((byte) flags).putShort((short) 65535).putShort((short) 0).putShort((short) 0) + .put(payload); + return decode(ipv4(src, dst, 6, tcp.array()), timestamp); + } + + public static byte[] ipv4(String src, String dst, int protocol, byte[] transport) { + ByteBuffer ip = ByteBuffer.allocate(20 + transport.length); + ip.put((byte) 0x45).put((byte) 0).putShort((short) (20 + transport.length)).putShort((short) 0).putShort((short) 0) + .put((byte) 64).put((byte) protocol).putShort((short) 0).put(address(src)).put(address(dst)).put(transport); + return ip.array(); + } + + public static PacketDecoder decode(byte[] rawIp, Instant timestamp) { + PacketDecoder packet = new PacketDecoder(); + packet.readPcapng(rawIp, LINKTYPE_RAW); + packet.setTimestamp(timestamp); + return packet; + } + + private static byte[] address(String host) { + try { + return InetAddress.getByName(host).getAddress(); + } catch (UnknownHostException e) { + throw new IllegalArgumentException(e); + } + } +} From b08a5202767bc13e2c1af6b62413f4105ab7b639 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 14:39:57 -0400 Subject: [PATCH 04/50] DRILL-8556: Add protocol decoder interfaces and registry --- .../store/pcap/protocol/DecodeResult.java | 60 ++++++++ .../store/pcap/protocol/DecoderContext.java | 31 ++++ .../pcap/protocol/PacketProtocolDecoder.java | 49 ++++++ .../store/pcap/protocol/ProtocolDecoder.java | 32 ++++ .../store/pcap/protocol/ProtocolDecoders.java | 143 ++++++++++++++++++ .../pcap/protocol/RowDecoderContext.java | 64 ++++++++ .../store/pcap/protocol/package-info.java | 22 +++ .../store/pcap/protocol/EchoTestDecoder.java | 68 +++++++++ .../pcap/protocol/TestProtocolDecoders.java | 114 ++++++++++++++ ....store.pcap.protocol.PacketProtocolDecoder | 1 + 10 files changed, 584 insertions(+) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecoderContext.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/PacketProtocolDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/RowDecoderContext.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/package-info.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/EchoTestDecoder.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java create mode 100644 contrib/format-pcapng/src/test/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java new file mode 100644 index 00000000000..9e32b547043 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** The decoder that handled a row, and its parsed value or the reason it failed. */ +public final class DecodeResult { + private final ProtocolDecoder decoder; + private final Object parsed; + private final String error; + + private DecodeResult(ProtocolDecoder decoder, Object parsed, String error) { + this.decoder = decoder; + this.parsed = parsed; + this.error = error; + } + + static DecodeResult success(ProtocolDecoder decoder, Object parsed) { + return new DecodeResult(decoder, parsed, null); + } + + static DecodeResult failure(ProtocolDecoder decoder, String error) { + return new DecodeResult(decoder, null, error); + } + + public String protocol() { + return decoder.protocol(); + } + + /** Null if parsing failed. */ + public Object parsed() { + return parsed; + } + + /** Null if parsing succeeded. */ + public String error() { + return error; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + public void write(TupleWriter fields) { + ((PacketProtocolDecoder) decoder).write(parsed, fields); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecoderContext.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecoderContext.java new file mode 100644 index 00000000000..a08bbaaa6d6 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecoderContext.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +/** What a decoder may use while parsing one packet or session. */ +public interface DecoderContext { + + /** True if decoders may store cleartext passwords (format option exposeCredentials). */ + boolean exposeCredentials(); + + /** + * Notes a problem that did not prevent decoding, such as a truncated list. + * The message appears in decode_error, prefixed with the protocol name. + */ + void warn(String message); +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/PacketProtocolDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/PacketProtocolDecoder.java new file mode 100644 index 00000000000..4e4c81d7c84 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/PacketProtocolDecoder.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** + * Decodes the payload of single packets. Implementations are listed in + * META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder. + * + * @param the parsed form of one packet + */ +public interface PacketProtocolDecoder extends ProtocolDecoder { + + /** Declares the fields of parsed_data.<protocol>. All fields should be nullable or repeated. */ + void defineSchema(SchemaBuilder fields); + + /** Cheap pre-check, typically transport and port. */ + boolean accepts(Packet packet); + + /** + * Fully validates and parses the payload. + * + * @param payload the TCP or UDP payload; null if the packet has none + * @return null if the payload is not this protocol + * @throws RuntimeException if the payload is this protocol but malformed + */ + T parse(Packet packet, byte[] payload, DecoderContext context); + + /** Writes a value returned by {@link #parse}. Skip null fields. */ + void write(T parsed, TupleWriter fields); +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoder.java new file mode 100644 index 00000000000..ae0dc15b1ca --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoder.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +/** + * Common part of packet and session protocol decoders. See docs/dev/PcapProtocolDecoders.md. + */ +public interface ProtocolDecoder { + + /** Value of parsed_protocol and the name of this decoder's sub-map in parsed_data. Lower case, unique. */ + String protocol(); + + /** When several decoders accept the same data, higher priority is tried first. */ + default int priority() { + return 0; + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java new file mode 100644 index 00000000000..b6f7c98d287 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; + +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The protocol decoders found with {@link ServiceLoader}, and the matching rules of + * docs/dev/PcapProtocolDecoders.md. + */ +public class ProtocolDecoders { + private static final Logger logger = LoggerFactory.getLogger(ProtocolDecoders.class); + private static volatile ProtocolDecoders instance; + + private final List> packetDecoders; + private final TupleMetadata packetDataSchema; + + /** The decoders on the classpath, loaded once. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public static ProtocolDecoders get() { + if (instance == null) { + synchronized (ProtocolDecoders.class) { + if (instance == null) { + ClassLoader loader = ProtocolDecoders.class.getClassLoader(); + instance = new ProtocolDecoders((Iterable) ServiceLoader.load(PacketProtocolDecoder.class, loader)); + } + } + } + return instance; + } + + public ProtocolDecoders(Iterable> packetDecoders) { + SchemaBuilder data = new SchemaBuilder(); + this.packetDecoders = usable(packetDecoders, data); + this.packetDataSchema = data.buildSchema(); + } + + public List> packetDecoders() { + return packetDecoders; + } + + /** Members of parsed_data in packet mode: one map per decoder. */ + public TupleMetadata packetDataSchema() { + return packetDataSchema; + } + + /** + * Finds the decoder that handles a packet. + * + * @return null if no decoder handles it + */ + public DecodeResult decodePacket(Packet packet, byte[] payload, RowDecoderContext context) { + for (PacketProtocolDecoder decoder : packetDecoders) { + try { + if (!decoder.accepts(packet)) { + continue; + } + } catch (RuntimeException e) { + // A pre-check that cannot evaluate the packet does not claim it + continue; + } + context.begin(decoder.protocol()); + try { + Object parsed = decoder.parse(packet, payload, context); + if (parsed != null) { + return DecodeResult.success(decoder, parsed); + } + context.discard(); + } catch (RuntimeException e) { + context.discard(); + return DecodeResult.failure(decoder, decoder.protocol() + ": " + describe(e)); + } + } + return null; + } + + public static String describe(Throwable e) { + return e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + } + + /** + * Orders decoders by priority, drops duplicate protocol names and decoders whose schema + * cannot be built, and adds each decoder's map to the parsed_data schema. + */ + private static List usable(Iterable decoders, SchemaBuilder data) { + List sorted = new ArrayList<>(); + decoders.forEach(sorted::add); + sorted.sort(Comparator.comparingInt(ProtocolDecoder::priority).reversed()); + Map byName = new HashMap<>(); + List result = new ArrayList<>(); + for (D decoder : sorted) { + D existing = byName.get(decoder.protocol()); + if (existing != null) { + logger.warn("Ignoring protocol decoder {}: protocol {} is already handled by {}", + decoder.getClass().getName(), decoder.protocol(), existing.getClass().getName()); + continue; + } + try { + SchemaBuilder fields = new SchemaBuilder(); + defineSchema(decoder, fields); + data.add(MetadataUtils.newMap(decoder.protocol(), fields.buildSchema())); + } catch (RuntimeException e) { + logger.warn("Ignoring protocol decoder {}: its schema could not be built", decoder.getClass().getName(), e); + continue; + } + byName.put(decoder.protocol(), decoder); + result.add(decoder); + } + return Collections.unmodifiableList(result); + } + + private static void defineSchema(ProtocolDecoder decoder, SchemaBuilder fields) { + ((PacketProtocolDecoder) decoder).defineSchema(fields); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/RowDecoderContext.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/RowDecoderContext.java new file mode 100644 index 00000000000..669a5b65437 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/RowDecoderContext.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.util.ArrayList; +import java.util.List; + +/** Decoder context for one row: collects warnings, prefixed with the decoder's protocol. */ +public class RowDecoderContext implements DecoderContext { + private final boolean exposeCredentials; + private final List warnings = new ArrayList<>(); + private String protocol; + private int mark; + + public RowDecoderContext(boolean exposeCredentials) { + this.exposeCredentials = exposeCredentials; + } + + @Override + public boolean exposeCredentials() { + return exposeCredentials; + } + + @Override + public void warn(String message) { + warnings.add(protocol + ": " + message); + } + + /** Starts a new row. */ + public void reset() { + warnings.clear(); + } + + public List warnings() { + return warnings; + } + + void begin(String protocol) { + this.protocol = protocol; + mark = warnings.size(); + } + + /** Drops warnings from a decoder that turned out not to handle the row. */ + void discard() { + while (warnings.size() > mark) { + warnings.remove(warnings.size() - 1); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/package-info.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/package-info.java new file mode 100644 index 00000000000..1a4a04afe29 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Pluggable decoders for application protocols carried in PCAP and PCAP-NG captures. + * See docs/dev/PcapProtocolDecoders.md. + */ +package org.apache.drill.exec.store.pcap.protocol; diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/EchoTestDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/EchoTestDecoder.java new file mode 100644 index 00000000000..41dacbb9914 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/EchoTestDecoder.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.nio.charset.StandardCharsets; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** Test-only decoder for UDP port 7, registered through src/test/resources/META-INF/services. */ +public class EchoTestDecoder implements PacketProtocolDecoder { + + @Override + public String protocol() { + return "echo_test"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("text", MinorType.VARCHAR); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isUdpPacket() && packet.getDst_port() == 7; + } + + @Override + public String parse(Packet packet, byte[] payload, DecoderContext context) { + String text = payload == null ? "" : new String(payload, StandardCharsets.US_ASCII); + if (text.startsWith("ECHO!")) { + throw new IllegalArgumentException("bad echo"); + } + if (!text.startsWith("ECHO:")) { + return null; + } + String value = text.substring(5); + if (value.length() > 10) { + context.warn("long text"); + } + return value; + } + + @Override + public void write(String parsed, TupleWriter fields) { + if (parsed.equals("write-fail")) { + throw new IllegalStateException("cannot write"); + } + fields.scalar("text").setString(parsed); + } +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java new file mode 100644 index 00000000000..92c53529f4d --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; + +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestProtocolDecoders extends BaseTest { + + private static Packet echo(String text) { + return TestPackets.udp("10.0.0.1", 4000, "10.0.0.2", 7, text.getBytes(StandardCharsets.US_ASCII)); + } + + private static DecodeResult decode(ProtocolDecoders decoders, Packet packet, RowDecoderContext context) { + return decoders.decodePacket(packet, packet.getData(), context); + } + + @Test + public void testServiceLoaderFindsTestDecoder() { + assertTrue(ProtocolDecoders.get().packetDecoders().stream().anyMatch(d -> d.protocol().equals("echo_test"))); + assertNotNull(ProtocolDecoders.get().packetDataSchema().metadata("echo_test")); + } + + @Test + public void testParsed() { + ProtocolDecoders decoders = new ProtocolDecoders(Collections.singletonList(new EchoTestDecoder())); + RowDecoderContext context = new RowDecoderContext(false); + DecodeResult result = decode(decoders, echo("ECHO:hi"), context); + assertEquals("echo_test", result.protocol()); + assertEquals("hi", result.parsed()); + assertNull(result.error()); + assertTrue(context.warnings().isEmpty()); + } + + @Test + public void testNotThisProtocol() { + ProtocolDecoders decoders = new ProtocolDecoders(Collections.singletonList(new EchoTestDecoder())); + assertNull(decode(decoders, echo("hello"), new RowDecoderContext(false))); + Packet otherPort = TestPackets.udp("10.0.0.1", 4000, "10.0.0.2", 9, "ECHO:hi".getBytes(StandardCharsets.US_ASCII)); + assertNull(decode(decoders, otherPort, new RowDecoderContext(false))); + } + + @Test + public void testParseFailureClaimsPacket() { + ProtocolDecoders decoders = new ProtocolDecoders(Collections.singletonList(new EchoTestDecoder())); + DecodeResult result = decode(decoders, echo("ECHO!x"), new RowDecoderContext(false)); + assertEquals("echo_test", result.protocol()); + assertNull(result.parsed()); + assertEquals("echo_test: bad echo", result.error()); + } + + @Test + public void testWarningsArePrefixed() { + ProtocolDecoders decoders = new ProtocolDecoders(Collections.singletonList(new EchoTestDecoder())); + RowDecoderContext context = new RowDecoderContext(false); + decode(decoders, echo("ECHO:a long echo text"), context); + assertEquals(Collections.singletonList("echo_test: long text"), context.warnings()); + } + + @Test + public void testDuplicateProtocolKeepsFirst() { + EchoTestDecoder first = new EchoTestDecoder(); + ProtocolDecoders decoders = new ProtocolDecoders(Arrays.asList(first, new EchoTestDecoder())); + assertEquals(1, decoders.packetDecoders().size()); + } + + @Test + public void testHigherPriorityFirst() { + EchoTestDecoder low = new EchoTestDecoder() { + @Override + public String protocol() { + return "low"; + } + }; + EchoTestDecoder high = new EchoTestDecoder() { + @Override + public String protocol() { + return "high"; + } + + @Override + public int priority() { + return 10; + } + }; + ProtocolDecoders decoders = new ProtocolDecoders(Arrays.asList(low, high)); + assertEquals("high", decode(decoders, echo("ECHO:hi"), new RowDecoderContext(false)).protocol()); + } +} diff --git a/contrib/format-pcapng/src/test/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder b/contrib/format-pcapng/src/test/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder new file mode 100644 index 00000000000..f4f6b534a12 --- /dev/null +++ b/contrib/format-pcapng/src/test/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder @@ -0,0 +1 @@ +org.apache.drill.exec.store.pcap.protocol.EchoTestDecoder From 2a6ee900d86cd0383da2e35e30ef7b4e4a5629e6 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 14:40:41 -0400 Subject: [PATCH 05/50] DRILL-8556: Add exposeCredentials format option --- .../store/pcap/plugin/PcapFormatConfig.java | 21 +++++++-- .../exec/store/pcap/TestPcapFormatConfig.java | 46 +++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapFormatConfig.java diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/PcapFormatConfig.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/PcapFormatConfig.java index 37226b11807..75c3691d86e 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/PcapFormatConfig.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/PcapFormatConfig.java @@ -38,14 +38,21 @@ public class PcapFormatConfig implements FormatPluginConfig { private final List extensions; private final boolean stat; private final boolean sessionizeTCPStreams; + private final boolean exposeCredentials; + + public PcapFormatConfig(List extensions, boolean stat, Boolean sessionizeTCPStreams) { + this(extensions, stat, sessionizeTCPStreams, null); + } @JsonCreator public PcapFormatConfig(@JsonProperty("extensions") List extensions, @JsonProperty("stat") boolean stat, - @JsonProperty("sessionizeTCPStreams") Boolean sessionizeTCPStreams) { + @JsonProperty("sessionizeTCPStreams") Boolean sessionizeTCPStreams, + @JsonProperty("exposeCredentials") Boolean exposeCredentials) { this.extensions = extensions == null ? DEFAULT_EXTNS : ImmutableList.copyOf(extensions); this.stat = stat; this.sessionizeTCPStreams = sessionizeTCPStreams != null && sessionizeTCPStreams; + this.exposeCredentials = exposeCredentials != null && exposeCredentials; } @JsonProperty("extensions") @@ -63,6 +70,12 @@ public boolean getSessionizeTCPStreams() { return sessionizeTCPStreams; } + /** True if protocol decoders may return cleartext passwords. */ + @JsonProperty("exposeCredentials") + public boolean getExposeCredentials() { + return exposeCredentials; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -73,12 +86,13 @@ public boolean equals(Object o) { } PcapFormatConfig that = (PcapFormatConfig) o; return Objects.equals(extensions, that.extensions) && Objects.equals(stat, that.getStat()) && - Objects.equals(sessionizeTCPStreams, that.sessionizeTCPStreams); + Objects.equals(sessionizeTCPStreams, that.sessionizeTCPStreams) && + Objects.equals(exposeCredentials, that.exposeCredentials); } @Override public int hashCode() { - return Objects.hash(extensions, stat, sessionizeTCPStreams); + return Objects.hash(extensions, stat, sessionizeTCPStreams, exposeCredentials); } @Override @@ -87,6 +101,7 @@ public String toString() { .field("extensions", extensions) .field("stat", stat) .field("sessionizeTCPStreams", sessionizeTCPStreams) + .field("exposeCredentials", exposeCredentials) .toString(); } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapFormatConfig.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapFormatConfig.java new file mode 100644 index 00000000000..0096eb9dce1 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapFormatConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.drill.exec.store.pcap.plugin.PcapFormatConfig; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestPcapFormatConfig extends BaseTest { + + @Test + public void testExposeCredentialsDefaultsToFalse() throws Exception { + PcapFormatConfig config = new ObjectMapper().readValue("{\"type\":\"pcap\"}", PcapFormatConfig.class); + assertFalse(config.getExposeCredentials()); + } + + @Test + public void testExposeCredentialsRoundTrip() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + PcapFormatConfig config = new PcapFormatConfig(null, false, false, true); + PcapFormatConfig copy = mapper.readValue(mapper.writeValueAsString(config), PcapFormatConfig.class); + assertTrue(copy.getExposeCredentials()); + assertEquals(config, copy); + assertEquals(config.hashCode(), copy.hashCode()); + } +} From dcf50af9c9f6509f6cdee30681b2b47a60f8aea6 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 14:43:35 -0400 Subject: [PATCH 06/50] DRILL-8556: Add decoder columns and never-fail errors to the PCAP-NG reader --- .../store/pcap/protocol/ProtocolColumns.java | 154 ++++++++++++++++++ .../drill/exec/store/pcapng/PcapColumn.java | 10 +- .../exec/store/pcapng/PcapngBatchReader.java | 114 ++++++++++--- .../drill/exec/store/pcapng/PcapngBlock.java | 20 +++ .../exec/store/pcapng/TestPcapngDecoding.java | 107 ++++++++++++ .../store/pcapng/TestPcapngRecordReader.java | 52 +++--- .../pcapng/TestPcapngStatRecordReader.java | 4 +- .../test/resources/fixtures/pcap_fixtures.py | 116 +++++++++++++ .../resources/pcapng/bad_block_length.pcapng | Bin 0 -> 260 bytes .../src/test/resources/pcapng/echo.pcapng | Bin 0 -> 416 bytes .../src/test/resources/pcapng/errors.pcapng | Bin 0 -> 484 bytes 11 files changed, 525 insertions(+), 52 deletions(-) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java create mode 100644 contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/bad_block_length.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/echo.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/errors.pcapng diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java new file mode 100644 index 00000000000..104b48e8bfc --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.physical.resultSet.RowSetLoader; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.vector.accessor.ScalarWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; +import org.slf4j.Logger; + +/** + * The parsed_protocol, parsed_data and decode_error columns, shared by the PCAP and PCAP-NG readers. + */ +public class ProtocolColumns { + public static final String PARSED_PROTOCOL = "parsed_protocol"; + public static final String PARSED_DATA = "parsed_data"; + public static final String DECODE_ERROR = "decode_error"; + + /** Packet rows, session rows, or stat rows (which only get decode_error). */ + public enum Mode { PACKET, SESSION, STAT } + + private final RowSetLoader loader; + private final ProtocolDecoders decoders; + private final RowDecoderContext context; + private final ScalarWriter protocolWriter; + private final TupleWriter dataWriter; + private final ScalarWriter errorWriter; + private final boolean decoding; + // Errors written so far, by prefix (dns, packet, file, ...), for the end-of-file warning + private final Map errorCounts = new TreeMap<>(); + + public static void addColumns(SchemaBuilder schema, Mode mode, ProtocolDecoders decoders) { + if (mode != Mode.STAT) { + schema.addNullable(PARSED_PROTOCOL, MinorType.VARCHAR); + schema.add(MetadataUtils.newMap(PARSED_DATA, dataSchema(mode, decoders))); + } + schema.addNullable(DECODE_ERROR, MinorType.VARCHAR); + } + + private static TupleMetadata dataSchema(Mode mode, ProtocolDecoders decoders) { + return decoders.packetDataSchema(); + } + + public ProtocolColumns(RowSetLoader loader, Mode mode, ProtocolDecoders decoders, boolean exposeCredentials) { + this.loader = loader; + this.decoders = decoders; + this.context = new RowDecoderContext(exposeCredentials); + this.errorWriter = loader.scalar(DECODE_ERROR); + if (mode == Mode.STAT) { + protocolWriter = null; + dataWriter = null; + decoding = false; + } else { + protocolWriter = loader.scalar(PARSED_PROTOCOL); + dataWriter = loader.tuple(PARSED_DATA); + decoding = loader.isProjected(PARSED_PROTOCOL) || loader.isProjected(PARSED_DATA); + } + } + + /** True if parsed_protocol or parsed_data is projected, so decoding is worth doing. */ + public boolean isDecoding() { + return decoding; + } + + /** + * Decodes a packet into the current row and writes decode_error. + * + * @param packet the decoded packet; null if it could not be decoded at all + * @param errors problems the reader already found for this row; may be null + */ + public void writePacket(Packet packet, List errors) { + List all = errors == null ? new ArrayList<>() : new ArrayList<>(errors); + if (decoding && packet != null) { + context.reset(); + byte[] payload = null; + try { + payload = packet.getData(); + } catch (RuntimeException e) { + all.add("packet: " + ProtocolDecoders.describe(e)); + } + write(decoders.decodePacket(packet, payload, context), all); + } + writeErrors(all); + } + + void write(DecodeResult result, List all) { + if (result != null) { + protocolWriter.setString(result.protocol()); + if (result.parsed() != null) { + try { + result.write(dataWriter.tuple(result.protocol())); + } catch (RuntimeException e) { + all.add(result.protocol() + ": write failed: " + ProtocolDecoders.describe(e)); + } + } + if (result.error() != null) { + all.add(result.error()); + } + } + all.addAll(context.warnings()); + } + + /** Sets decode_error on the current row; does nothing for an empty list. */ + public void writeErrors(List errors) { + if (errors != null && !errors.isEmpty()) { + errorWriter.setString(String.join("; ", errors)); + errors.forEach(this::count); + } + } + + /** Writes a row with only decode_error set. */ + public void writeErrorRow(String error) { + loader.start(); + errorWriter.setString(error); + loader.save(); + count(error); + } + + private void count(String error) { + int colon = error.indexOf(':'); + errorCounts.merge(colon > 0 ? error.substring(0, colon) : error, 1, Integer::sum); + } + + /** Logs one warning for the file if any row had an error, such as "{dns=3, packet=1}". */ + public void logSummary(Logger logger, String file) { + if (!errorCounts.isEmpty()) { + logger.warn("{}: rows with decode_error by kind: {}", file, errorCounts); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java index e0fcf68b79e..71cd5552e5a 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java @@ -157,7 +157,7 @@ static class PcapTimestamp extends PcapColumn { @Override MajorType getType() { - return Types.required(MinorType.TIMESTAMP); + return Types.optional(MinorType.TIMESTAMP); } @Override @@ -170,7 +170,7 @@ static class PcapPacketLength extends PcapColumn { @Override MajorType getType() { - return Types.required(MinorType.INT); + return Types.optional(MinorType.INT); } @Override @@ -530,7 +530,7 @@ static class PcapCapturedLength extends PcapColumn { @Override MajorType getType() { - return Types.required(MinorType.INT); + return Types.optional(MinorType.INT); } @Override @@ -543,7 +543,7 @@ static class PcapInterfaceId extends PcapColumn { @Override MajorType getType() { - return Types.required(MinorType.INT); + return Types.optional(MinorType.INT); } @Override @@ -572,7 +572,7 @@ static class PcapLinkType extends PcapColumn { @Override MajorType getType() { - return Types.required(MinorType.INT); + return Types.optional(MinorType.INT); } @Override diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java index 1fc0dcf6e0f..ad52bcc82e5 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java @@ -51,6 +51,8 @@ import org.apache.drill.exec.store.dfs.easy.EasySubScan; import org.apache.drill.exec.store.pcap.TcpSessionizer; import org.apache.drill.exec.store.pcap.plugin.PcapFormatConfig; +import org.apache.drill.exec.store.pcap.protocol.ProtocolColumns; +import org.apache.drill.exec.store.pcap.protocol.ProtocolDecoders; import org.apache.drill.exec.store.pcap.schema.Schema; import org.apache.drill.exec.util.Utilities; import org.apache.drill.exec.vector.accessor.ScalarWriter; @@ -92,8 +94,13 @@ public class PcapngBatchReader implements ManagedReader { private RowSetLoader loader; // Set when TCP packets are grouped into sessions instead of returned as rows private TcpSessionizer sessionizer; + private ProtocolColumns protocolColumns; private InputStream in; private Path path; + // Bytes of the file consumed so far, for error messages + private long position; + // Set when damage makes the rest of the file unreadable + private boolean finished; public PcapngBatchReader(final PcapFormatConfig config, final EasySubScan scan, FileSchemaNegotiator negotiator) { @@ -129,6 +136,11 @@ public PcapngBatchReader(final PcapFormatConfig config, final EasySubScan scan, loader = resultSetLoader.writer(); // bind the writer for columns bindColumns(loader); + protocolColumns = new ProtocolColumns(loader, rowMode(), ProtocolDecoders.get(), config.getExposeCredentials()); + } + + private ProtocolColumns.Mode rowMode() { + return config.getStat() ? ProtocolColumns.Mode.STAT : ProtocolColumns.Mode.PACKET; } /** @@ -183,20 +195,27 @@ public boolean next() { */ private PcapngBlock nextRow() throws IOException { while (true) { + if (finished) { + return null; + } + long blockStart = position; int n = IOUtils.read(in, header, 0, 8); if (n == 0) { return null; } if (n < 8) { - logger.warn("Ignoring truncated pcapng block header at end of {}", path); - return null; + finished = true; + return PcapngBlock.errorRow("file: truncated block header at byte " + blockStart); } ByteBuffer buf = ByteBuffer.wrap(header).order(byteOrder); int type = buf.getInt(0); int consumed = 8; if (type == SECTION_HEADER_TYPE) { // Each section declares its own byte order; the type itself is a palindrome - IOUtils.readFully(in, header, 8, 4); + if (IOUtils.read(in, header, 8, 4) < 4) { + finished = true; + return PcapngBlock.errorRow("file: truncated block header at byte " + blockStart); + } consumed = 12; byteOrder = ByteBuffer.wrap(header, 8, 4).order(ByteOrder.BIG_ENDIAN).getInt() == BYTE_ORDER_MAGIC ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; @@ -204,7 +223,8 @@ private PcapngBlock nextRow() throws IOException { } int totalLength = buf.getInt(4); if (totalLength < consumed + 4 || totalLength % 4 != 0) { - throw new IOException("Invalid pcapng block length " + totalLength + " for block type " + type); + finished = true; + return PcapngBlock.errorRow("file: invalid block length " + totalLength + " at byte " + blockStart); } // Body excludes the header and the trailing copy of the block length byte[] body = new byte[totalLength - consumed - 4]; @@ -212,18 +232,19 @@ private PcapngBlock nextRow() throws IOException { IOUtils.readFully(in, body); IOUtils.skipFully(in, 4); } catch (EOFException e) { - logger.warn("Ignoring truncated pcapng block at end of {}", path); - return null; + finished = true; + return PcapngBlock.errorRow("file: truncated block at byte " + blockStart); } + position += totalLength; ByteBuffer block = ByteBuffer.wrap(body).order(byteOrder); if (type == SECTION_HEADER_TYPE) { // Interface IDs are scoped to their section interfaces.clear(); } else if (type == INTERFACE_DESCRIPTION_TYPE) { - interfaces.add(new Interface(block)); + interfaces.add(new Interface(block, interfaces.size())); } else if (type == ENHANCED_PACKET_TYPE && !config.getStat()) { - return readPacket(block); + return readPacket(block, blockStart); } if (config.getStat() && (type == SECTION_HEADER_TYPE || type == INTERFACE_DESCRIPTION_TYPE @@ -231,18 +252,28 @@ private PcapngBlock nextRow() throws IOException { PcapngBlock row = new PcapngBlock(); row.comment = readComment(block, optionsOffset(type, block)); row.stats = readStats(type, block); + if (type == INTERFACE_DESCRIPTION_TYPE && interfaces.get(interfaces.size() - 1).error != null) { + row.addError(interfaces.get(interfaces.size() - 1).error); + } return row; } } } - private PcapngBlock readPacket(ByteBuffer block) throws IOException { + private PcapngBlock readPacket(ByteBuffer block, long blockStart) { PcapngBlock row = new PcapngBlock(); row.interfaceId = block.getInt(0); + Interface iface; if (row.interfaceId < 0 || row.interfaceId >= interfaces.size()) { - throw new IOException("Packet references undefined interface " + row.interfaceId); + // Unknown link type and resolution: keep the row, decode nothing + iface = Interface.UNDEFINED; + row.addError("file: packet references undefined interface " + row.interfaceId); + } else { + iface = interfaces.get(row.interfaceId); + if (iface.error != null) { + row.addError(iface.error); + } } - Interface iface = interfaces.get(row.interfaceId); row.interfaceName = iface.name; row.linkType = iface.linkType; // 64-bit timestamp stored as high then low 32-bit words @@ -250,7 +281,8 @@ private PcapngBlock readPacket(ByteBuffer block) throws IOException { row.capturedLength = block.getInt(12); row.originalLength = block.getInt(16); if (row.capturedLength < 0 || 20 + row.capturedLength > block.capacity()) { - throw new IOException("Invalid captured length " + row.capturedLength); + return PcapngBlock.errorRow("file: block at byte " + blockStart + " has invalid captured length " + + row.capturedLength); } row.data = Arrays.copyOfRange(block.array(), 20, 20 + row.capturedLength); @@ -281,6 +313,8 @@ private PcapngBlock readPacket(ByteBuffer block) throws IOException { if (packet.readPcapng(row.data, row.linkType)) { packet.setTimestamp(row.timestamp); row.packet = packet; + } else if (packet.getDecodeError() != null) { + row.addError("packet: " + packet.getDecodeError()); } } return row; @@ -463,20 +497,23 @@ private static int pad4(int length) { /** What packets need from their Interface Description Block. */ private static class Interface { - static final Interface DEFAULT = new Interface(); + static final Interface DEFAULT = new Interface(PacketDecoder.LINKTYPE_ETHERNET); + // For packets naming an interface that was never described: nothing is decoded + static final Interface UNDEFINED = new Interface(-1); final int linkType; String name; + // Why the interface's settings could not all be used, or null + String error; // Timestamps are counted in units of 1 / unitsPerSecond, plus an offset in seconds long unitsPerSecond = 1_000_000; long offsetSeconds; - private Interface() { - linkType = PacketDecoder.LINKTYPE_ETHERNET; + private Interface(int linkType) { + this.linkType = linkType; } - Interface(ByteBuffer block) throws IOException { + Interface(ByteBuffer block, int index) { linkType = block.getShort(0) & 0xFFFF; - IOException[] error = new IOException[1]; forEachOption(block, 8, (code, start, length) -> { switch (code) { case IF_NAME: @@ -490,7 +527,8 @@ private Interface() { } else if ((resolution & 0x80) != 0 && exponent <= 62) { unitsPerSecond = 1L << exponent; } else { - error[0] = new IOException("Unsupported if_tsresol " + resolution); + error = "file: interface " + index + " has unsupported if_tsresol " + resolution + + "; timestamps assume microseconds"; } break; case IF_TSOFFSET: @@ -500,9 +538,6 @@ private Interface() { break; } }); - if (error[0] != null) { - throw error[0]; - } } Instant toInstant(long timestamp) { @@ -518,23 +553,44 @@ Instant toInstant(long timestamp) { @Override public void close() { + if (protocolColumns != null) { + protocolColumns.logSummary(logger, path.toString()); + } AutoCloseables.closeSilently(in); } private void processBlock(PcapngBlock block) { + if (block.errorRow) { + protocolColumns.writeErrorRow(String.join("; ", block.errors)); + return; + } loader.start(); for (ColumnDefn columnDefn : projectedColumns) { - // pcapng file name - if (columnDefn.getName().equals(PcapColumn.PATH_NAME)) { - columnDefn.load(path.getName()); - } else { - // pcapng block data - columnDefn.load(block); + try { + if (columnDefn.getName().equals(PcapColumn.PATH_NAME)) { + // pcapng file name + columnDefn.load(path.getName()); + } else { + // pcapng block data + columnDefn.load(block); + } + } catch (RuntimeException e) { + block.addError("packet: " + columnDefn.getName() + ": " + ProtocolDecoders.describe(e)); } } + if (config.getStat()) { + protocolColumns.writeErrors(block.errors); + } else { + protocolColumns.writePacket(block.packet, block.errors); + } loader.save(); } + private static boolean isDecoderColumn(String name) { + return name.equals(ProtocolColumns.PARSED_PROTOCOL) || name.equals(ProtocolColumns.PARSED_DATA) + || name.equals(ProtocolColumns.DECODE_ERROR); + } + /** * Stat queries take precedence: they describe the capture, not its packets. */ @@ -556,6 +612,7 @@ private TupleMetadata defineMetadata() { for (ColumnDefn columnDefn : projectedColumns) { columnDefn.define(builder); } + ProtocolColumns.addColumns(builder, rowMode(), ProtocolDecoders.get()); return builder.buildSchema(); } @@ -583,6 +640,9 @@ private void processProjected(List columns) { for (SchemaPath schemaPath : columns) { // Support Case-Insensitive String projectedName = schemaPath.rootName().toLowerCase(); + if (isDecoderColumn(projectedName)) { + continue; + } PcapColumn pcapColumn; if (config.getStat()) { pcapColumn = PcapColumn.getSummaryColumns().get(projectedName); diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBlock.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBlock.java index d872fd13577..ffb96872b29 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBlock.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBlock.java @@ -18,6 +18,8 @@ package org.apache.drill.exec.store.pcapng; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import java.util.Map; /** @@ -45,4 +47,22 @@ class PcapngBlock { String hash; /** Decoded packet, or null if the link type or protocol is not supported. */ PacketDecoder packet; + /** Problems found while reading this block, or null. */ + List errors; + /** True for a row that only reports an error. */ + boolean errorRow; + + void addError(String error) { + if (errors == null) { + errors = new ArrayList<>(); + } + errors.add(error); + } + + static PcapngBlock errorRow(String error) { + PcapngBlock row = new PcapngBlock(); + row.errorRow = true; + row.addError(error); + return row; + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java new file mode 100644 index 00000000000..d22a70580c3 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcapng; + +import static org.junit.Assert.assertEquals; + +import java.nio.file.Paths; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.physical.rowSet.RowSet; +import org.apache.drill.exec.physical.rowSet.RowSetBuilder; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.ClusterTest; +import org.apache.drill.test.rowSet.RowSetComparison; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestPcapngDecoding extends ClusterTest { + + @BeforeClass + public static void setup() throws Exception { + ClusterTest.startCluster(ClusterFixture.builder(dirTestWatcher)); + dirTestWatcher.copyResourceToRoot(Paths.get("pcapng/")); + } + + @Test + public void testDecoderColumns() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.echo_test.text as text, decode_error " + + "from dfs.`pcapng/echo.pcapng` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("text", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("echo_test", "hello", null) + .addRow(null, null, null) + .addRow("echo_test", null, "echo_test: bad echo") + .addRow("echo_test", null, "echo_test: write failed: cannot write") + .addRow("echo_test", "a long echo text", "echo_test: long text") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testPacketErrorsDoNotStopTheFile() throws Exception { + String sql = "select interface_id, src_ip, decode_error from dfs.`pcapng/errors.pcapng`"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("interface_id", MinorType.INT) + .addNullable("src_ip", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow(0, null, "packet: Invalid IPv4 header length 12") + .addRow(0, "10.0.0.1", null) + .addRow(5, null, "file: packet references undefined interface 5") + .addRow(1, "10.0.0.1", "file: interface 1 has unsupported if_tsresol 127; timestamps assume microseconds") + .addRow(null, null, "file: block at byte 348 has invalid captured length 9999") + .addRow(0, "10.0.0.1", null) + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testStructuralDamageStopsWithErrorRow() throws Exception { + String sql = "select src_ip, decode_error from dfs.`pcapng/bad_block_length.pcapng`"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("src_ip", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("10.0.0.1", null) + .addRow("10.0.0.1", null) + .addRow(null, "file: invalid block length 7 at byte 184") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testDecodingSkippedWhenNotProjected() throws Exception { + // Without parsed_* projected, the parse failure is not reported + String sql = "select decode_error from dfs.`pcapng/echo.pcapng` where decode_error is not null"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + assertEquals(0, results.rowCount()); + results.clear(); + } +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java index cd6bf82cf6d..5e002b4fdf1 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java @@ -18,6 +18,8 @@ package org.apache.drill.exec.store.pcapng; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; import java.nio.file.Paths; import java.time.Instant; @@ -27,6 +29,7 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.physical.rowSet.RowSet; import org.apache.drill.exec.physical.rowSet.RowSetBuilder; +import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.test.ClusterFixture; @@ -65,8 +68,8 @@ public void testExplicitQuery() throws Exception { TupleMetadata schema = new SchemaBuilder() .addNullable("type", MinorType.VARCHAR) - .add("packet_length", MinorType.INT) - .add("timestamp", MinorType.TIMESTAMP) + .addNullable("packet_length", MinorType.INT) + .addNullable("timestamp", MinorType.TIMESTAMP) .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) @@ -119,8 +122,8 @@ public void testExplicitQueryWithCompressedFile() throws Exception { TupleMetadata schema = new SchemaBuilder() .addNullable("type", MinorType.VARCHAR) - .add("packet_length", MinorType.INT) - .add("timestamp", MinorType.TIMESTAMP) + .addNullable("packet_length", MinorType.INT) + .addNullable("timestamp", MinorType.TIMESTAMP) .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) @@ -152,7 +155,7 @@ public void testWhereSyntaxQuery() throws Exception { .addNullable("type", MinorType.VARCHAR) .addNullable("src_ip", MinorType.VARCHAR) .addNullable("dst_ip", MinorType.VARCHAR) - .add("packet_length", MinorType.INT) + .addNullable("packet_length", MinorType.INT) .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) @@ -171,8 +174,8 @@ public void testValidHeaders() throws Exception { RowSet sets = client.queryBuilder().sql(sql).rowSet(); TupleMetadata schema = new SchemaBuilder() - .add("timestamp", MinorType.TIMESTAMP) - .add("packet_length", MinorType.INT) + .addNullable("timestamp", MinorType.TIMESTAMP) + .addNullable("packet_length", MinorType.INT) .addNullable("type", MinorType.VARCHAR) .addNullable("src_ip", MinorType.VARCHAR) .addNullable("dst_ip", MinorType.VARCHAR) @@ -196,22 +199,33 @@ public void testValidHeaders() throws Exception { .addNullable("tcp_flags_fin", MinorType.INT) .addNullable("tcp_parsed_flags", MinorType.VARCHAR) .addNullable("packet_data", MinorType.VARCHAR) - .add("captured_length", MinorType.INT) - .add("interface_id", MinorType.INT) + .addNullable("captured_length", MinorType.INT) + .addNullable("interface_id", MinorType.INT) .addNullable("interface_name", MinorType.VARCHAR) - .add("link_type", MinorType.INT) + .addNullable("link_type", MinorType.INT) .addNullable("comment", MinorType.VARCHAR) .addNullable("direction", MinorType.VARCHAR) .addNullable("reception_type", MinorType.VARCHAR) .addNullable("fcs_length", MinorType.INT) .addNullable("drop_count", MinorType.BIGINT) .addNullable("packet_hash", MinorType.VARCHAR) - .build(); + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); - RowSet expected = new RowSetBuilder(client.allocator(), schema).build(); - new RowSetComparison(expected).verifyAndClearAll(sets); + // parsed_data holds one map per registered decoder, so only its presence is pinned here + TupleMetadata actual = sets.schema(); + assertEquals(schema.size() + 1, actual.size()); + for (int i = 0; i < schema.size(); i++) { + ColumnMetadata column = schema.metadata(i); + assertEquals(column.name(), column.majorType(), actual.metadata(column.name()).majorType()); + } + assertTrue(actual.metadata("parsed_data").isMap()); + assertNotNull(actual.metadata("parsed_data").tupleSchema().metadata("echo_test")); + sets.clear(); } + @Test public void testBigEndian() throws Exception { // The same capture written in both byte orders must decode identically. @@ -244,14 +258,14 @@ public void testLinkTypesAndInterfaces() throws Exception { RowSet results = client.queryBuilder().sql(sql).rowSet(); TupleMetadata schema = new SchemaBuilder() - .add("interface_id", MinorType.INT) - .add("link_type", MinorType.INT) + .addNullable("interface_id", MinorType.INT) + .addNullable("link_type", MinorType.INT) .addNullable("interface_name", MinorType.VARCHAR) - .add("timestamp", MinorType.TIMESTAMP) + .addNullable("timestamp", MinorType.TIMESTAMP) .addNullable("src_ip", MinorType.VARCHAR) .addNullable("src_port", MinorType.INT) .addNullable("src_mac_address", MinorType.VARCHAR) - .add("captured_length", MinorType.INT) + .addNullable("captured_length", MinorType.INT) .buildSchema(); Instant ts = Instant.parse("2024-01-02T03:04:05.678Z"); @@ -398,8 +412,8 @@ public void testInlineSchema() throws Exception { TupleMetadata schema = new SchemaBuilder() .addNullable("type", MinorType.VARCHAR) - .add("packet_length", MinorType.INT) - .add("timestamp", MinorType.TIMESTAMP) + .addNullable("packet_length", MinorType.INT) + .addNullable("timestamp", MinorType.TIMESTAMP) .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java index b49f8ec47e2..5927091e978 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java @@ -114,7 +114,8 @@ public void testSerDe() throws Exception { String plan = queryBuilder().sql(sql).explainJson(); long cnt = queryBuilder().physical(plan).singletonLong(); - assertEquals("Counts should match", 55, cnt); + // The error row of bad_block_length.pcapng is counted too + assertEquals("Counts should match", 63, cnt); } @Test @@ -150,6 +151,7 @@ public void testValidHeaders() throws Exception { .addNullable("isb_filteraccept", MinorType.BIGINT) .addNullable("isb_osdrop", MinorType.BIGINT) .addNullable("isb_usrdeliv", MinorType.BIGINT) + .addNullable("decode_error", MinorType.VARCHAR) .build(); RowSet expected = new RowSetBuilder(client.allocator(), schema).build(); diff --git a/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py new file mode 100644 index 00000000000..068cbab4e90 --- /dev/null +++ b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Generates the synthetic PCAP and PCAP-NG test fixtures. Run from any directory: + python3 contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py""" +import os +import socket +import struct + +HERE = os.path.dirname(os.path.abspath(__file__)) +PCAPNG = os.path.join(HERE, '..', 'pcapng') +PCAP = os.path.join(HERE, '..', 'pcap') +TS = 1704164645678000 # 2024-01-02T03:04:05.678Z in microseconds + + +def pad(b): + return b + b'\0' * (-len(b) % 4) + + +def opt(code, val, e='<'): + return struct.pack(e + 'HH', code, len(val)) + pad(val) + + +def opts(*o, e='<'): + return b''.join(o) + (struct.pack(e + 'HH', 0, 0) if o else b'') + + +def block(t, body, e='<'): + length = 12 + len(body) + return struct.pack(e + 'II', t, length) + body + struct.pack(e + 'I', length) + + +def shb(e='<'): + return block(0x0A0D0D0A, struct.pack(e + 'IHHq', 0x1A2B3C4D, 1, 0, -1), e) + + +def idb(link_type, *options, e='<'): + return block(1, struct.pack(e + 'HHI', link_type, 0, 65535) + opts(*options, e=e), e) + + +def epb(interface, data, ts=TS, e='<'): + return block(6, struct.pack(e + 'IIIII', interface, ts >> 32, ts & 0xffffffff, len(data), len(data)) + pad(data), e) + + +def ipv4(proto, payload, src, dst, ihl=5): + header = struct.pack('>BBHHHBBH4s4s', 0x40 | ihl, 0, 20 + len(payload), 0, 0, 64, proto, 0, + socket.inet_aton(src), socket.inet_aton(dst)) + return header + payload + + +def udp(sport, dport, data): + return struct.pack('>HHHH', sport, dport, 8 + len(data), 0) + data + + +def tcp(sport, dport, seq, flags, data, ack=0): + return struct.pack('>HHIIBBHHH', sport, dport, seq, ack, 5 << 4, flags, 65535, 0, 0) + data + + +def write(path, data): + with open(path, 'wb') as f: + f.write(data) + + +def echo(): + # Link type 101 (raw IP). Rows: parsed, not echo, parse failure, write failure, long text warning + f = shb() + idb(101) + for text in (b'ECHO:hello', b'hello', b'ECHO!bad', b'ECHO:write-fail', b'ECHO:a long echo text'): + f += epb(0, ipv4(17, udp(4000, 7, text), '10.0.0.1', '10.0.0.2')) + write(os.path.join(PCAPNG, 'echo.pcapng'), f) + + +def errors(): + # Packet errors that must not stop the file; a valid packet follows each one + good = ipv4(17, udp(4000, 7, b'ECHO:ok'), '10.0.0.1', '10.0.0.2') + f = shb() + idb(101) + f += idb(101, opt(9, bytes([0x7F]))) # interface 1: unsupported if_tsresol (10^-127) + f += epb(0, ipv4(17, udp(4000, 7, b'x'), '10.0.0.1', '10.0.0.2', ihl=3)) # malformed IPv4 header + f += epb(0, good) + f += epb(5, good) # undefined interface + f += epb(1, good) # interface with bad if_tsresol + print('errors.pcapng: bad captured length block at byte', len(f)) + bad = epb(0, good) + bad = bad[:20] + struct.pack('iuLU|{gI(UxKa(*L0VBnIM(0C6f<9KvP;vR!}}1m5zpF>KI|IG_w-VBi932Vy4v1q|#A{6INp4}YutY_MKxnZXXUlNx3L07=XubN~PV literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/pcapng/echo.pcapng b/contrib/format-pcapng/src/test/resources/pcapng/echo.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..a134d2806f98023afee8f944ad2c5f9227b1330e GIT binary patch literal 416 zcmZvWI|{-;7=(WkNI|OrR;%-Y-V1+2LBerJ+8Zu&i75WkOPSGQI@>C{dNP)<^{}?0mC@^SMM$7!gJ%lH|PsW-TG37rz zWC{#gmC-W{|9*=zJvoPQk~7HKv4AOAQ)CJ(T9q{{SfP`o*NdZ%QR-y+k>M^`KX=zV AfdBvi literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/pcapng/errors.pcapng b/contrib/format-pcapng/src/test/resources/pcapng/errors.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..dc0b6e64ebbf23a8f5d8d7bc3d9fc6c869096515 GIT binary patch literal 484 zcmd<$<>iuLU|{gI(UxKa(*L0VBnIM(0C6f<9Kr^PDIke)0#(%m`5>SG#B4z90K_2h zmY0oTgLcFLSs)vPogoy1gCGL~7tnYhX5wGKz|O$Qz)%5{cK~9LeitAH0hoSeARB~T sArz*5exRJQhrd;RHdrsr3|1 Date: Fri, 2 Oct 2026 14:46:07 -0400 Subject: [PATCH 07/50] DRILL-8556: Add decoder columns and never-fail errors to the PCAP reader --- .../exec/store/pcap/PcapBatchReader.java | 156 ++++++++++++------ .../pcap/plugin/BasePcapFormatPlugin.java | 10 +- .../drill/exec/store/pcap/schema/Schema.java | 8 +- .../exec/store/pcap/TestPcapDecoding.java | 85 ++++++++++ .../exec/store/pcap/TestPcapEVFReader.java | 31 ++-- .../test/resources/fixtures/pcap_fixtures.py | 28 ++++ .../src/test/resources/pcap/bad_record.pcap | Bin 0 -> 154 bytes .../src/test/resources/pcap/echo.pcap | Bin 0 -> 221 bytes .../src/test/resources/pcap/lfs_pointer.pcap | 3 + 9 files changed, 250 insertions(+), 71 deletions(-) create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java create mode 100644 contrib/format-pcapng/src/test/resources/pcap/bad_record.pcap create mode 100644 contrib/format-pcapng/src/test/resources/pcap/echo.pcap create mode 100644 contrib/format-pcapng/src/test/resources/pcap/lfs_pointer.pcap diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java index 379e5fceb6d..b2342d77e32 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java @@ -29,6 +29,8 @@ import org.apache.drill.exec.store.pcap.decoder.PacketDecoder; import org.apache.drill.exec.store.pcap.schema.Schema; import org.apache.drill.exec.store.pcap.plugin.PcapFormatConfig; +import org.apache.drill.exec.store.pcap.protocol.ProtocolColumns; +import org.apache.drill.exec.store.pcap.protocol.ProtocolDecoders; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +38,8 @@ import java.io.IOException; import java.io.InputStream; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import static org.apache.drill.exec.store.pcap.PcapFormatUtils.parseBytesToASCII; @@ -80,6 +84,12 @@ public class PcapBatchReader implements ManagedReader { private ScalarWriter isCorruptWriter; private final PcapFormatConfig readerConfig; private TcpSessionizer sessionizer; + private ProtocolColumns protocolColumns; + // Why the file cannot be read, reported once as an error row + private String fileError; + // Set when a damaged record makes the rest of the file unreadable + private boolean finished; + private int packetCount; public PcapBatchReader(PcapFormatConfig readerConfig, FileSchemaNegotiator negotiator) { @@ -89,7 +99,11 @@ public PcapBatchReader(PcapFormatConfig readerConfig, FileSchemaNegotiator negot openFile(); SchemaBuilder builder = new SchemaBuilder(); Schema pcapSchema = new Schema(readerConfig.getSessionizeTCPStreams()); - TupleMetadata schema = pcapSchema.buildSchema(builder); + pcapSchema.addColumns(builder); + if (!readerConfig.getSessionizeTCPStreams()) { + ProtocolColumns.addColumns(builder, ProtocolColumns.Mode.PACKET, ProtocolDecoders.get()); + } + TupleMetadata schema = builder.buildSchema(); negotiator.tableSchema(schema, false); ResultSetLoader loader = negotiator.build(); @@ -99,11 +113,23 @@ public PcapBatchReader(PcapFormatConfig readerConfig, FileSchemaNegotiator negot sessionizer = new TcpSessionizer(rowWriter); } else { populateColumnWriters(rowWriter); + protocolColumns = new ProtocolColumns(rowWriter, ProtocolColumns.Mode.PACKET, ProtocolDecoders.get(), + readerConfig.getExposeCredentials()); } } @Override public boolean next() { + if (fileError != null) { + if (protocolColumns != null) { + protocolColumns.writeErrorRow(fileError); + } + fileError = null; + return false; + } + if (decoder == null || finished) { + return false; + } while (!rowWriter.isFull()) { if (!parseNextPacket(rowWriter)) { // At end of file, more batches are needed only for open sessions that did not fit @@ -115,10 +141,13 @@ public boolean next() { @Override public void close() { - - + if (protocolColumns != null) { + protocolColumns.logSummary(logger, file.split().getPath().toString()); + } try { - fsStream.close(); + if (fsStream != null) { + fsStream.close(); + } } catch (IOException e) { throw UserException. dataReadError() @@ -136,11 +165,12 @@ private void openFile() { decoder = new PacketDecoder(fsStream); buffer = new byte[BUFFER_SIZE + decoder.getMaxLength()]; validBytes = fsStream.read(buffer); - } catch (IOException io) { - throw UserException - .dataReadError(io) - .addContext("File name:", file.split().getPath().toString()) - .build(logger); + } catch (IOException e) { + decoder = null; + fileError = "file: cannot read: " + ProtocolDecoders.describe(e); + } catch (RuntimeException e) { + decoder = null; + fileError = "file: not a PCAP or PCAP-NG file: " + ProtocolDecoders.describe(e); } } @@ -192,7 +222,18 @@ private boolean parseNextPacket(RowSetLoader rowWriter) { } int old = offset; - offset = decoder.decodePacket(buffer, offset, packet, decoder.getMaxLength(), validBytes); + try { + offset = decoder.decodePacket(buffer, offset, packet, decoder.getMaxLength(), validBytes); + } catch (RuntimeException e) { + // The record's length cannot be trusted, so neither can anything after it + finished = true; + if (protocolColumns != null) { + protocolColumns.writeErrorRow("file: invalid packet record after packet " + packetCount + ": " + + ProtocolDecoders.describe(e)); + } + return false; + } + packetCount++; if (offset > validBytes) { // Mark that the packet is corrupt and extract whatever data can be extracted from it packet.setIsCorrupt(true); @@ -238,54 +279,63 @@ private boolean getNextPacket(RowSetLoader rowWriter) { private void addDataToTable(Packet packet, int networkType, RowSetLoader rowWriter) { rowWriter.start(); - - typeWriter.setString(packet.getPacketType()); - timestampWriter.setTimestamp(Instant.ofEpochMilli(packet.getTimestamp())); - timestampMicroWriter.setLong(packet.getTimestampMicro()); - networkWriter.setInt(networkType); - srcMacAddressWriter.setString(packet.getEthernetSource()); - dstMacAddressWriter.setString(packet.getEthernetDestination()); - - String destinationIp = packet.getDestinationIpAddressString(); - if (destinationIp == null) { - dstIPWriter.setNull(); - } else { - dstIPWriter.setString(destinationIp); + List errors = new ArrayList<>(); + if (packet.getDecodeError() != null) { + errors.add("packet: " + packet.getDecodeError()); } + try { + typeWriter.setString(packet.getPacketType()); + timestampWriter.setTimestamp(Instant.ofEpochMilli(packet.getTimestamp())); + timestampMicroWriter.setLong(packet.getTimestampMicro()); + networkWriter.setInt(networkType); + srcMacAddressWriter.setString(packet.getEthernetSource()); + dstMacAddressWriter.setString(packet.getEthernetDestination()); - String sourceIp = packet.getSourceIpAddressString(); - if (sourceIp == null) { - srcIPWriter.setNull(); - } else { - srcIPWriter.setString(sourceIp); - } - srcPortWriter.setInt(packet.getSrc_port()); - dstPortWriter.setInt(packet.getDst_port()); - packetLengthWriter.setInt(packet.getPacketLength()); + String destinationIp = packet.getDestinationIpAddressString(); + if (destinationIp == null) { + dstIPWriter.setNull(); + } else { + dstIPWriter.setString(destinationIp); + } - // TCP Only Packet Data - tcpSessionWriter.setLong(packet.getSessionHash()); - tcpSequenceWriter.setInt(packet.getSequenceNumber()); - tcpAckNumberWriter.setInt(packet.getAckNumber()); - tcpFlagsWriter.setInt(packet.getFlags()); - tcpParsedFlagsWriter.setString(packet.getParsedFlags()); + String sourceIp = packet.getSourceIpAddressString(); + if (sourceIp == null) { + srcIPWriter.setNull(); + } else { + srcIPWriter.setString(sourceIp); + } + srcPortWriter.setInt(packet.getSrc_port()); + dstPortWriter.setInt(packet.getDst_port()); + packetLengthWriter.setInt(packet.getPacketLength()); + + // TCP Only Packet Data + tcpSessionWriter.setLong(packet.getSessionHash()); + tcpSequenceWriter.setInt(packet.getSequenceNumber()); + tcpAckNumberWriter.setInt(packet.getAckNumber()); + tcpFlagsWriter.setInt(packet.getFlags()); + tcpParsedFlagsWriter.setString(packet.getParsedFlags()); - // TCP Flags - tcpNsWriter.setBoolean((packet.getFlags() & 0x100) != 0); - tcpCwrWriter.setBoolean((packet.getFlags() & 0x80) != 0); - tcpEceWriter.setBoolean((packet.getFlags() & 0x40) != 0); - tcpFlagsEceEcnCapableWriter.setBoolean((packet.getFlags() & 0x42) == 0x42); - tcpFlagsCongestionWriter.setBoolean((packet.getFlags() & 0x42) == 0x40); - tcpUrgWriter.setBoolean((packet.getFlags() & 0x20) != 0); - tcpAckWriter.setBoolean((packet.getFlags() & 0x10) != 0); - tcpPshWriter.setBoolean((packet.getFlags() & 0x8) != 0); - tcpRstWriter.setBoolean((packet.getFlags() & 0x4) != 0); - tcpSynWriter.setBoolean((packet.getFlags() & 0x2) != 0); - tcpFinWriter.setBoolean((packet.getFlags() & 0x1) != 0); + // TCP Flags + tcpNsWriter.setBoolean((packet.getFlags() & 0x100) != 0); + tcpCwrWriter.setBoolean((packet.getFlags() & 0x80) != 0); + tcpEceWriter.setBoolean((packet.getFlags() & 0x40) != 0); + tcpFlagsEceEcnCapableWriter.setBoolean((packet.getFlags() & 0x42) == 0x42); + tcpFlagsCongestionWriter.setBoolean((packet.getFlags() & 0x42) == 0x40); + tcpUrgWriter.setBoolean((packet.getFlags() & 0x20) != 0); + tcpAckWriter.setBoolean((packet.getFlags() & 0x10) != 0); + tcpPshWriter.setBoolean((packet.getFlags() & 0x8) != 0); + tcpRstWriter.setBoolean((packet.getFlags() & 0x4) != 0); + tcpSynWriter.setBoolean((packet.getFlags() & 0x2) != 0); + tcpFinWriter.setBoolean((packet.getFlags() & 0x1) != 0); - // Note: getData() MUST be called before isCorrupt - dataWriter.setString(parseBytesToASCII(packet.getData())); - isCorruptWriter.setBoolean(packet.isCorrupt()); + // Note: getData() MUST be called before isCorrupt + dataWriter.setString(parseBytesToASCII(packet.getData())); + isCorruptWriter.setBoolean(packet.isCorrupt()); + } catch (RuntimeException e) { + // Keep the fields written so far + errors.add("packet: " + ProtocolDecoders.describe(e)); + } + protocolColumns.writePacket(packet, errors); rowWriter.save(); // TODO Parse Data Packet Here: diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java index c5798994f9e..77b459623a9 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.pcap.plugin; -import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; @@ -129,11 +128,10 @@ private static PacketDecoder.FileFormat getFileFormat(DrillFileSystem dfs, Path try (InputStream inputStream = dfs.openPossiblyCompressedStream(path)) { PacketDecoder decoder = new PacketDecoder(inputStream); return decoder.getFileFormat(); - } catch (IOException io) { - throw UserException - .dataReadError(io) - .addContext("File name:", path.toString()) - .build(logger); + } catch (IOException | RuntimeException e) { + // Not readable or not a capture: the PCAP reader reports it as an error row + logger.debug("Cannot detect the format of {}", path, e); + return PacketDecoder.FileFormat.UNKNOWN; } } } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/schema/Schema.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/schema/Schema.java index 0e85720c159..08b5852739a 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/schema/Schema.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/schema/Schema.java @@ -116,10 +116,14 @@ public int getNumberOfColumns() { } public TupleMetadata buildSchema(SchemaBuilder builder) { + addColumns(builder); + return builder.buildSchema(); + } + + /** Adds the columns to a schema that the caller may extend before building it. */ + public void addColumns(SchemaBuilder builder) { for (ColumnDto column : columns) { builder.addNullable(column.getColumnName(), typeMap[column.getColumnType().ordinal()]); } - - return builder.buildSchema(); } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java new file mode 100644 index 00000000000..302dd3123ab --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.file.Paths; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.physical.rowSet.RowSet; +import org.apache.drill.exec.physical.rowSet.RowSetBuilder; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.ClusterTest; +import org.apache.drill.test.rowSet.RowSetComparison; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestPcapDecoding extends ClusterTest { + + @BeforeClass + public static void setup() throws Exception { + ClusterTest.startCluster(ClusterFixture.builder(dirTestWatcher)); + dirTestWatcher.copyResourceToRoot(Paths.get("pcap/")); + } + + @Test + public void testDecoderColumns() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.echo_test.text as text, decode_error from dfs.`pcap/echo.pcap` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("text", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("echo_test", "hello", null) + .addRow(null, null, null) + .addRow("echo_test", null, "echo_test: bad echo") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testNotACaptureFileReturnsErrorRow() throws Exception { + String sql = "select src_ip, decode_error from dfs.`pcap/lfs_pointer.pcap`"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("src_ip", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow(null, "file: not a PCAP or PCAP-NG file: Bad magic number = 73726576") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testBadRecordStopsWithErrorRow() throws Exception { + String sql = "select src_ip, decode_error from dfs.`pcap/bad_record.pcap`"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + assertEquals(2, results.rowCount()); + results.clear(); + sql = "select decode_error from dfs.`pcap/bad_record.pcap` where decode_error is not null"; + String error = client.queryBuilder().sql(sql).singletonString(); + assertTrue(error, error.startsWith("file: invalid packet record after packet 1: Packet too long")); + } +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapEVFReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapEVFReader.java index aa48a360c7f..75d94c2c2ed 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapEVFReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapEVFReader.java @@ -18,7 +18,11 @@ package org.apache.drill.exec.store.pcap; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import org.apache.drill.categories.RowSetTest; +import org.apache.drill.exec.physical.rowSet.RowSet; import org.apache.drill.exec.store.pcap.plugin.PcapFormatConfig; import org.apache.drill.test.ClusterFixture; import org.apache.drill.test.ClusterTest; @@ -26,6 +30,9 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import java.time.Month; @@ -40,17 +47,21 @@ public static void setup() throws Exception { @Test public void testStarQuery() throws Exception { + // Values are pinned by testExplicitAllQuery; this checks what select * returns String sql = "SELECT * FROM cp.`pcap/synscan.pcap` LIMIT 1"; - - testBuilder() - .sqlQuery(sql) - .unOrdered() - .baselineColumns("type", "packet_timestamp", "timestamp_micro", "network", "src_mac_address", "dst_mac_address", "dst_ip", "src_ip", "src_port", "dst_port", "packet_length", - "tcp_session", "tcp_sequence", "tcp_ack", "tcp_flags", "tcp_parsed_flags", "tcp_flags_ns", "tcp_flags_cwr", "tcp_flags_ece", "tcp_flags_ece_ecn_capable", "tcp_flags_ece_congestion_experienced", "tcp_flags_urg", "tcp_flags_ack", "tcp_flags_psh", "tcp_flags_rst", "tcp_flags_syn", "tcp_flags_fin", "data", "is_corrupt") - .baselineValues("TCP", LocalDateTime.of(2010, Month.JULY, 4, 20, 24, 16, 274000000), 1278275056274870L, 1, "00:25:B3:BF:91:EE", "00:26:0B:31:07:33", - "64.13.134.52", "172.16.0.8", 36050, 443, 58, - 317740574511239903L, -581795048, false, 2,"SYN", false, false, false, false, false, false, false, false, false, true, false, "[]", false) - .go(); + RowSet results = client.queryBuilder().sql(sql).rowSet(); + Set names = new HashSet<>(); + results.schema().toMetadataList().forEach(c -> names.add(c.name())); + Set expected = new HashSet<>(Arrays.asList("type", "packet_timestamp", "timestamp_micro", "network", + "src_mac_address", "dst_mac_address", "dst_ip", "src_ip", "src_port", "dst_port", "packet_length", + "tcp_session", "tcp_sequence", "tcp_ack", "tcp_flags", "tcp_parsed_flags", "tcp_flags_ns", "tcp_flags_cwr", + "tcp_flags_ece", "tcp_flags_ece_ecn_capable", "tcp_flags_ece_congestion_experienced", "tcp_flags_urg", + "tcp_flags_ack", "tcp_flags_psh", "tcp_flags_rst", "tcp_flags_syn", "tcp_flags_fin", "data", "is_corrupt", + "parsed_protocol", "parsed_data", "decode_error")); + assertEquals(expected, names); + assertTrue(results.schema().metadata("parsed_data").isMap()); + assertEquals(1, results.rowCount()); + results.clear(); } @Test diff --git a/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py index 068cbab4e90..028ee88b617 100644 --- a/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py +++ b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py @@ -110,7 +110,35 @@ def bad_block_length(): write(os.path.join(PCAPNG, 'bad_block_length.pcapng'), f) +ETHERNET = bytes.fromhex('020000000002' '020000000001' '0800') + + +def pcap_file(path, frames, link_type=1): + data = struct.pack('D1!tX1Q{5(fQo^biGKkD bI|DxhgR8TLzg2!VSSe6RfgmFolWYJ0jr0x& literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/pcap/echo.pcap b/contrib/format-pcapng/src/test/resources/pcap/echo.pcap new file mode 100644 index 0000000000000000000000000000000000000000..7fccb9fce098b340e029e0bd8f44992b53c222d1 GIT binary patch literal 221 zcmca|c+)~A1{MYw`2U}Qff2}2t(}|-5;K8fCJ=`SOfqsXxH2%Pfdm``85p>L8i1II ze*ptKgAfCQtFwo{RYqz~PQDt{41J(V5XLY=5vv)z3=ClH>QL=QKvf`&p Date: Fri, 2 Oct 2026 14:48:55 -0400 Subject: [PATCH 08/50] DRILL-8556: Reassemble TCP streams for session protocol decoders --- .../exec/store/pcap/PcapBatchReader.java | 12 +- .../drill/exec/store/pcap/TcpSessionizer.java | 6 +- .../exec/store/pcap/decoder/TcpSession.java | 8 ++ .../store/pcap/protocol/DecodeResult.java | 6 +- .../store/pcap/protocol/ProtocolColumns.java | 13 +- .../store/pcap/protocol/ProtocolDecoders.java | 69 ++++++++- .../pcap/protocol/SessionProtocolDecoder.java | 45 ++++++ .../exec/store/pcap/protocol/TcpStream.java | 131 ++++++++++++++++++ .../exec/store/pcapng/PcapngBatchReader.java | 13 +- .../exec/store/pcap/TestSessionizePCAP.java | 64 +++------ .../store/pcap/protocol/TestTcpStream.java | 105 ++++++++++++++ 11 files changed, 408 insertions(+), 64 deletions(-) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/SessionProtocolDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/TcpStream.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestTcpStream.java diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java index b2342d77e32..99fe4b40e4e 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java @@ -100,21 +100,21 @@ public PcapBatchReader(PcapFormatConfig readerConfig, FileSchemaNegotiator negot SchemaBuilder builder = new SchemaBuilder(); Schema pcapSchema = new Schema(readerConfig.getSessionizeTCPStreams()); pcapSchema.addColumns(builder); - if (!readerConfig.getSessionizeTCPStreams()) { - ProtocolColumns.addColumns(builder, ProtocolColumns.Mode.PACKET, ProtocolDecoders.get()); - } + ProtocolColumns.Mode mode = readerConfig.getSessionizeTCPStreams() + ? ProtocolColumns.Mode.SESSION : ProtocolColumns.Mode.PACKET; + ProtocolColumns.addColumns(builder, mode, ProtocolDecoders.get()); TupleMetadata schema = builder.buildSchema(); negotiator.tableSchema(schema, false); ResultSetLoader loader = negotiator.build(); // Creates writers for all fields (Since schema is known) rowWriter = loader.writer(); + protocolColumns = new ProtocolColumns(rowWriter, mode, ProtocolDecoders.get(), + readerConfig.getExposeCredentials()); if (readerConfig.getSessionizeTCPStreams()) { - sessionizer = new TcpSessionizer(rowWriter); + sessionizer = new TcpSessionizer(rowWriter, protocolColumns); } else { populateColumnWriters(rowWriter); - protocolColumns = new ProtocolColumns(rowWriter, ProtocolColumns.Mode.PACKET, ProtocolDecoders.get(), - readerConfig.getExposeCredentials()); } } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java index e9993dbf416..22ce71078f7 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java @@ -26,6 +26,7 @@ import org.apache.drill.exec.physical.resultSet.RowSetLoader; import org.apache.drill.exec.store.pcap.decoder.Packet; import org.apache.drill.exec.store.pcap.decoder.TcpSession; +import org.apache.drill.exec.store.pcap.protocol.ProtocolColumns; import org.apache.drill.exec.vector.accessor.ScalarWriter; /** @@ -39,6 +40,7 @@ public class TcpSessionizer { // Sessions already written, so the ACKs that trail a FIN do not start a new one private final Set closedSessions = new HashSet<>(); private final RowSetLoader rowWriter; + private final ProtocolColumns protocolColumns; private final ScalarWriter srcMacAddressWriter; private final ScalarWriter dstMacAddressWriter; private final ScalarWriter dstIPWriter; @@ -60,8 +62,9 @@ public class TcpSessionizer { private final ScalarWriter isCorruptWriter; private final ScalarWriter sessionClosedWriter; - public TcpSessionizer(RowSetLoader rowWriter) { + public TcpSessionizer(RowSetLoader rowWriter, ProtocolColumns protocolColumns) { this.rowWriter = rowWriter; + this.protocolColumns = protocolColumns; srcMacAddressWriter = rowWriter.scalar("src_mac_address"); dstMacAddressWriter = rowWriter.scalar("dst_mac_address"); dstIPWriter = rowWriter.scalar("dst_ip"); @@ -156,6 +159,7 @@ private void write(TcpSession session) { hostDataWriter.setString(session.getDataFromOriginatorAsString()); remoteDataWriter.setString(session.getDataFromRemoteAsString()); + protocolColumns.writeSession(session); rowWriter.save(); } } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java index 2fa945b39d0..df46c9787ab 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java @@ -190,6 +190,14 @@ public boolean connectionEstablished() { return handshake.isConnected(); } + public List getPacketsFromSender() { + return Collections.unmodifiableList(packetsFromSender); + } + + public List getPacketsFromReceiver() { + return Collections.unmodifiableList(packetsFromReceiver); + } + public boolean connectionClosed() { return handshake.isClosed(); } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java index 9e32b547043..677c03883aa 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java @@ -55,6 +55,10 @@ public String error() { @SuppressWarnings({"unchecked", "rawtypes"}) public void write(TupleWriter fields) { - ((PacketProtocolDecoder) decoder).write(parsed, fields); + if (decoder instanceof SessionProtocolDecoder) { + ((SessionProtocolDecoder) decoder).write(parsed, fields); + } else { + ((PacketProtocolDecoder) decoder).write(parsed, fields); + } } } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java index 104b48e8bfc..cf88f58b770 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.TupleWriter; import org.slf4j.Logger; @@ -62,7 +63,7 @@ public static void addColumns(SchemaBuilder schema, Mode mode, ProtocolDecoders } private static TupleMetadata dataSchema(Mode mode, ProtocolDecoders decoders) { - return decoders.packetDataSchema(); + return mode == Mode.SESSION ? decoders.sessionDataSchema() : decoders.packetDataSchema(); } public ProtocolColumns(RowSetLoader loader, Mode mode, ProtocolDecoders decoders, boolean exposeCredentials) { @@ -107,6 +108,16 @@ public void writePacket(Packet packet, List errors) { writeErrors(all); } + /** Decodes a session into the current row and writes decode_error. */ + public void writeSession(TcpSession session) { + List all = new ArrayList<>(); + if (decoding) { + context.reset(); + write(decoders.decodeSession(session, context), all); + } + writeErrors(all); + } + void write(DecodeResult result, List all) { if (result != null) { protocolWriter.setString(result.protocol()); diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java index b6f7c98d287..41e7988f3a8 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java @@ -29,6 +29,7 @@ import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +43,8 @@ public class ProtocolDecoders { private final List> packetDecoders; private final TupleMetadata packetDataSchema; + private final List> sessionDecoders; + private final TupleMetadata sessionDataSchema; /** The decoders on the classpath, loaded once. */ @SuppressWarnings({"unchecked", "rawtypes"}) @@ -50,7 +53,8 @@ public static ProtocolDecoders get() { synchronized (ProtocolDecoders.class) { if (instance == null) { ClassLoader loader = ProtocolDecoders.class.getClassLoader(); - instance = new ProtocolDecoders((Iterable) ServiceLoader.load(PacketProtocolDecoder.class, loader)); + instance = new ProtocolDecoders((Iterable) ServiceLoader.load(PacketProtocolDecoder.class, loader), + (Iterable) ServiceLoader.load(SessionProtocolDecoder.class, loader)); } } } @@ -58,9 +62,17 @@ public static ProtocolDecoders get() { } public ProtocolDecoders(Iterable> packetDecoders) { - SchemaBuilder data = new SchemaBuilder(); - this.packetDecoders = usable(packetDecoders, data); - this.packetDataSchema = data.buildSchema(); + this(packetDecoders, Collections.emptyList()); + } + + public ProtocolDecoders(Iterable> packetDecoders, + Iterable> sessionDecoders) { + SchemaBuilder packetData = new SchemaBuilder(); + this.packetDecoders = usable(packetDecoders, packetData); + this.packetDataSchema = packetData.buildSchema(); + SchemaBuilder sessionData = new SchemaBuilder(); + this.sessionDecoders = usable(sessionDecoders, sessionData); + this.sessionDataSchema = sessionData.buildSchema(); } public List> packetDecoders() { @@ -72,6 +84,49 @@ public TupleMetadata packetDataSchema() { return packetDataSchema; } + public List> sessionDecoders() { + return sessionDecoders; + } + + /** Members of parsed_data in session mode: one map per decoder. */ + public TupleMetadata sessionDataSchema() { + return sessionDataSchema; + } + + /** + * Finds the decoder that handles a TCP session. The client and server streams are + * reassembled only if some decoder accepts the session. + * + * @return null if no decoder handles it + */ + public DecodeResult decodeSession(TcpSession session, RowDecoderContext context) { + TcpStream[] streams = null; + for (SessionProtocolDecoder decoder : sessionDecoders) { + try { + if (!decoder.accepts(session)) { + continue; + } + } catch (RuntimeException e) { + continue; + } + context.begin(decoder.protocol()); + try { + if (streams == null) { + streams = TcpStream.clientServer(session); + } + Object parsed = decoder.parse(streams[0], streams[1], context); + if (parsed != null) { + return DecodeResult.success(decoder, parsed); + } + context.discard(); + } catch (RuntimeException e) { + context.discard(); + return DecodeResult.failure(decoder, decoder.protocol() + ": " + describe(e)); + } + } + return null; + } + /** * Finds the decoder that handles a packet. * @@ -138,6 +193,10 @@ private static List usable(Iterable } private static void defineSchema(ProtocolDecoder decoder, SchemaBuilder fields) { - ((PacketProtocolDecoder) decoder).defineSchema(fields); + if (decoder instanceof SessionProtocolDecoder) { + ((SessionProtocolDecoder) decoder).defineSchema(fields); + } else { + ((PacketProtocolDecoder) decoder).defineSchema(fields); + } } } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/SessionProtocolDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/SessionProtocolDecoder.java new file mode 100644 index 00000000000..30d1550b356 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/SessionProtocolDecoder.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** + * Decodes reassembled TCP sessions. Implementations are listed in + * META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder. + * + * @param the parsed form of one session + */ +public interface SessionProtocolDecoder extends ProtocolDecoder { + + /** Declares the fields of parsed_data.<protocol> for session rows. */ + void defineSchema(SchemaBuilder fields); + + /** Cheap pre-check, typically ports. */ + boolean accepts(TcpSession session); + + /** + * @return null if the session is not this protocol + * @throws RuntimeException if the session is this protocol but malformed + */ + T parse(TcpStream fromClient, TcpStream fromServer, DecoderContext context); + + void write(T parsed, TupleWriter fields); +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/TcpStream.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/TcpStream.java new file mode 100644 index 00000000000..b97f189254a --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/TcpStream.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; + +/** + * The reassembled bytes of one direction of a TCP session: ordered by sequence number, + * retransmissions removed, and cut at the first missing range. + */ +public final class TcpStream { + private final byte[] data; + private final long firstGap; + + private TcpStream(byte[] data, long firstGap) { + this.data = data; + this.firstGap = firstGap; + } + + /** Contiguous bytes from the start of the stream up to the first gap. */ + public byte[] data() { + return data; + } + + public boolean hasGaps() { + return firstGap >= 0; + } + + /** Offset of the first missing byte, or -1. */ + public long firstGap() { + return firstGap; + } + + /** The client stream and the server stream of a session, in that order. */ + public static TcpStream[] clientServer(TcpSession session) { + List sender = session.getPacketsFromSender(); + List receiver = session.getPacketsFromReceiver(); + boolean senderIsClient; + if (hasInitialSyn(receiver)) { + senderIsClient = false; + } else if (hasInitialSyn(sender)) { + senderIsClient = true; + } else { + // No handshake captured: the endpoint with the lower port is taken as the server + senderIsClient = session.getSrcPort() > session.getDstPort(); + } + TcpStream fromSender = of(sender); + TcpStream fromReceiver = of(receiver); + return senderIsClient ? new TcpStream[] {fromSender, fromReceiver} : new TcpStream[] {fromReceiver, fromSender}; + } + + private static boolean hasInitialSyn(List packets) { + for (Packet p : packets) { + if (p.getSynFlag() && !p.getAckFlag()) { + return true; + } + } + return false; + } + + public static TcpStream of(List packets) { + if (packets.isEmpty()) { + return new TcpStream(new byte[0], -1); + } + // Sequence numbers relative to the first packet, as signed 32-bit differences, handle wraparound + long reference = packets.get(0).getSequenceNumber() & 0xFFFFFFFFL; + Integer base = null; + for (Packet p : packets) { + if (p.getSynFlag()) { + base = relative(p, reference) + 1; + break; + } + } + List ranges = new ArrayList<>(); + List payloads = new ArrayList<>(); + for (Packet p : packets) { + byte[] payload = p.getData(); + if (payload == null || payload.length == 0) { + continue; + } + ranges.add(new long[] {relative(p, reference), payloads.size()}); + payloads.add(payload); + } + if (payloads.isEmpty()) { + return new TcpStream(new byte[0], -1); + } + ranges.sort((a, b) -> Long.compare(a[0], b[0])); + long start = base != null ? base : ranges.get(0)[0]; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + long next = start; + for (long[] range : ranges) { + byte[] payload = payloads.get((int) range[1]); + long segmentStart = range[0]; + long segmentEnd = segmentStart + payload.length; + if (segmentEnd <= next) { + continue; // retransmission of bytes already taken + } + if (segmentStart > next) { + return new TcpStream(out.toByteArray(), next - start); + } + int skip = (int) (next - segmentStart); + out.write(payload, skip, payload.length - skip); + next = segmentEnd; + } + return new TcpStream(out.toByteArray(), -1); + } + + private static int relative(Packet p, long reference) { + return (int) ((p.getSequenceNumber() & 0xFFFFFFFFL) - reference); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java index ad52bcc82e5..ea0cd4faec8 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java @@ -125,9 +125,14 @@ public PcapngBatchReader(final PcapFormatConfig config, final EasySubScan scan, .build(logger); } if (isSessionQuery()) { - negotiator.tableSchema(new Schema(true).buildSchema(new SchemaBuilder()), false); + SchemaBuilder builder = new SchemaBuilder(); + new Schema(true).addColumns(builder); + ProtocolColumns.addColumns(builder, ProtocolColumns.Mode.SESSION, ProtocolDecoders.get()); + negotiator.tableSchema(builder.buildSchema(), false); loader = negotiator.build().writer(); - sessionizer = new TcpSessionizer(loader); + protocolColumns = new ProtocolColumns(loader, ProtocolColumns.Mode.SESSION, ProtocolDecoders.get(), + config.getExposeCredentials()); + sessionizer = new TcpSessionizer(loader, protocolColumns); return; } // define the schema @@ -176,7 +181,9 @@ public boolean next() { return sessionizer != null && !sessionizer.writeOpenSessions(); } if (sessionizer != null) { - if (block.packet != null) { + if (block.errorRow) { + protocolColumns.writeErrorRow(String.join("; ", block.errors)); + } else if (block.packet != null) { sessionizer.addPacket(block.packet); } } else { diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java index aa88a7bdfbb..5daab6673fa 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java @@ -34,10 +34,14 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import org.junit.BeforeClass; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class TestSessionizePCAP extends ClusterTest { @@ -52,54 +56,20 @@ public static void setup() throws Exception { @Test public void testSessionizedStarQuery() throws Exception { + // Values are pinned by testSessionizedSpecificQuery; this checks what select * returns String sql = "SELECT * FROM cp.`/pcap/http.pcap` WHERE session_closed = true"; - String dataFromRemote = readAFileIntoString(dirTestWatcher.getRootDir().getAbsolutePath() + "/pcap/dataFromRemote.txt"); - - QueryBuilder q = client.queryBuilder().sql(sql); - RowSet results = q.rowSet(); - - TupleMetadata expectedSchema = new SchemaBuilder() - .addNullable("src_ip", TypeProtos.MinorType.VARCHAR) - .addNullable("dst_ip", TypeProtos.MinorType.VARCHAR) - .addNullable("src_port", TypeProtos.MinorType.INT) - .addNullable("dst_port", TypeProtos.MinorType.INT) - .addNullable("src_mac_address", TypeProtos.MinorType.VARCHAR) - .addNullable("dst_mac_address", TypeProtos.MinorType.VARCHAR) - .addNullable("session_start_time", TypeProtos.MinorType.TIMESTAMP) - .addNullable("session_end_time", TypeProtos.MinorType.TIMESTAMP) - .addNullable("session_duration", TypeProtos.MinorType.INTERVAL) - .addNullable("total_packet_count", TypeProtos.MinorType.INT) - .addNullable("data_volume_from_origin", TypeProtos.MinorType.INT) - .addNullable("data_volume_from_remote", TypeProtos.MinorType.INT) - .addNullable("packet_count_from_origin", TypeProtos.MinorType.INT) - .addNullable("packet_count_from_remote", TypeProtos.MinorType.INT) - .addNullable("connection_time", TypeProtos.MinorType.INTERVAL) - .addNullable("tcp_session", TypeProtos.MinorType.BIGINT) - .addNullable("is_corrupt", TypeProtos.MinorType.BIT) - .addNullable("session_closed", TypeProtos.MinorType.BIT) - .addNullable("data_from_originator", TypeProtos.MinorType.VARCHAR) - .addNullable("data_from_remote", TypeProtos.MinorType.VARCHAR) - .buildSchema(); - - RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema) - .addRow( - "145.254.160.237", - "65.208.228.223", - 3372, 80, - "00:00:01:00:00:00", - "FE:FF:20:00:01:00", - 1084443427311L, - 1084443445216L, - Period.parse("PT17.905S"), 31, - 479, 18364, 14, 17, - Period.parse("PT0.911S"), - -789689725566200012L, false, true, - "GET /download.html HTTP/1.1..Host: www.ethereal.com..User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113..Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1..Accept-Language: en-us,en;q=0.5..Accept-Encoding: gzip,deflate..Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7..Keep-Alive: 300..Connection: keep-alive..Referer: http://www.ethereal.com/development.html", - dataFromRemote - ) - .build(); - - new RowSetComparison(expected).verifyAndClearAll(results); + RowSet results = client.queryBuilder().sql(sql).rowSet(); + Set names = new HashSet<>(); + results.schema().toMetadataList().forEach(c -> names.add(c.name())); + Set expected = new HashSet<>(Arrays.asList("src_ip", "dst_ip", "src_port", "dst_port", + "src_mac_address", "dst_mac_address", "session_start_time", "session_end_time", "session_duration", + "total_packet_count", "data_volume_from_origin", "data_volume_from_remote", "packet_count_from_origin", + "packet_count_from_remote", "connection_time", "tcp_session", "is_corrupt", "session_closed", + "data_from_originator", "data_from_remote", "parsed_protocol", "parsed_data", "decode_error")); + assertEquals(expected, names); + assertTrue(results.schema().metadata("parsed_data").isMap()); + assertEquals(1, results.rowCount()); + results.clear(); } @Test diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestTcpStream.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestTcpStream.java new file mode 100644 index 00000000000..56378d66ca2 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestTcpStream.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestTcpStream extends BaseTest { + + private static Packet seg(long seq, String data) { + return TestPackets.tcp("10.0.0.1", 40000, "10.0.0.2", 80, seq, TestPackets.ACK, data.getBytes(StandardCharsets.US_ASCII)); + } + + private static String text(TcpStream stream) { + return new String(stream.data(), StandardCharsets.US_ASCII); + } + + @Test + public void testOutOfOrderAndRetransmitted() { + TcpStream stream = TcpStream.of(Arrays.asList(seg(1006, "world"), seg(1000, "hello "), seg(1000, "hello "), seg(1003, "lo w"))); + assertEquals("hello world", text(stream)); + assertFalse(stream.hasGaps()); + assertEquals(-1, stream.firstGap()); + } + + @Test + public void testGapStopsData() { + TcpStream stream = TcpStream.of(Arrays.asList(seg(1000, "abc"), seg(1010, "xyz"))); + assertEquals("abc", text(stream)); + assertTrue(stream.hasGaps()); + assertEquals(3, stream.firstGap()); + } + + @Test + public void testSequenceWraparound() { + TcpStream stream = TcpStream.of(Arrays.asList(seg(0xFFFFFFFEL, "ab"), seg(0, "cd"))); + assertEquals("abcd", text(stream)); + } + + @Test + public void testSynConsumesOneSequenceNumber() { + Packet syn = TestPackets.tcp("10.0.0.1", 40000, "10.0.0.2", 80, 999, TestPackets.SYN, new byte[0]); + TcpStream stream = TcpStream.of(Arrays.asList(seg(1005, "later"), syn, seg(1000, "first"))); + assertEquals("firstlater", text(stream)); + } + + @Test + public void testEmpty() { + assertArrayEquals(new byte[0], TcpStream.of(Arrays.asList()).data()); + } + + @Test + public void testClientIsSynSender() { + // First packet seen is from the server; the SYN identifies the client + Packet fromServer = TestPackets.tcp("10.0.0.2", 80, "10.0.0.1", 40000, 5000, TestPackets.ACK, + "HTTP/1.1 200 OK\r\n\r\n".getBytes(StandardCharsets.US_ASCII)); + Packet syn = TestPackets.tcp("10.0.0.1", 40000, "10.0.0.2", 80, 999, TestPackets.SYN, new byte[0]); + Packet request = seg(1000, "GET / HTTP/1.1\r\n\r\n"); + TcpSession session = new TcpSession(fromServer.getSessionHash()); + session.addPacket(fromServer); + session.addPacket(syn); + session.addPacket(request); + TcpStream[] streams = TcpStream.clientServer(session); + assertTrue(text(streams[0]).startsWith("GET")); + assertTrue(text(streams[1]).startsWith("HTTP")); + } + + @Test + public void testServerIsLowerPortWithoutHandshake() { + Packet fromServer = TestPackets.tcp("10.0.0.2", 80, "10.0.0.1", 40000, 5000, TestPackets.ACK, + "HTTP/1.1 200 OK\r\n\r\n".getBytes(StandardCharsets.US_ASCII)); + Packet request = seg(1000, "GET / HTTP/1.1\r\n\r\n"); + TcpSession session = new TcpSession(fromServer.getSessionHash()); + session.addPacket(fromServer); + session.addPacket(request); + TcpStream[] streams = TcpStream.clientServer(session); + assertTrue(text(streams[0]).startsWith("GET")); + assertEquals(2, streams.length); + } +} From 5315771484916ad33afe45b829780e532c040ed4 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:01:47 -0400 Subject: [PATCH 09/50] DRILL-8556: Keep map arrays nested in maps intact in scan output OutputBatchBuilder rebuilt any nested column whose metadata isMap() as a new map vector, including map arrays. A map array inside a map lost its offsets, so the batch failed validation (row count n, value count 0). --- .../scan/v3/lifecycle/OutputBatchBuilder.java | 5 ++-- .../v3/lifecycle/TestOutputBatchBuilder.java | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/v3/lifecycle/OutputBatchBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/v3/lifecycle/OutputBatchBuilder.java index 3ba58daa0ad..59215df57bd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/v3/lifecycle/OutputBatchBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/v3/lifecycle/OutputBatchBuilder.java @@ -178,7 +178,8 @@ private void defineSourceMapMapping(TupleMetadata sourceSchema, int source) { int outputIndex = mapSchema.index(col.name()); Preconditions.checkState(outputIndex >= 0); VectorSource vectorSource = new VectorSource(source, i); - if (col.isMap()) { + // A map array carries its own offsets, so it is passed through whole like a scalar + if (col.isMap() && !col.isArray()) { if (memberSources[outputIndex] == null) { memberSources[outputIndex] = new ArrayList(); } @@ -202,7 +203,7 @@ public AbstractMapVector build(BufferAllocator allocator) { for (int i = 0; i < mapSchema.size(); i++) { ColumnMetadata outputCol = mapSchema.metadata(i); ValueVector outputVector; - if (outputCol.isMap()) { + if (outputCol.isMap() && !outputCol.isArray()) { outputVector = buildNestedMap(allocator, outputCol, (List) memberSources[i]); } else { outputVector = getMember((VectorSource) memberSources[i]); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/lifecycle/TestOutputBatchBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/lifecycle/TestOutputBatchBuilder.java index 09f62fc7731..97d49d41b62 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/lifecycle/TestOutputBatchBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/lifecycle/TestOutputBatchBuilder.java @@ -327,6 +327,36 @@ public void testNestedMap() { RowSetUtilities.verify(fixture.wrap(input), fixture.wrap(output)); } + @Test + public void testMapArrayInMap() { + // A repeated map inside a map must keep its offsets, not just its members + final TupleMetadata schema = new SchemaBuilder() + .add("a", MinorType.VARCHAR) + .addMap("m") + .add("x", MinorType.INT) + .addMapArray("y") + .add("p", MinorType.INT) + .add("q", MinorType.VARCHAR) + .resumeMap() + .resumeSchema() + .buildSchema(); + + final VectorContainer input = fixture.rowSetBuilder(schema) + .addRow("barney", mapValue(1, mapArray(mapValue(10, "betty"), mapValue(11, "bambam")))) + .addRow("fred", mapValue(2, mapArray())) + .addRow("wilma", mapValue(3, mapArray(mapValue(30, "pebbles")))) + .build() + .container(); + + final OutputBatchBuilder builder = new OutputBatchBuilder(schema, + Collections.singletonList(new BatchSource(schema, input)), + fixture.allocator()); + builder.load(input.getRecordCount()); + VectorContainer output = builder.outputContainer(); + + RowSetUtilities.verify(fixture.wrap(input), fixture.wrap(output)); + } + @Test public void testListArray() { final TupleMetadata schema = new SchemaBuilder() From fddb04ef18038f6f369f2c219aec65eefdccfb27 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:01:47 -0400 Subject: [PATCH 10/50] DRILL-8556: Count projected array dimensions per column ScanProjectionParser took a column's array dimensions from its depth in the whole path, so m.a[0] became a two-dimensional array, its index was dropped, and projecting an element of an array inside a map failed as incompatible. --- .../scan/v3/schema/ScanProjectionParser.java | 3 ++- .../scan/v3/schema/TestProjectionParser.java | 23 +++++++++++++++++++ .../schema/TestSchemaTrackerInputSchema.java | 4 ++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/v3/schema/ScanProjectionParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/v3/schema/ScanProjectionParser.java index a37cb7bcb7d..7882a5f155f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/v3/schema/ScanProjectionParser.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/v3/schema/ScanProjectionParser.java @@ -88,7 +88,8 @@ private void parseMember(TupleMetadata tuple, int depth, NameSegment nameSeg) { } } else { ProjectedColumn col = project(tuple, nameSeg.getPath()); - parseChildSeg(col, depth + 1, nameSeg); + // Array dimensions are counted from this column, whatever its depth in the path + parseChildSeg(col, 1, nameSeg); } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/schema/TestProjectionParser.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/schema/TestProjectionParser.java index d67d7e66080..82b9d17f275 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/schema/TestProjectionParser.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/schema/TestProjectionParser.java @@ -319,6 +319,29 @@ public void testArray() { assertTrue(indexes[3]); } + @Test + public void testArrayInMap() { + // Array dimensions count from the column, not from the root of the path + ProjectionParseResult result = ScanProjectionParser.parse( + RowSetTestUtils.projectList("m.a[1]", "m.b[0].x")); + TupleMetadata projSet = result.dynamicSchema; + ProjectedColumn m = (ProjectedColumn) projSet.metadata("m"); + assertTrue(m.isMap()); + assertFalse(m.isArray()); + + ProjectedColumn a = (ProjectedColumn) m.tupleSchema().metadata("a"); + assertTrue(a.isArray()); + assertEquals(1, a.arrayDims()); + assertTrue(a.hasIndex(1)); + + ProjectedColumn b = (ProjectedColumn) m.tupleSchema().metadata("b"); + assertTrue(b.isArray()); + assertEquals(1, b.arrayDims()); + assertTrue(b.hasIndex(0)); + assertTrue(b.isMap()); + assertNotNull(b.tupleSchema().metadata("x")); + } + @Test public void testMultiDimArray() { ProjectionParseResult result = ScanProjectionParser.parse( diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/schema/TestSchemaTrackerInputSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/schema/TestSchemaTrackerInputSchema.java index 2113d0116e3..89d79aa5d92 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/schema/TestSchemaTrackerInputSchema.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/scan/v3/schema/TestSchemaTrackerInputSchema.java @@ -464,7 +464,7 @@ public void testProvidedArrayProjectConflictInMap() { fail(); } catch (UserException e) { assertTrue(e.getMessage().contains("not compatible")); - assertTrue(e.getMessage().contains("Projected column: x[]")); + assertTrue(e.getMessage().contains("Projected column: x[2]")); assertTrue(e.getMessage().contains("Provided column: `x` BIGINT NOT NULL")); } } @@ -478,7 +478,7 @@ public void testReaderArrayProjectConflictInMap() { fail(); } catch (UserException e) { assertTrue(e.getMessage().contains("not compatible")); - assertTrue(e.getMessage().contains("Projected column: x[]")); + assertTrue(e.getMessage().contains("Projected column: x[2]")); assertTrue(e.getMessage().contains("Reader column: `x` BIGINT NOT NULL")); } } From bf9b3b5d8771d1c920507fcca7fad69c944ef482 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:01:47 -0400 Subject: [PATCH 11/50] DRILL-8556: Add DNS protocol decoder --- .../store/pcap/protocol/ProtocolDecoders.java | 26 +- .../store/pcap/protocol/dns/DnsDecoder.java | 124 ++++++++ .../store/pcap/protocol/dns/DnsMessage.java | 51 +++ .../store/pcap/protocol/dns/DnsParser.java | 297 ++++++++++++++++++ ....store.pcap.protocol.PacketProtocolDecoder | 1 + .../pcap/protocol/TestProtocolDecoders.java | 11 + .../pcap/protocol/dns/TestDnsDecoder.java | 160 ++++++++++ .../exec/store/pcapng/TestPcapngDecoding.java | 20 ++ .../pcapng/TestPcapngStatRecordReader.java | 2 +- .../drill/exec/store/pcapng/TmpLoader.java | 50 +++ .../test/resources/fixtures/pcap_fixtures.py | 27 ++ .../src/test/resources/pcapng/dns.pcapng | Bin 0 -> 428 bytes 12 files changed, 767 insertions(+), 2 deletions(-) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsMessage.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsParser.java create mode 100644 contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dns/TestDnsDecoder.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TmpLoader.java create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/dns.pcapng diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java index 41e7988f3a8..ed5a83a7169 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.ServiceLoader; +import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.MetadataUtils; import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; @@ -40,6 +41,8 @@ public class ProtocolDecoders { private static final Logger logger = LoggerFactory.getLogger(ProtocolDecoders.class); private static volatile ProtocolDecoders instance; + /** Initial width of VARCHAR fields in parsed_data, which most rows leave empty. */ + public static final int SPARSE_WIDTH = 8; private final List> packetDecoders; private final TupleMetadata packetDataSchema; @@ -181,7 +184,9 @@ private static List usable(Iterable try { SchemaBuilder fields = new SchemaBuilder(); defineSchema(decoder, fields); - data.add(MetadataUtils.newMap(decoder.protocol(), fields.buildSchema())); + TupleMetadata schema = fields.buildSchema(); + sizeSparse(schema); + data.add(MetadataUtils.newMap(decoder.protocol(), schema)); } catch (RuntimeException e) { logger.warn("Ignoring protocol decoder {}: its schema could not be built", decoder.getClass().getName(), e); continue; @@ -192,6 +197,25 @@ private static List usable(Iterable return Collections.unmodifiableList(result); } + /** + * Starts every vector small. Drill otherwise reserves room for a full batch of + * values per column, and ten elements per array row, which for the many mostly + * empty decoder fields would use up the batch memory budget before any row is + * written. Vectors still grow when a row needs more. + */ + private static void sizeSparse(TupleMetadata schema) { + for (ColumnMetadata column : schema) { + if (column.isArray()) { + column.setExpectedElementCount(1); + } + if (column.isMap()) { + sizeSparse(column.tupleSchema()); + } else if (column.isVariableWidth()) { + column.setExpectedWidth(SPARSE_WIDTH); + } + } + } + private static void defineSchema(ProtocolDecoder decoder, SchemaBuilder fields) { if (decoder instanceof SessionProtocolDecoder) { ((SessionProtocolDecoder) decoder).defineSchema(fields); diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsDecoder.java new file mode 100644 index 00000000000..d8f12169a7c --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsDecoder.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.dns; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** DNS, mDNS and LLMNR over UDP, and single-segment DNS over TCP. */ +public class DnsDecoder implements PacketProtocolDecoder { + private static final Set PORTS = new HashSet<>(Arrays.asList(53, 5353, 5355)); + private static final String[] SECTIONS = {"answers", "authorities", "additionals"}; + + @Override + public String protocol() { + return "dns"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("transaction_id", MinorType.INT) + .addNullable("is_response", MinorType.BIT) + .addNullable("opcode", MinorType.INT) + .addNullable("rcode", MinorType.INT) + .addNullable("authoritative", MinorType.BIT) + .addNullable("truncated", MinorType.BIT) + .addNullable("recursion_desired", MinorType.BIT) + .addNullable("recursion_available", MinorType.BIT) + .addMapArray("questions") + .addNullable("name", MinorType.VARCHAR) + .addNullable("type", MinorType.VARCHAR) + .addNullable("class", MinorType.INT) + .resumeSchema(); + for (String section : SECTIONS) { + fields.addMapArray(section) + .addNullable("name", MinorType.VARCHAR) + .addNullable("type", MinorType.VARCHAR) + .addNullable("class", MinorType.INT) + .addNullable("ttl", MinorType.BIGINT) + .addNullable("data", MinorType.VARCHAR) + .resumeSchema(); + } + } + + @Override + public boolean accepts(Packet packet) { + return (packet.isUdpPacket() || packet.isTcpPacket()) + && (PORTS.contains(packet.getSrc_port()) || PORTS.contains(packet.getDst_port())); + } + + @Override + public DnsMessage parse(Packet packet, byte[] payload, DecoderContext context) { + if (payload == null) { + return null; + } + int offset = 0; + if (packet.isTcpPacket()) { + // DNS over TCP starts with a 2-byte length; only whole messages in one segment are handled here + if (payload.length < 2 || (((payload[0] & 0xFF) << 8) | (payload[1] & 0xFF)) != payload.length - 2) { + return null; + } + offset = 2; + } + return DnsParser.parse(payload, offset, context); + } + + @Override + public void write(DnsMessage m, TupleWriter fields) { + fields.scalar("transaction_id").setInt(m.transactionId); + fields.scalar("is_response").setBoolean(m.isResponse); + fields.scalar("opcode").setInt(m.opcode); + fields.scalar("rcode").setInt(m.rcode); + fields.scalar("authoritative").setBoolean(m.authoritative); + fields.scalar("truncated").setBoolean(m.truncated); + fields.scalar("recursion_desired").setBoolean(m.recursionDesired); + fields.scalar("recursion_available").setBoolean(m.recursionAvailable); + ArrayWriter questions = fields.array("questions"); + for (DnsMessage.Question q : m.questions) { + TupleWriter t = questions.tuple(); + t.scalar("name").setString(q.name); + t.scalar("type").setString(q.type); + t.scalar("class").setInt(q.dnsClass); + questions.save(); + } + List[] lists = {m.answers, m.authorities, m.additionals}; + for (int s = 0; s < SECTIONS.length; s++) { + ArrayWriter records = fields.array(SECTIONS[s]); + for (Object o : lists[s]) { + DnsMessage.Record r = (DnsMessage.Record) o; + TupleWriter t = records.tuple(); + t.scalar("name").setString(r.name); + t.scalar("type").setString(r.type); + t.scalar("class").setInt(r.dnsClass); + t.scalar("ttl").setLong(r.ttl); + t.scalar("data").setString(r.data); + records.save(); + } + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsMessage.java new file mode 100644 index 00000000000..f669521dc26 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsMessage.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.dns; + +import java.util.ArrayList; +import java.util.List; + +/** A parsed DNS message. */ +public class DnsMessage { + public int transactionId; + public boolean isResponse; + public int opcode; + public int rcode; + public boolean authoritative; + public boolean truncated; + public boolean recursionDesired; + public boolean recursionAvailable; + public final List questions = new ArrayList<>(); + public final List answers = new ArrayList<>(); + public final List authorities = new ArrayList<>(); + public final List additionals = new ArrayList<>(); + + public static class Question { + public String name; + public String type; + public int dnsClass; + } + + public static class Record { + public String name; + public String type; + public int dnsClass; + public long ttl; + public String data; + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsParser.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsParser.java new file mode 100644 index 00000000000..ad32ca17fdd --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsParser.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.dns; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; + +/** Parses DNS messages (RFC 1035) with bounded work. */ +public final class DnsParser { + static final int MAX_ITEMS = 64; + private static final int MAX_POINTER_JUMPS = 16; + private static final int MAX_STRING = 4096; + private static final Map TYPES = new HashMap<>(); + + static { + String[][] types = {{"1", "A"}, {"2", "NS"}, {"5", "CNAME"}, {"6", "SOA"}, {"12", "PTR"}, {"13", "HINFO"}, + {"15", "MX"}, {"16", "TXT"}, {"28", "AAAA"}, {"33", "SRV"}, {"35", "NAPTR"}, {"39", "DNAME"}, + {"41", "OPT"}, {"43", "DS"}, {"46", "RRSIG"}, {"47", "NSEC"}, {"48", "DNSKEY"}, {"64", "SVCB"}, + {"65", "HTTPS"}, {"99", "SPF"}, {"252", "AXFR"}, {"255", "ANY"}, {"257", "CAA"}}; + for (String[] t : types) { + TYPES.put(Integer.parseInt(t[0]), t[1]); + } + } + + private final byte[] b; + private final int start; + private final int end; + private int pos; + + private DnsParser(byte[] b, int start) { + this.b = b; + this.start = start; + this.end = b.length; + this.pos = start; + } + + /** + * @return the message, or null if the data is not DNS + * @throws IllegalArgumentException if the data is DNS but malformed + */ + public static DnsMessage parse(byte[] data, int offset, DecoderContext context) { + return new DnsParser(data, offset).message(context); + } + + private DnsMessage message(DecoderContext context) { + if (end - start < 12) { + return null; + } + int flags = u16(start + 2); + int qd = u16(start + 4); + int an = u16(start + 6); + int ns = u16(start + 8); + int ar = u16(start + 10); + DnsMessage m = new DnsMessage(); + m.transactionId = u16(start); + m.isResponse = (flags & 0x8000) != 0; + m.opcode = (flags >> 11) & 0xF; + m.authoritative = (flags & 0x0400) != 0; + m.truncated = (flags & 0x0200) != 0; + m.recursionDesired = (flags & 0x0100) != 0; + m.recursionAvailable = (flags & 0x0080) != 0; + m.rcode = flags & 0xF; + boolean zBit = (flags & 0x0040) != 0; + int records = an + ns + ar; + if (m.opcode > 6 || zBit || qd + records == 0 || qd * 5 + records * 11 > end - start - 12) { + return null; + } + pos = start + 12; + boolean confident = false; + for (int i = 0; i < qd; i++) { + try { + DnsMessage.Question q = new DnsMessage.Question(); + q.name = name(); + q.type = type(u16(need(4))); + q.dnsClass = u16(pos - 2); + if (m.questions.size() < MAX_ITEMS) { + m.questions.add(q); + } + confident = true; + } catch (IllegalArgumentException e) { + if (!confident) { + return null; + } + throw new IllegalArgumentException("truncated question " + (i + 1) + ": " + e.getMessage()); + } + } + if (qd > MAX_ITEMS) { + context.warn("questions truncated to " + MAX_ITEMS); + } + String[] sections = {"answer", "authority", "additional"}; + int[] counts = {an, ns, ar}; + List[] lists = {m.answers, m.authorities, m.additionals}; + for (int s = 0; s < 3; s++) { + for (int i = 0; i < counts[s]; i++) { + if (i == MAX_ITEMS) { + context.warn(sections[s] + "s truncated to " + MAX_ITEMS); + return m; + } + try { + @SuppressWarnings("unchecked") + List list = (List) lists[s]; + list.add(record()); + confident = true; + } catch (IllegalArgumentException e) { + if (!confident) { + return null; + } + throw new IllegalArgumentException("truncated " + sections[s] + " " + (i + 1) + ": " + e.getMessage()); + } + } + } + return m; + } + + private DnsMessage.Record record() { + DnsMessage.Record r = new DnsMessage.Record(); + r.name = name(); + int at = need(10); + int type = u16(at); + r.type = type(type); + r.dnsClass = u16(at + 2); + r.ttl = u32(at + 4); + int length = u16(at + 8); + int data = need(length); + r.data = rdata(type, data, length); + return r; + } + + private String rdata(int type, int at, int length) { + switch (type) { + case 1: + case 28: + if (length != (type == 1 ? 4 : 16)) { + throw new IllegalArgumentException("bad address length " + length); + } + try { + return InetAddress.getByAddress(Arrays.copyOfRange(b, at, at + length)).getHostAddress(); + } catch (UnknownHostException e) { + throw new IllegalArgumentException(e.getMessage()); + } + case 2: + case 5: + case 12: + case 39: + return nameAt(at, at + length); + case 15: + return u16(at) + " " + nameAt(at + 2, at + length); + case 33: + return u16(at) + " " + u16(at + 2) + " " + u16(at + 4) + " " + nameAt(at + 6, at + length); + case 16: + return text(at, at + length); + default: + return hex(at, length); + } + } + + /** Character strings joined as "a" "b", like dig. */ + private String text(int at, int limit) { + StringBuilder out = new StringBuilder(); + int p = at; + while (p < limit) { + int len = b[p] & 0xFF; + if (p + 1 + len > limit) { + throw new IllegalArgumentException("bad TXT string length"); + } + if (out.length() > 0) { + out.append(' '); + } + out.append('"').append(new String(b, p + 1, len, StandardCharsets.ISO_8859_1)).append('"'); + p += 1 + len; + } + return cap(out.toString()); + } + + private String hex(int at, int length) { + StringBuilder out = new StringBuilder(); + for (int i = at; i < at + length && out.length() < MAX_STRING; i++) { + out.append(String.format("%02x", b[i])); + } + return cap(out.toString()); + } + + /** Reads a name at pos and advances pos past it. */ + private String name() { + int[] next = new int[1]; + String n = readName(pos, next); + pos = next[0]; + return n; + } + + /** A name inside RDATA, which must fit before limit. */ + private String nameAt(int at, int limit) { + int[] next = new int[1]; + String n = readName(at, next); + if (next[0] > limit) { + throw new IllegalArgumentException("name overruns record data"); + } + return n; + } + + /** next[0] receives the position after the name in the original sequence. */ + private String readName(int at, int[] next) { + StringBuilder out = new StringBuilder(); + int p = at; + int jumps = 0; + int resume = -1; + while (true) { + if (p >= end) { + throw new IllegalArgumentException("name runs past end of message"); + } + int len = b[p] & 0xFF; + if (len == 0) { + p++; + break; + } + if ((len & 0xC0) == 0xC0) { + if (p + 1 >= end) { + throw new IllegalArgumentException("truncated compression pointer"); + } + int target = start + (((len & 0x3F) << 8) | (b[p + 1] & 0xFF)); + if (++jumps > MAX_POINTER_JUMPS || target >= p) { + throw new IllegalArgumentException("invalid compression pointer"); + } + if (resume < 0) { + resume = p + 2; + } + p = target; + continue; + } + if ((len & 0xC0) != 0 || len > 63 || p + 1 + len > end) { + throw new IllegalArgumentException("invalid label"); + } + if (out.length() > 0) { + out.append('.'); + } + out.append(new String(b, p + 1, len, StandardCharsets.ISO_8859_1)); + if (out.length() > 255) { + throw new IllegalArgumentException("name longer than 255 bytes"); + } + p += 1 + len; + } + next[0] = resume >= 0 ? resume : p; + return out.length() == 0 ? "." : out.toString(); + } + + private static String type(int type) { + String name = TYPES.get(type); + return name != null ? name : String.valueOf(type); + } + + private static String cap(String s) { + return s.length() > MAX_STRING ? s.substring(0, MAX_STRING) : s; + } + + /** Reserves count bytes at pos, returning their start. */ + private int need(int count) { + if (pos + count > end) { + throw new IllegalArgumentException("needs " + count + " bytes at offset " + (pos - start)); + } + int at = pos; + pos += count; + return at; + } + + private int u16(int at) { + if (at + 2 > end) { + throw new IllegalArgumentException("truncated field at offset " + (at - start)); + } + return ((b[at] & 0xFF) << 8) | (b[at + 1] & 0xFF); + } + + private long u32(int at) { + return ((long) u16(at) << 16) | u16(at + 2); + } +} diff --git a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder new file mode 100644 index 00000000000..8f35f754e4d --- /dev/null +++ b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder @@ -0,0 +1 @@ +org.apache.drill.exec.store.pcap.protocol.dns.DnsDecoder diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java index 92c53529f4d..5851eea33fe 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java @@ -26,6 +26,8 @@ import java.util.Arrays; import java.util.Collections; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.store.pcap.decoder.Packet; import org.apache.drill.test.BaseTest; import org.junit.Test; @@ -46,6 +48,15 @@ public void testServiceLoaderFindsTestDecoder() { assertNotNull(ProtocolDecoders.get().packetDataSchema().metadata("echo_test")); } + @Test + public void testDecoderFieldsSizedSparse() { + // Most rows have no decoded data, so vectors start small instead of reserving room for every row + TupleMetadata dns = ProtocolDecoders.get().packetDataSchema().metadata("dns").tupleSchema(); + ColumnMetadata answers = dns.metadata("answers"); + assertEquals(1, answers.expectedElementCount()); + assertEquals(ProtocolDecoders.SPARSE_WIDTH, answers.tupleSchema().metadata("data").expectedWidth()); + } + @Test public void testParsed() { ProtocolDecoders decoders = new ProtocolDecoders(Collections.singletonList(new EchoTestDecoder())); diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dns/TestDnsDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dns/TestDnsDecoder.java new file mode 100644 index 00000000000..f12c67aa615 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dns/TestDnsDecoder.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.dns; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestDnsDecoder extends BaseTest { + + private static final class Context implements DecoderContext { + final List warnings = new ArrayList<>(); + + @Override + public boolean exposeCredentials() { + return false; + } + + @Override + public void warn(String message) { + warnings.add(message); + } + } + + /** Header plus raw body. */ + private static byte[] message(int id, int flags, int qd, int an, int ns, int ar, byte[] body) { + return ByteBuffer.allocate(12 + body.length).putShort((short) id).putShort((short) flags) + .putShort((short) qd).putShort((short) an).putShort((short) ns).putShort((short) ar).put(body).array(); + } + + private static byte[] name(String dotted) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (String label : dotted.split("\\.")) { + out.write(label.length()); + out.write(label.getBytes(), 0, label.length()); + } + out.write(0); + return out.toByteArray(); + } + + private static byte[] concat(byte[]... parts) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (byte[] p : parts) { + out.write(p, 0, p.length); + } + return out.toByteArray(); + } + + private static byte[] question(String n, int type) { + return concat(name(n), ByteBuffer.allocate(4).putShort((short) type).putShort((short) 1).array()); + } + + /** Answer whose name is a compression pointer to offset 12 (the question name). */ + private static byte[] answerA(int ttl, byte[] address) { + return ByteBuffer.allocate(2 + 10 + address.length).put((byte) 0xC0).put((byte) 12) + .putShort((short) 1).putShort((short) 1).putInt(ttl).putShort((short) address.length).put(address).array(); + } + + @Test + public void testQueryAndResponse() { + byte[] response = message(0x1234, 0x8180, 1, 1, 0, 0, + concat(question("example.com", 1), answerA(300, new byte[] {93, (byte) 184, (byte) 216, 34}))); + DnsMessage m = DnsParser.parse(response, 0, new Context()); + assertEquals(0x1234, m.transactionId); + assertTrue(m.isResponse); + assertTrue(m.recursionDesired); + assertTrue(m.recursionAvailable); + assertEquals("example.com", m.questions.get(0).name); + assertEquals("A", m.questions.get(0).type); + assertEquals("example.com", m.answers.get(0).name); + assertEquals(300L, m.answers.get(0).ttl); + assertEquals("93.184.216.34", m.answers.get(0).data); + } + + @Test + public void testNotDns() { + assertNull(DnsParser.parse("GET / HTTP/1.1\r\nHost: x\r\n\r\n".getBytes(), 0, new Context())); + assertNull(DnsParser.parse(new byte[] {1, 2, 3}, 0, new Context())); + assertNull(DnsParser.parse(message(1, 0, 0, 0, 0, 0, new byte[0]), 0, new Context())); + // Opcode 15 is not assigned + assertNull(DnsParser.parse(message(1, 15 << 11, 1, 0, 0, 0, question("a.b", 1)), 0, new Context())); + } + + @Test + public void testTruncatedAnswerIsMalformedDns() { + byte[] full = message(1, 0x8180, 1, 1, 0, 0, concat(question("example.com", 1), answerA(300, new byte[4]))); + byte[] cut = java.util.Arrays.copyOf(full, full.length - 3); + try { + DnsParser.parse(cut, 0, new Context()); + fail("expected malformed DNS"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("answer 1")); + } + } + + @Test + public void testCompressionPointerLoop() { + // Answer name points to itself + byte[] loop = ByteBuffer.allocate(12).put((byte) 0xC0).put((byte) 29).putShort((short) 1).putShort((short) 1) + .putInt(0).putShort((short) 0).array(); + byte[] m = message(1, 0x8180, 1, 1, 0, 0, concat(question("a.b", 1), loop)); + try { + DnsParser.parse(m, 0, new Context()); + fail("expected malformed DNS"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("pointer")); + } + } + + @Test + public void testRecordCap() { + ByteArrayOutputStream body = new ByteArrayOutputStream(); + byte[] q = question("a.b", 1); + body.write(q, 0, q.length); + for (int i = 0; i < 70; i++) { + byte[] a = answerA(1, new byte[4]); + body.write(a, 0, a.length); + } + Context context = new Context(); + DnsMessage m = DnsParser.parse(message(1, 0x8180, 1, 70, 0, 0, body.toByteArray()), 0, context); + assertEquals(64, m.answers.size()); + assertEquals(1, context.warnings.size()); + } + + @Test + public void testRecordTypes() { + byte[] mx = ByteBuffer.allocate(2 + 10 + 2 + 2).put((byte) 0xC0).put((byte) 12).putShort((short) 15).putShort((short) 1) + .putInt(60).putShort((short) 4).putShort((short) 10).put((byte) 0xC0).put((byte) 12).array(); + DnsMessage m = DnsParser.parse(message(1, 0x8180, 1, 1, 0, 0, concat(question("example.com", 15), mx)), 0, new Context()); + assertEquals("MX", m.answers.get(0).type); + assertEquals("10 example.com", m.answers.get(0).data); + assertFalse(m.truncated); + } +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java index d22a70580c3..fc4cc6217fd 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java @@ -104,4 +104,24 @@ public void testDecodingSkippedWhenNotProjected() throws Exception { assertEquals(0, results.rowCount()); results.clear(); } + + @Test + public void testDns() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.dns.questions[0].name as qname, " + + "t.parsed_data.dns.answers[0].data as answer, decode_error from dfs.`pcapng/dns.pcapng` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("qname", MinorType.VARCHAR) + .addNullable("answer", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("dns", "example.com", null, null) + .addRow("dns", "example.com", "93.184.216.34", null) + .addRow(null, null, null, null) + .addRow("dns", null, null, "dns: truncated answer 1: needs 4 bytes at offset 41") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java index 5927091e978..d4915fea082 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java @@ -115,7 +115,7 @@ public void testSerDe() throws Exception { long cnt = queryBuilder().physical(plan).singletonLong(); // The error row of bad_block_length.pcapng is counted too - assertEquals("Counts should match", 63, cnt); + assertEquals("Counts should match", 65, cnt); } @Test diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TmpLoader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TmpLoader.java new file mode 100644 index 00000000000..cdf90b34878 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TmpLoader.java @@ -0,0 +1,50 @@ +package org.apache.drill.exec.store.pcapng; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.memory.RootAllocatorFactory; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.physical.impl.validate.BatchValidator; +import org.apache.drill.exec.physical.resultSet.ResultSetLoader; +import org.apache.drill.exec.physical.resultSet.RowSetLoader; +import org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl; +import org.apache.drill.exec.physical.resultSet.impl.ResultSetOptionBuilder; +import org.apache.drill.exec.record.VectorContainer; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; +import org.junit.Test; + +public class TmpLoader { + @Test + public void probe() { + BufferAllocator allocator = RootAllocatorFactory.newRoot(DrillConfig.create()); + TupleMetadata dns = new SchemaBuilder().addNullable("id", MinorType.INT) + .addMapArray("questions").addNullable("name", MinorType.VARCHAR).resumeSchema().buildSchema(); + TupleMetadata data = new SchemaBuilder().add(MetadataUtils.newMap("dns", dns)).buildSchema(); + TupleMetadata schema = new SchemaBuilder().addNullable("p", MinorType.VARCHAR) + .add(MetadataUtils.newMap("parsed_data", data)).buildSchema(); + ResultSetLoader rsl = new ResultSetLoaderImpl(allocator, new ResultSetOptionBuilder().readerSchema(schema).build()); + RowSetLoader loader = rsl.writer(); + TupleWriter parsed = loader.tuple("parsed_data"); + rsl.startBatch(); + for (int row = 0; row < 4; row++) { + loader.start(); + if (row < 2) { + TupleWriter d = parsed.tuple("dns"); + d.scalar("id").setInt(row); + ArrayWriter q = d.array("questions"); + q.tuple().scalar("name").setString("q" + row); + q.save(); + } + loader.save(); + } + VectorContainer c = rsl.harvest(); + System.out.println("PROBE rows=" + c.getRecordCount() + " valid=" + BatchValidator.validate(c)); + c.zeroVectors(); + rsl.close(); + allocator.close(); + } +} diff --git a/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py index 028ee88b617..c046182821f 100644 --- a/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py +++ b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py @@ -137,8 +137,35 @@ def classic(): 'oid sha256:8d48391c3dde43c518b22c3d6ba4bbba315efac713803cc582d00e37144f577a\n' 'size 867464224\n') +def dns_name(dotted): + out = b'' + for label in dotted.split('.'): + out += bytes([len(label)]) + label.encode() + return out + b'\0' + + +def dns_message(ident, flags, questions, answers): + body = b''.join(dns_name(n) + struct.pack('>HH', t, 1) for n, t in questions) + for ttl, address in answers: + # Name is a compression pointer to the first question name at offset 12 + body += b'\xc0\x0c' + struct.pack('>HHIH', 1, 1, ttl, len(address)) + address + return struct.pack('>HHHHHH', ident, flags, len(questions), len(answers), 0, 0) + body + + +def dns(): + # Link type 101. Rows: query, response, non-DNS bytes on port 53, response cut 3 bytes short + query = dns_message(0x1234, 0x0100, [('example.com', 1)], []) + response = dns_message(0x1234, 0x8180, [('example.com', 1)], [(300, bytes([93, 184, 216, 34]))]) + f = shb() + idb(101) + f += epb(0, ipv4(17, udp(5000, 53, query), '10.0.0.1', '8.8.8.8')) + f += epb(0, ipv4(17, udp(53, 5000, response), '8.8.8.8', '10.0.0.1')) + f += epb(0, ipv4(17, udp(5000, 53, bytes(range(1, 14))), '10.0.0.1', '8.8.8.8')) + f += epb(0, ipv4(17, udp(53, 5000, response[:-3]), '8.8.8.8', '10.0.0.1')) + write(os.path.join(PCAPNG, 'dns.pcapng'), f) + if __name__ == '__main__': echo() errors() bad_block_length() classic() + dns() diff --git a/contrib/format-pcapng/src/test/resources/pcapng/dns.pcapng b/contrib/format-pcapng/src/test/resources/pcapng/dns.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..df1639e94105d846ed86032b2317476d850326bc GIT binary patch literal 428 zcma)&uMUD>6vZ!sfXEm8SDeYs;4nD}ZU&2PZrTkRoMSV Date: Fri, 2 Oct 2026 15:04:41 -0400 Subject: [PATCH 12/50] DRILL-8556: Add HTTP packet decoder --- .../store/pcap/protocol/http/HttpFields.java | 127 ++++++++++++++++ .../store/pcap/protocol/http/HttpMessage.java | 79 ++++++++++ .../pcap/protocol/http/HttpPacketDecoder.java | 59 ++++++++ .../store/pcap/protocol/http/HttpParser.java | 135 ++++++++++++++++++ ....store.pcap.protocol.PacketProtocolDecoder | 1 + .../exec/store/pcap/TestPcapDecoding.java | 23 +++ .../pcap/protocol/http/TestHttpParser.java | 129 +++++++++++++++++ 7 files changed, 553 insertions(+) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpFields.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpMessage.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpPacketDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpParser.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpParser.java diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpFields.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpFields.java new file mode 100644 index 00000000000..ac33e2edf64 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpFields.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.http; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.MapBuilder; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** Fields shared by the HTTP packet and session decoders. */ +final class HttpFields { + + private HttpFields() { } + + static void setString(TupleWriter t, String name, String value) { + if (value != null) { + t.scalar(name).setString(value); + } + } + + static void writeRequest(HttpMessage m, TupleWriter t, String headersField) { + setString(t, "method", m.method); + setString(t, "uri", m.uri); + setString(t, "version", m.version); + setString(t, "host", m.header("Host")); + setString(t, "user_agent", m.header("User-Agent")); + setString(t, "referer", m.header("Referer")); + setString(t, "username", m.username); + t.scalar("password_present").setBoolean(m.passwordPresent); + setString(t, "password", m.password); + if (headersField.equals("headers")) { + // Packet mode: a request body's type and length. In session mode these names hold the response's. + setString(t, "content_type", m.header("Content-Type")); + Long length = m.contentLength(); + if (length != null) { + t.scalar("content_length").setLong(length); + } + } + writeHeaders(m, t, headersField); + } + + static void writeResponse(HttpMessage m, TupleWriter t, String headersField) { + if (m.statusCode != null) { + t.scalar("status_code").setInt(m.statusCode); + } + setString(t, "reason", m.reason); + setString(t, "content_type", m.header("Content-Type")); + Long length = m.contentLength(); + if (length != null) { + t.scalar("content_length").setLong(length); + } + writeHeaders(m, t, headersField); + } + + static void writeHeaders(HttpMessage m, TupleWriter t, String field) { + ArrayWriter headers = t.array(field); + for (String[] h : m.headers) { + TupleWriter header = headers.tuple(); + header.scalar("name").setString(h[0]); + header.scalar("value").setString(h[1]); + headers.save(); + } + } + + /** Fields of the packet decoder's sub-map. */ + static void definePacket(SchemaBuilder f) { + f.addNullable("is_request", MinorType.BIT) + .addNullable("method", MinorType.VARCHAR) + .addNullable("uri", MinorType.VARCHAR) + .addNullable("version", MinorType.VARCHAR) + .addNullable("status_code", MinorType.INT) + .addNullable("reason", MinorType.VARCHAR) + .addNullable("host", MinorType.VARCHAR) + .addNullable("user_agent", MinorType.VARCHAR) + .addNullable("content_type", MinorType.VARCHAR) + .addNullable("referer", MinorType.VARCHAR) + .addNullable("content_length", MinorType.BIGINT) + .addNullable("username", MinorType.VARCHAR) + .addNullable("password_present", MinorType.BIT) + .addNullable("password", MinorType.VARCHAR) + .addMapArray("headers") + .addNullable("name", MinorType.VARCHAR) + .addNullable("value", MinorType.VARCHAR) + .resumeSchema(); + } + + /** Fields of one entry of the session decoder's exchanges list. */ + static void defineExchange(MapBuilder e) { + e.addNullable("method", MinorType.VARCHAR) + .addNullable("uri", MinorType.VARCHAR) + .addNullable("version", MinorType.VARCHAR) + .addNullable("host", MinorType.VARCHAR) + .addNullable("user_agent", MinorType.VARCHAR) + .addNullable("referer", MinorType.VARCHAR) + .addNullable("username", MinorType.VARCHAR) + .addNullable("password_present", MinorType.BIT) + .addNullable("password", MinorType.VARCHAR) + .addMapArray("request_headers") + .addNullable("name", MinorType.VARCHAR) + .addNullable("value", MinorType.VARCHAR) + .resumeMap() + .addNullable("status_code", MinorType.INT) + .addNullable("reason", MinorType.VARCHAR) + .addNullable("content_type", MinorType.VARCHAR) + .addNullable("content_length", MinorType.BIGINT) + .addMapArray("response_headers") + .addNullable("name", MinorType.VARCHAR) + .addNullable("value", MinorType.VARCHAR) + .resumeMap(); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpMessage.java new file mode 100644 index 00000000000..4a6cc015a74 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpMessage.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.http; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; + +/** One HTTP/1.x request or response: start line and headers. */ +public class HttpMessage { + public boolean isRequest; + public String method; + public String uri; + public String version; + public Integer statusCode; + public String reason; + public final List headers = new ArrayList<>(); + /** Offset just past the blank line ending the headers, or -1 if the headers are incomplete. */ + public int headerEnd = -1; + public String username; + public boolean passwordPresent; + public String password; + private DecoderContext context; + + void setContext(DecoderContext context) { + this.context = context; + } + + /** First header with this name, ignoring case; null if absent. */ + public String header(String name) { + for (String[] h : headers) { + if (h[0].equalsIgnoreCase(name)) { + return h[1]; + } + } + return null; + } + + /** Content-Length, or null if absent or invalid. */ + public Long contentLength() { + String value = header("Content-Length"); + if (value == null) { + return null; + } + try { + long length = Long.parseLong(value.trim()); + if (length >= 0) { + return length; + } + } catch (NumberFormatException e) { + // reported below + } + if (context != null) { + context.warn("invalid Content-Length " + value); + } + return null; + } + + public boolean isChunked() { + String value = header("Transfer-Encoding"); + return value != null && value.toLowerCase().contains("chunked"); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpPacketDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpPacketDecoder.java new file mode 100644 index 00000000000..7d81722be61 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpPacketDecoder.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.http; + +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** The HTTP/1.x request or response that starts in a TCP segment. */ +public class HttpPacketDecoder implements PacketProtocolDecoder { + + @Override + public String protocol() { + return "http"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + HttpFields.definePacket(fields); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isTcpPacket() + && (HttpParser.PORTS.contains(packet.getSrc_port()) || HttpParser.PORTS.contains(packet.getDst_port())); + } + + @Override + public HttpMessage parse(Packet packet, byte[] payload, DecoderContext context) { + return payload == null ? null : HttpParser.parseMessage(payload, 0, payload.length, context); + } + + @Override + public void write(HttpMessage m, TupleWriter fields) { + fields.scalar("is_request").setBoolean(m.isRequest); + if (m.isRequest) { + HttpFields.writeRequest(m, fields, "headers"); + } else { + HttpFields.writeResponse(m, fields, "headers"); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpParser.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpParser.java new file mode 100644 index 00000000000..b5c53055508 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpParser.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.http; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; + +/** Parses HTTP/1.x start lines and headers with bounded work. */ +public final class HttpParser { + public static final Set PORTS = + Collections.unmodifiableSet(new HashSet<>(Arrays.asList(80, 591, 3128, 8000, 8008, 8080, 8888))); + static final int MAX_HEADERS = 64; + static final int MAX_STRING = 4096; + private static final int MAX_LINE = 8192; + private static final Pattern REQUEST = + Pattern.compile("(GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH|CONNECT|TRACE) (\\S+) HTTP/(1\\.[01])"); + private static final Pattern STATUS = Pattern.compile("HTTP/(1\\.[01]) ([1-5]\\d\\d)(?: (.*))?"); + private static final Pattern HEADER = Pattern.compile("([!#$%&'*+.^_`|~0-9A-Za-z-]+):[ \\t]*(.*?)[ \\t]*"); + + private HttpParser() { } + + /** + * @return the message at offset, or null if no request or status line starts there + */ + public static HttpMessage parseMessage(byte[] data, int offset, int limit, DecoderContext context) { + int lineEnd = lineEnd(data, offset, limit); + if (lineEnd < 0) { + return null; + } + String startLine = new String(data, offset, lineEnd - offset, StandardCharsets.ISO_8859_1); + HttpMessage m = new HttpMessage(); + m.setContext(context); + Matcher request = REQUEST.matcher(startLine); + Matcher status = STATUS.matcher(startLine); + if (request.matches()) { + m.isRequest = true; + m.method = request.group(1); + m.uri = cap(request.group(2)); + m.version = request.group(3); + } else if (status.matches()) { + m.version = status.group(1); + m.statusCode = Integer.parseInt(status.group(2)); + m.reason = status.group(3) == null ? null : cap(status.group(3)); + } else { + return null; + } + int p = lineEnd + 2; + boolean capped = false; + while (true) { + int end = lineEnd(data, p, limit); + if (end < 0) { + break; // headers continue in a later segment; headerEnd stays -1 + } + if (end == p) { + m.headerEnd = p + 2; + break; + } + String line = new String(data, p, end - p, StandardCharsets.ISO_8859_1); + Matcher header = HEADER.matcher(line); + if (!header.matches()) { + context.warn("malformed header line"); + break; + } + if (m.headers.size() < MAX_HEADERS) { + m.headers.add(new String[] {header.group(1), cap(header.group(2))}); + } else if (!capped) { + context.warn("headers truncated to " + MAX_HEADERS); + capped = true; + } + p = end + 2; + } + credentials(m, context); + return m; + } + + private static void credentials(HttpMessage m, DecoderContext context) { + String auth = m.header("Authorization"); + if (auth == null || !auth.regionMatches(true, 0, "Basic ", 0, 6)) { + return; + } + try { + String decoded = new String(Base64.getDecoder().decode(auth.substring(6).trim()), StandardCharsets.UTF_8); + int colon = decoded.indexOf(':'); + if (colon < 0) { + context.warn("Basic credentials without a colon"); + return; + } + m.username = cap(decoded.substring(0, colon)); + m.passwordPresent = true; + if (context.exposeCredentials()) { + m.password = cap(decoded.substring(colon + 1)); + } + } catch (IllegalArgumentException e) { + context.warn("invalid Basic credentials"); + } + } + + /** Offset of the CR of the next CRLF, or -1 if there is none within MAX_LINE bytes. */ + static int lineEnd(byte[] data, int from, int limit) { + int max = Math.min(limit, from + MAX_LINE); + for (int i = from; i + 1 < max; i++) { + if (data[i] == '\r' && data[i + 1] == '\n') { + return i; + } + } + return -1; + } + + static String cap(String s) { + return s.length() > MAX_STRING ? s.substring(0, MAX_STRING) : s; + } +} diff --git a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder index 8f35f754e4d..556133a8f60 100644 --- a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder +++ b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder @@ -1 +1,2 @@ org.apache.drill.exec.store.pcap.protocol.dns.DnsDecoder +org.apache.drill.exec.store.pcap.protocol.http.HttpPacketDecoder diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java index 302dd3123ab..83ef54eef80 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java @@ -82,4 +82,27 @@ public void testBadRecordStopsWithErrorRow() throws Exception { String error = client.queryBuilder().sql(sql).singletonString(); assertTrue(error, error.startsWith("file: invalid packet record after packet 1: Packet too long")); } + + @Test + public void testHttpPackets() throws Exception { + String sql = "select t.parsed_data.http.method as method, t.parsed_data.http.uri as uri, " + + "t.parsed_data.http.host as host, t.parsed_data.http.status_code as status " + + "from dfs.`pcap/http.pcap` t where parsed_protocol = 'http'"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("method", MinorType.VARCHAR) + .addNullable("uri", MinorType.VARCHAR) + .addNullable("host", MinorType.VARCHAR) + .addNullable("status", MinorType.INT) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + // Capture order, as listed by scapy + .addRow("GET", "/download.html", "www.ethereal.com", null) + .addRow(null, null, null, 200) + .addRow("GET", "/pagead/ads?client=ca-pub-2309191948673629&random=1084443430285&lmt=1082467020&format=468x60_as&output=html&url=http%3A%2F%2Fwww.ethereal.com%2Fdownload.html&color_bg=FFFFFF&color_text=333333&color_link=000000&color_url=666633&color_border=666633", "pagead2.googlesyndication.com", null) + .addRow(null, null, null, 200) + .addRow(null, null, null, 200) + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpParser.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpParser.java new file mode 100644 index 00000000000..eaea50c42dd --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpParser.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.http; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestHttpParser extends BaseTest { + + static final class Context implements DecoderContext { + final boolean expose; + final List warnings = new ArrayList<>(); + + Context(boolean expose) { + this.expose = expose; + } + + @Override + public boolean exposeCredentials() { + return expose; + } + + @Override + public void warn(String message) { + warnings.add(message); + } + } + + static byte[] bytes(String s) { + return s.getBytes(StandardCharsets.ISO_8859_1); + } + + static HttpMessage parse(String s, boolean expose) { + byte[] b = bytes(s); + return HttpParser.parseMessage(b, 0, b.length, new Context(expose)); + } + + @Test + public void testRequest() { + HttpMessage m = parse("GET /index.html?q=1 HTTP/1.1\r\nHost: example.com\r\nUser-Agent: curl/8\r\n\r\n", false); + assertTrue(m.isRequest); + assertEquals("GET", m.method); + assertEquals("/index.html?q=1", m.uri); + assertEquals("1.1", m.version); + assertEquals("example.com", m.header("host")); + assertEquals(2, m.headers.size()); + assertTrue(m.headerEnd > 0); + } + + @Test + public void testResponse() { + HttpMessage m = parse("HTTP/1.1 404 Not Found\r\nContent-Length: 12\r\n\r\n", false); + assertFalse(m.isRequest); + assertEquals(Integer.valueOf(404), m.statusCode); + assertEquals("Not Found", m.reason); + assertEquals(Long.valueOf(12), m.contentLength()); + } + + @Test + public void testIncompleteHeaders() { + HttpMessage m = parse("POST /api HTTP/1.0\r\nHost: a\r\nContent-Ty", false); + assertEquals("POST", m.method); + assertEquals(-1, m.headerEnd); + assertEquals(1, m.headers.size()); + } + + @Test + public void testNotHttp() { + assertNull(parse("SSH-2.0-OpenSSH_9.0\r\n", false)); + assertNull(parse("get / HTTP/1.1\r\n\r\n", false)); + assertNull(parse("GET / HTTP/2\r\n\r\n", false)); + assertNull(parse("\u0016\u0003\u0001\u0002\u0000", false)); + } + + @Test + public void testBasicAuth() { + // dXNlcjpzM2NyZXQ= is user:s3cret + String request = "GET / HTTP/1.1\r\nAuthorization: Basic dXNlcjpzM2NyZXQ=\r\n\r\n"; + HttpMessage hidden = parse(request, false); + assertEquals("user", hidden.username); + assertTrue(hidden.passwordPresent); + assertNull(hidden.password); + assertEquals("s3cret", parse(request, true).password); + } + + @Test + public void testHeaderCap() { + StringBuilder s = new StringBuilder("GET / HTTP/1.1\r\n"); + for (int i = 0; i < 70; i++) { + s.append("X-H").append(i).append(": v\r\n"); + } + s.append("\r\n"); + byte[] b = bytes(s.toString()); + Context context = new Context(false); + HttpMessage m = HttpParser.parseMessage(b, 0, b.length, context); + assertEquals(64, m.headers.size()); + assertEquals(1, context.warnings.size()); + } + + @Test + public void testChunked() { + assertTrue(parse("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", false).isChunked()); + } +} From 96716740cbb9bde783dc554b5656f6e48010a8ab Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:06:07 -0400 Subject: [PATCH 13/50] DRILL-8556: Add HTTP session decoder --- .../pcap/protocol/http/HttpExchanges.java | 27 +++ .../protocol/http/HttpSessionDecoder.java | 183 ++++++++++++++++++ ...store.pcap.protocol.SessionProtocolDecoder | 1 + .../exec/store/pcap/TestSessionizePCAP.java | 19 ++ .../protocol/http/TestHttpSessionDecoder.java | 77 ++++++++ 5 files changed, 307 insertions(+) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpExchanges.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java create mode 100644 contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpExchanges.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpExchanges.java new file mode 100644 index 00000000000..d3edf7ae8dc --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpExchanges.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.http; + +import java.util.ArrayList; +import java.util.List; + +/** The requests and responses of one HTTP connection, in order. */ +public class HttpExchanges { + public final List requests = new ArrayList<>(); + public final List responses = new ArrayList<>(); +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java new file mode 100644 index 00000000000..72e50dff552 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.http; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.drill.exec.record.metadata.MapBuilder; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder; +import org.apache.drill.exec.store.pcap.protocol.TcpStream; +import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** Every HTTP/1.x request and response of a TCP connection, paired in order. */ +public class HttpSessionDecoder implements SessionProtocolDecoder { + static final int MAX_MESSAGES = 1000; + + @Override + public String protocol() { + return "http"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + MapBuilder exchanges = fields.addMapArray("exchanges"); + HttpFields.defineExchange(exchanges); + exchanges.resumeSchema(); + } + + @Override + public boolean accepts(TcpSession session) { + return HttpParser.PORTS.contains(session.getSrcPort()) || HttpParser.PORTS.contains(session.getDstPort()); + } + + @Override + public HttpExchanges parse(TcpStream fromClient, TcpStream fromServer, DecoderContext context) { + HttpExchanges exchanges = new HttpExchanges(); + exchanges.requests.addAll(parseStream(fromClient.data(), true, Collections.emptyList(), context)); + if (exchanges.requests.isEmpty()) { + return null; + } + exchanges.responses.addAll(parseStream(fromServer.data(), false, exchanges.requests, context)); + if (fromClient.hasGaps()) { + context.warn("stopped at missing data in client stream at byte " + fromClient.firstGap()); + } + if (fromServer.hasGaps()) { + context.warn("stopped at missing data in server stream at byte " + fromServer.firstGap()); + } + return exchanges; + } + + /** + * Parses consecutive messages. + * + * @param requestsForResponses for responses: the requests they answer, so a HEAD response has no body + */ + static List parseStream(byte[] data, boolean requests, List requestsForResponses, + DecoderContext context) { + List messages = new ArrayList<>(); + String direction = requests ? "client" : "server"; + int offset = 0; + while (offset < data.length && messages.size() < MAX_MESSAGES) { + HttpMessage m = HttpParser.parseMessage(data, offset, data.length, context); + if (m == null || m.isRequest != requests) { + if (!messages.isEmpty()) { + context.warn("unparseable data in " + direction + " stream at byte " + offset); + } + break; + } + messages.add(m); + if (m.headerEnd < 0) { + break; + } + int index = messages.size() - 1; + long bodyEnd = bodyEnd(m, data, requests ? null : request(requestsForResponses, index)); + if (bodyEnd < 0 || bodyEnd > data.length) { + break; // body continues past the captured data + } + offset = (int) bodyEnd; + } + if (messages.size() == MAX_MESSAGES && offset < data.length) { + context.warn("stopped after " + MAX_MESSAGES + " messages in " + direction + " stream"); + } + return messages; + } + + private static HttpMessage request(List requests, int index) { + return index < requests.size() ? requests.get(index) : null; + } + + /** Offset just past the message body, or -1 if it cannot be determined from the data. */ + private static long bodyEnd(HttpMessage m, byte[] data, HttpMessage request) { + int start = m.headerEnd; + if (!m.isRequest) { + int code = m.statusCode; + if ((request != null && "HEAD".equals(request.method)) || code / 100 == 1 || code == 204 || code == 304) { + return start; + } + } + if (m.isChunked()) { + return chunkedEnd(data, start); + } + Long length = m.contentLength(); + if (length != null) { + return start + length; + } + // A response without a length runs to the end of the connection; a request has no body + return m.isRequest ? start : data.length; + } + + private static long chunkedEnd(byte[] data, int start) { + int p = start; + while (true) { + int lineEnd = HttpParser.lineEnd(data, p, data.length); + if (lineEnd < 0) { + return -1; + } + String sizeLine = new String(data, p, lineEnd - p, StandardCharsets.ISO_8859_1); + int semicolon = sizeLine.indexOf(';'); + long size; + try { + size = Long.parseLong((semicolon >= 0 ? sizeLine.substring(0, semicolon) : sizeLine).trim(), 16); + } catch (NumberFormatException e) { + return -1; + } + p = lineEnd + 2; + if (size == 0) { + // Optional trailer lines, then a blank line + while (true) { + int end = HttpParser.lineEnd(data, p, data.length); + if (end < 0) { + return -1; + } + boolean empty = end == p; + p = end + 2; + if (empty) { + return p; + } + } + } + if (size < 0 || p + size + 2 > data.length) { + return -1; + } + p += (int) size + 2; + } + } + + @Override + public void write(HttpExchanges parsed, TupleWriter fields) { + ArrayWriter exchanges = fields.array("exchanges"); + int count = Math.max(parsed.requests.size(), parsed.responses.size()); + for (int i = 0; i < count; i++) { + TupleWriter exchange = exchanges.tuple(); + if (i < parsed.requests.size()) { + HttpFields.writeRequest(parsed.requests.get(i), exchange, "request_headers"); + } + if (i < parsed.responses.size()) { + HttpFields.writeResponse(parsed.responses.get(i), exchange, "response_headers"); + } + exchanges.save(); + } + } +} diff --git a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder new file mode 100644 index 00000000000..b576a7739c1 --- /dev/null +++ b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder @@ -0,0 +1 @@ +org.apache.drill.exec.store.pcap.protocol.http.HttpSessionDecoder diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java index 5daab6673fa..abe1ccedbf7 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestSessionizePCAP.java @@ -163,6 +163,25 @@ public void testUnclosedSession() throws Exception { new RowSetComparison(expected).verifyAndClearAll(results); } + @Test + public void testHttpSessions() throws Exception { + String sql = "SELECT parsed_protocol, t.parsed_data.http.exchanges[0].uri AS uri, " + + "t.parsed_data.http.exchanges[0].status_code AS status, t.parsed_data.http.exchanges[0].content_length AS length " + + "FROM cp.`/pcap/http.pcap` t ORDER BY session_closed DESC"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", TypeProtos.MinorType.VARCHAR) + .addNullable("uri", TypeProtos.MinorType.VARCHAR) + .addNullable("status", TypeProtos.MinorType.INT) + .addNullable("length", TypeProtos.MinorType.BIGINT) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("http", "/download.html", 200, 18070L) + .addRow("http", "/pagead/ads?client=ca-pub-2309191948673629&random=1084443430285&lmt=1082467020&format=468x60_as&output=html&url=http%3A%2F%2Fwww.ethereal.com%2Fdownload.html&color_bg=FFFFFF&color_text=333333&color_link=000000&color_url=666633&color_border=666633", 200, 1272L) + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + /** * Helper function to read a file into a String. * @param filePath Input file which is to be read into a String diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java new file mode 100644 index 00000000000..92c44c60207 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.http; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Collections; +import java.util.List; + +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestHttpSessionDecoder extends BaseTest { + + @Test + public void testKeepAliveWithChunkedBody() { + String client = "GET /a HTTP/1.1\r\nHost: h\r\n\r\n" + + "POST /b HTTP/1.1\r\nHost: h\r\nContent-Length: 5\r\n\r\nhello" + + "HEAD /c HTTP/1.1\r\nHost: h\r\n\r\n"; + String server = "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nwiki\r\n5;ext=1\r\npedia\r\n0\r\nX-Trailer: t\r\n\r\n" + + "HTTP/1.1 201 Created\r\nContent-Length: 2\r\n\r\nok" + + "HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n"; + TestHttpParser.Context context = new TestHttpParser.Context(false); + List requests = HttpSessionDecoder.parseStream(TestHttpParser.bytes(client), true, + Collections.emptyList(), context); + List responses = HttpSessionDecoder.parseStream(TestHttpParser.bytes(server), false, requests, context); + assertEquals(3, requests.size()); + assertEquals("/b", requests.get(1).uri); + assertEquals("HEAD", requests.get(2).method); + assertEquals(3, responses.size()); + assertEquals(Integer.valueOf(201), responses.get(1).statusCode); + // The HEAD response has Content-Length but no body + assertEquals(Integer.valueOf(200), responses.get(2).statusCode); + assertTrue(context.warnings.isEmpty()); + } + + @Test + public void testNotHttpSession() { + TestHttpParser.Context context = new TestHttpParser.Context(false); + assertTrue(HttpSessionDecoder.parseStream(TestHttpParser.bytes("SSH-2.0-x\r\n"), true, + Collections.emptyList(), context).isEmpty()); + } + + @Test + public void testGarbageAfterFirstMessageWarns() { + TestHttpParser.Context context = new TestHttpParser.Context(false); + List requests = HttpSessionDecoder.parseStream( + TestHttpParser.bytes("GET / HTTP/1.1\r\n\r\n\u0000\u0001garbage"), true, Collections.emptyList(), context); + assertEquals(1, requests.size()); + assertEquals(1, context.warnings.size()); + assertTrue(context.warnings.get(0), context.warnings.get(0).startsWith("unparseable data in client stream at byte 18")); + } + + @Test + public void testParseReturnsNullWithoutRequest() { + HttpSessionDecoder decoder = new HttpSessionDecoder(); + assertNull(decoder.parse(org.apache.drill.exec.store.pcap.protocol.TcpStream.of(Collections.emptyList()), + org.apache.drill.exec.store.pcap.protocol.TcpStream.of(Collections.emptyList()), new TestHttpParser.Context(false))); + } +} From 84149b63cb70b7e0c3f23575b29115e89a2c9a15 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:07:46 -0400 Subject: [PATCH 14/50] DRILL-8556: Document protocol decoders --- contrib/format-pcapng/README.md | 46 ++++++++++++++++++++++++++++++++ docs/dev/PcapProtocolDecoders.md | 18 +++++++++++++ 2 files changed, 64 insertions(+) diff --git a/contrib/format-pcapng/README.md b/contrib/format-pcapng/README.md index cfeea638de4..6f876e64589 100644 --- a/contrib/format-pcapng/README.md +++ b/contrib/format-pcapng/README.md @@ -30,6 +30,7 @@ Attribute|Default Value|Description ---------|-------------|----------- stat|false|return the statistics data about the each pcapng file if true sessionizeTCPStreams|false|return one row per TCP session instead of one row per packet, for both PCAP and PCAP-NG. A session is written when it closes (FIN or RST). Sessions still open at the end of the file, such as ones that outlived the capture, are written last with `session_closed` false +exposeCredentials|false|include cleartext passwords found by protocol decoders in `parsed_data`; usernames and a `password_present` flag are always included ## PCAP-NG columns @@ -57,3 +58,48 @@ and the MAC address columns are only set for Ethernet. With `stat` set to true, options a block does not have are null. The `comment` column holds the section, interface, name resolution or statistics block comment. + +## Protocol decoding + +Packets and TCP sessions whose application protocol is recognized are decoded into three columns, in both +PCAP and PCAP-NG files: + +Column|Description +------|----------- +parsed_protocol|Name of the decoder that handled the row, such as `dns` or `http`; null if none did +parsed_data|One map per decoder, named after its protocol; only the one named by `parsed_protocol` is filled +decode_error|Why the row could not be fully decoded; null when there was no problem + +```sql +-- DNS questions and answers +SELECT t.parsed_data.dns.questions[0].name AS query, t.parsed_data.dns.answers +FROM dfs.`capture.pcapng` t +WHERE parsed_protocol = 'dns'; + +-- HTTP requests, one row per connection +SELECT src_ip, t.parsed_data.http.exchanges[0].host AS host, t.parsed_data.http.exchanges[0].uri AS uri +FROM table(dfs.`capture.pcap` (type => 'pcap', sessionizeTCPStreams => true)) t +WHERE parsed_protocol = 'http'; + +-- What could not be decoded +SELECT decode_error, count(*) FROM dfs.`capture.pcapng` +WHERE decode_error IS NOT NULL GROUP BY decode_error; +``` + +Built-in decoders: + +Protocol|Rows|Matches +--------|----|------- +dns|packets|DNS, mDNS and LLMNR on ports 53, 5353 and 5355 +http|packets and sessions|HTTP/1.x on ports 80, 591, 3128, 8000, 8008, 8080 and 8888 + +A decoder only handles traffic that parses as its protocol: other traffic on the same port is left undecoded. +Decoding runs only when `parsed_protocol` or `parsed_data` is queried. + +Errors never stop a file from being read. Malformed packets keep the fields that could be read and explain +the problem in `decode_error`. Damage that makes the rest of a file unreadable, or a file that is not a +capture at all, produces one row with only `decode_error` set; exclude such rows with +`WHERE decode_error IS NULL`. + +Fields of each decoder, and how to write a decoder of your own, are described in +`docs/dev/PcapProtocolDecoders.md`. diff --git a/docs/dev/PcapProtocolDecoders.md b/docs/dev/PcapProtocolDecoders.md index f377f256edf..4825c1cb713 100644 --- a/docs/dev/PcapProtocolDecoders.md +++ b/docs/dev/PcapProtocolDecoders.md @@ -282,3 +282,21 @@ already carry the raw bytes. fields are nullable). 5. List the class in the matching `META-INF/services` file. 6. Add the tests described above. + +## Implementation Notes + +Differences from the design above, recorded during phase 1: + +- The two `Packet` accessors for ICMP and ARP bytes arrive with the phase 2 ICMP and ARP decoders; no phase 1 + decoder needs them. +- DNS separates "not DNS" from "malformed DNS" by confidence: a payload whose header is implausible, or whose + first question (or first record, when there are no questions) does not parse, is not DNS and is left + undecoded. Once one question or record has parsed, a later failure is reported in `decode_error`, for + example `dns: truncated answer 1: ...`. +- Every field of `parsed_data` is created with sparse vector sizing (one expected element per array, a small + VARCHAR width). Drill otherwise reserves room for a full batch of values in each of the many mostly empty + decoder fields, which exhausted the batch memory budget. +- Two Drill scan framework bugs had to be fixed for decoder fields that are lists: a repeated map inside a map + lost its offsets in the scan output (`OutputBatchBuilder`), and an array index inside a map was parsed as an + extra array dimension (`ScanProjectionParser`), which rejected projections such as + `parsed_data.dns.questions[0].name`. From 55c34e3411db77c4f647a4d9659c425b36f88b92 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:19:10 -0400 Subject: [PATCH 15/50] DRILL-8556: Remove a debugging test and license the service files A leftover probe test and the three ServiceLoader registration files had no license header, which failed the Apache RAT check. --- ....store.pcap.protocol.PacketProtocolDecoder | 17 +++++++ ...store.pcap.protocol.SessionProtocolDecoder | 17 +++++++ .../drill/exec/store/pcapng/TmpLoader.java | 50 ------------------- ....store.pcap.protocol.PacketProtocolDecoder | 17 +++++++ 4 files changed, 51 insertions(+), 50 deletions(-) delete mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TmpLoader.java diff --git a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder index 556133a8f60..5b788645c0f 100644 --- a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder +++ b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder @@ -1,2 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# org.apache.drill.exec.store.pcap.protocol.dns.DnsDecoder org.apache.drill.exec.store.pcap.protocol.http.HttpPacketDecoder diff --git a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder index b576a7739c1..7dcf9aef5e9 100644 --- a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder +++ b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder @@ -1 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# org.apache.drill.exec.store.pcap.protocol.http.HttpSessionDecoder diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TmpLoader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TmpLoader.java deleted file mode 100644 index cdf90b34878..00000000000 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TmpLoader.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.apache.drill.exec.store.pcapng; - -import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.exec.memory.BufferAllocator; -import org.apache.drill.exec.memory.RootAllocatorFactory; -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.exec.physical.impl.validate.BatchValidator; -import org.apache.drill.exec.physical.resultSet.ResultSetLoader; -import org.apache.drill.exec.physical.resultSet.RowSetLoader; -import org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl; -import org.apache.drill.exec.physical.resultSet.impl.ResultSetOptionBuilder; -import org.apache.drill.exec.record.VectorContainer; -import org.apache.drill.exec.record.metadata.MetadataUtils; -import org.apache.drill.exec.record.metadata.SchemaBuilder; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.vector.accessor.ArrayWriter; -import org.apache.drill.exec.vector.accessor.TupleWriter; -import org.junit.Test; - -public class TmpLoader { - @Test - public void probe() { - BufferAllocator allocator = RootAllocatorFactory.newRoot(DrillConfig.create()); - TupleMetadata dns = new SchemaBuilder().addNullable("id", MinorType.INT) - .addMapArray("questions").addNullable("name", MinorType.VARCHAR).resumeSchema().buildSchema(); - TupleMetadata data = new SchemaBuilder().add(MetadataUtils.newMap("dns", dns)).buildSchema(); - TupleMetadata schema = new SchemaBuilder().addNullable("p", MinorType.VARCHAR) - .add(MetadataUtils.newMap("parsed_data", data)).buildSchema(); - ResultSetLoader rsl = new ResultSetLoaderImpl(allocator, new ResultSetOptionBuilder().readerSchema(schema).build()); - RowSetLoader loader = rsl.writer(); - TupleWriter parsed = loader.tuple("parsed_data"); - rsl.startBatch(); - for (int row = 0; row < 4; row++) { - loader.start(); - if (row < 2) { - TupleWriter d = parsed.tuple("dns"); - d.scalar("id").setInt(row); - ArrayWriter q = d.array("questions"); - q.tuple().scalar("name").setString("q" + row); - q.save(); - } - loader.save(); - } - VectorContainer c = rsl.harvest(); - System.out.println("PROBE rows=" + c.getRecordCount() + " valid=" + BatchValidator.validate(c)); - c.zeroVectors(); - rsl.close(); - allocator.close(); - } -} diff --git a/contrib/format-pcapng/src/test/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder b/contrib/format-pcapng/src/test/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder index f4f6b534a12..a8738c50aac 100644 --- a/contrib/format-pcapng/src/test/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder +++ b/contrib/format-pcapng/src/test/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder @@ -1 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# org.apache.drill.exec.store.pcap.protocol.EchoTestDecoder From 78864e2e494f793f94483f4390c53e3ef6bb6c98 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:22:51 -0400 Subject: [PATCH 16/50] DRILL-8556: Pair HTTP requests with their final responses Interim 1xx responses such as 100 Continue were paired as answers, shifting every later request-response pair on the connection by one. --- .../pcap/protocol/http/HttpSessionDecoder.java | 8 ++++++-- .../protocol/http/TestHttpSessionDecoder.java | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java index 72e50dff552..928a8909587 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java @@ -87,11 +87,15 @@ static List parseStream(byte[] data, boolean requests, List data.length) { break; // body continues past the captured data diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java index 92c44c60207..b715a2cafac 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java @@ -74,4 +74,20 @@ public void testParseReturnsNullWithoutRequest() { assertNull(decoder.parse(org.apache.drill.exec.store.pcap.protocol.TcpStream.of(Collections.emptyList()), org.apache.drill.exec.store.pcap.protocol.TcpStream.of(Collections.emptyList()), new TestHttpParser.Context(false))); } + + @Test + public void testInterimResponsesDoNotShiftPairing() { + // 100 Continue is not the answer to the request; the final status is + String client = "POST /upload HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\nok" + + "GET /next HTTP/1.1\r\n\r\n"; + String server = "HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 201 Created\r\nContent-Length: 0\r\n\r\n" + + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"; + TestHttpParser.Context context = new TestHttpParser.Context(false); + List requests = HttpSessionDecoder.parseStream(TestHttpParser.bytes(client), true, + Collections.emptyList(), context); + List responses = HttpSessionDecoder.parseStream(TestHttpParser.bytes(server), false, requests, context); + assertEquals(2, responses.size()); + assertEquals(Integer.valueOf(201), responses.get(0).statusCode); + assertEquals(Integer.valueOf(200), responses.get(1).statusCode); + } } From 400fc7f5546f13583b9310165ad61492f5e1d898 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:22:51 -0400 Subject: [PATCH 17/50] DRILL-8556: Bound length fields and report packet errors in session mode - Cap PCAP-NG block lengths at 16 MiB and the PCAP snapshot length at libpcap's 262144 bytes, so a damaged or crafted header becomes an error row instead of a multi-gigabyte allocation - In session mode, where packets are not rows, report each distinct packet or interface error once as an error row instead of dropping it --- .../exec/store/pcap/PcapBatchReader.java | 19 +++++++--- .../store/pcap/protocol/ProtocolColumns.java | 19 ++++++++++ .../exec/store/pcapng/PcapngBatchReader.java | 26 +++++++++++--- .../exec/store/pcap/TestPcapDecoding.java | 23 ++++++++++++ .../exec/store/pcapng/TestPcapngDecoding.java | 33 ++++++++++++++++++ .../pcapng/TestPcapngStatRecordReader.java | 2 +- .../test/resources/fixtures/pcap_fixtures.py | 18 ++++++++++ .../src/test/resources/pcap/huge_snaplen.pcap | Bin 0 -> 154 bytes .../src/test/resources/pcap/malformed.pcap | Bin 0 -> 219 bytes .../test/resources/pcapng/huge_block.pcapng | Bin 0 -> 140 bytes 10 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 contrib/format-pcapng/src/test/resources/pcap/huge_snaplen.pcap create mode 100644 contrib/format-pcapng/src/test/resources/pcap/malformed.pcap create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/huge_block.pcapng diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java index 99fe4b40e4e..4ca783ddb9f 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java @@ -45,6 +45,8 @@ public class PcapBatchReader implements ManagedReader { protected static final int BUFFER_SIZE = 500_000; + // libpcap's largest snapshot length; a larger header value is not trusted for allocation + private static final int MAX_SNAPLEN = 262_144; private static final Logger logger = LoggerFactory.getLogger(PcapBatchReader.class); private final FileDescrip file; private PacketDecoder decoder; @@ -90,6 +92,8 @@ public class PcapBatchReader implements ManagedReader { // Set when a damaged record makes the rest of the file unreadable private boolean finished; private int packetCount; + // Snapshot length from the file header, capped at MAX_SNAPLEN + private int maxLength; public PcapBatchReader(PcapFormatConfig readerConfig, FileSchemaNegotiator negotiator) { @@ -163,7 +167,9 @@ private void openFile() { try { fsStream = file.fileSystem().openPossiblyCompressedStream(file.split().getPath()); decoder = new PacketDecoder(fsStream); - buffer = new byte[BUFFER_SIZE + decoder.getMaxLength()]; + int snaplen = decoder.getMaxLength(); + maxLength = snaplen <= 0 || snaplen > MAX_SNAPLEN ? MAX_SNAPLEN : snaplen; + buffer = new byte[BUFFER_SIZE + maxLength]; validBytes = fsStream.read(buffer); } catch (IOException e) { decoder = null; @@ -217,13 +223,13 @@ private boolean parseNextPacket(RowSetLoader rowWriter) { if (offset >= validBytes) { return false; } - if (validBytes - offset < decoder.getMaxLength()) { + if (validBytes - offset < maxLength) { getNextPacket(rowWriter); } int old = offset; try { - offset = decoder.decodePacket(buffer, offset, packet, decoder.getMaxLength(), validBytes); + offset = decoder.decodePacket(buffer, offset, packet, maxLength, validBytes); } catch (RuntimeException e) { // The record's length cannot be trusted, so neither can anything after it finished = true; @@ -241,7 +247,12 @@ private boolean parseNextPacket(RowSetLoader rowWriter) { } if (sessionizer != null) { - sessionizer.addPacket(packet); + if (packet.getDecodeError() != null) { + // Packets are not rows here; a packet that failed to decode cannot join a session + protocolColumns.writeErrorRowOnce("packet: " + packet.getDecodeError()); + } else { + sessionizer.addPacket(packet); + } } else { addDataToTable(packet, decoder.getNetwork(), rowWriter); } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java index cf88f58b770..0b0797f92fe 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java @@ -18,8 +18,10 @@ package org.apache.drill.exec.store.pcap.protocol; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -53,6 +55,8 @@ public enum Mode { PACKET, SESSION, STAT } private final boolean decoding; // Errors written so far, by prefix (dns, packet, file, ...), for the end-of-file warning private final Map errorCounts = new TreeMap<>(); + // Errors already written by writeErrorRowOnce + private final Set reportedErrors = new HashSet<>(); public static void addColumns(SchemaBuilder schema, Mode mode, ProtocolDecoders decoders) { if (mode != Mode.STAT) { @@ -151,6 +155,21 @@ public void writeErrorRow(String error) { count(error); } + /** + * Writes an error row unless the same error was already written. Used where + * packets are not rows (session mode), so each distinct problem appears once. + * + * @return true if a row was written + */ + public boolean writeErrorRowOnce(String error) { + if (!reportedErrors.add(error)) { + count(error); + return false; + } + writeErrorRow(error); + return true; + } + private void count(String error) { int colon = error.indexOf(':'); errorCounts.merge(colon > 0 ? error.substring(0, colon) : error, 1, Integer::sum); diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java index ea0cd4faec8..fe20a27659b 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java @@ -69,6 +69,8 @@ public class PcapngBatchReader implements ManagedReader { private static final int INTERFACE_STATISTICS_TYPE = 5; private static final int ENHANCED_PACKET_TYPE = 6; private static final int BYTE_ORDER_MAGIC = 0x1A2B3C4D; + // libpcap's limit; a longer block is treated as damage rather than allocated + private static final int MAX_BLOCK_LENGTH = 16 * 1024 * 1024; // Option codes private static final int OPT_ENDOFOPT = 0; private static final int OPT_COMMENT = 1; @@ -94,6 +96,8 @@ public class PcapngBatchReader implements ManagedReader { private RowSetLoader loader; // Set when TCP packets are grouped into sessions instead of returned as rows private TcpSessionizer sessionizer; + // A packet held back because its block already wrote an error row this iteration + private PacketDecoder pendingPacket; private ProtocolColumns protocolColumns; private InputStream in; private Path path; @@ -166,6 +170,12 @@ private ProtocolColumns.Mode rowMode() { @Override public boolean next() { while (!loader.isFull()) { + if (pendingPacket != null) { + PacketDecoder packet = pendingPacket; + pendingPacket = null; + sessionizer.addPacket(packet); + continue; + } PcapngBlock block; try { block = nextRow(); @@ -181,10 +191,16 @@ public boolean next() { return sessionizer != null && !sessionizer.writeOpenSessions(); } if (sessionizer != null) { - if (block.errorRow) { - protocolColumns.writeErrorRow(String.join("; ", block.errors)); - } else if (block.packet != null) { - sessionizer.addPacket(block.packet); + // Packets are not rows here, so their problems become error rows, once each. + // A block writes at most one row per iteration so a full batch is never overrun. + boolean wroteRow = block.errors != null + && protocolColumns.writeErrorRowOnce(String.join("; ", block.errors)); + if (block.packet != null) { + if (wroteRow) { + pendingPacket = block.packet; + } else { + sessionizer.addPacket(block.packet); + } } } else { processBlock(block); @@ -229,7 +245,7 @@ private PcapngBlock nextRow() throws IOException { buf.order(byteOrder); } int totalLength = buf.getInt(4); - if (totalLength < consumed + 4 || totalLength % 4 != 0) { + if (totalLength < consumed + 4 || totalLength % 4 != 0 || totalLength > MAX_BLOCK_LENGTH) { finished = true; return PcapngBlock.errorRow("file: invalid block length " + totalLength + " at byte " + blockStart); } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java index 83ef54eef80..0c80648bf49 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java @@ -105,4 +105,27 @@ public void testHttpPackets() throws Exception { .build(); new RowSetComparison(expected).verifyAndClearAll(results); } + + @Test + public void testOversizedSnapshotLengthIsClamped() throws Exception { + // The buffer is sized from a capped snapshot length; normal packets still read + String sql = "select src_ip, decode_error from dfs.`pcap/huge_snaplen.pcap`"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("src_ip", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("10.0.0.1", null) + .addRow("10.0.0.1", null) + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testSessionModeReportsPacketErrors() throws Exception { + String sql = "select decode_error from table(dfs.`pcap/malformed.pcap` (type => 'pcap', sessionizeTCPStreams => true)) " + + "where decode_error is not null"; + assertEquals("packet: Invalid IPv4 header length 12", client.queryBuilder().sql(sql).singletonString()); + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java index fc4cc6217fd..a076e96f41a 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java @@ -124,4 +124,37 @@ public void testDns() throws Exception { .build(); new RowSetComparison(expected).verifyAndClearAll(results); } + + @Test + public void testOversizedBlockLengthIsReported() throws Exception { + String sql = "select src_ip, decode_error from dfs.`pcapng/huge_block.pcapng`"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("src_ip", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("10.0.0.1", null) + .addRow(null, "file: invalid block length 2147483632 at byte 116") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testSessionModeReportsPacketErrors() throws Exception { + // Packets are not rows in session mode, so each distinct problem gets one error row + String sql = "select decode_error from table(dfs.`pcapng/errors.pcapng` (type => 'pcapng', sessionizeTCPStreams => true)) " + + "where decode_error is not null"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("packet: Invalid IPv4 header length 12") + .addRow("file: packet references undefined interface 5") + .addRow("file: interface 1 has unsupported if_tsresol 127; timestamps assume microseconds") + .addRow("file: block at byte 348 has invalid captured length 9999") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java index d4915fea082..028b32cd2fd 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngStatRecordReader.java @@ -115,7 +115,7 @@ public void testSerDe() throws Exception { long cnt = queryBuilder().physical(plan).singletonLong(); // The error row of bad_block_length.pcapng is counted too - assertEquals("Counts should match", 65, cnt); + assertEquals("Counts should match", 68, cnt); } @Test diff --git a/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py index c046182821f..14d63982951 100644 --- a/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py +++ b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py @@ -163,9 +163,27 @@ def dns(): f += epb(0, ipv4(17, udp(53, 5000, response[:-3]), '8.8.8.8', '10.0.0.1')) write(os.path.join(PCAPNG, 'dns.pcapng'), f) +def oversized(): + # A block length near 2 GB must be reported, not allocated + good = ipv4(17, udp(4000, 7, b'ECHO:ok'), '10.0.0.1', '10.0.0.2') + f = shb() + idb(101) + epb(0, good) + print('huge_block.pcapng: huge block at byte', len(f)) + f += struct.pack('`kf3=CWh42(d`#J_-n Voq?Z$!PVKr-zq;FW(e5^001oW4$=Ss literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/pcap/malformed.pcap b/contrib/format-pcapng/src/test/resources/pcap/malformed.pcap new file mode 100644 index 0000000000000000000000000000000000000000..77f64247264a711be965bc453f9519b1ce1e21ea GIT binary patch literal 219 zcmca|c+)~A1{MYw`2U}Qff2}2t(}|-5;KHiCJ=`SOfqsXxH2#(g9ID|85p>L8i1II ge*ptK13v?UtFwo{RerV_UIUy78=#KY03uuf0DBG=Y5)KL literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/pcapng/huge_block.pcapng b/contrib/format-pcapng/src/test/resources/pcapng/huge_block.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..36ae11edee77c9f9284432e438fa5a772f703f54 GIT binary patch literal 140 zcmd<$<>iuLU|{gI(UxKa(*L0VBnIM(0C6f<9KvP;vR!}}1m5zpF>KI|IG_w-VBi932Vy4v1q|#A{6INp4}YutY_MLC86W=tuLn{n002!V7Q6re literal 0 HcmV?d00001 From ae6595e815ef77a062be9ae1ff76c25de9d23821 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:36:44 -0400 Subject: [PATCH 18/50] DRILL-8556: Rename the PCAP-NG timestamp column to packet_timestamp timestamp is a reserved word, so every query had to quote it, and an unquoted reference such as MIN(timestamp) was a parse error. packet_timestamp is also the name classic PCAP files already use. --- .../drill/exec/store/pcapng/PcapColumn.java | 2 +- .../exec/store/pcap/TestPcapRecordReader.java | 2 +- .../store/pcapng/TestPcapngRecordReader.java | 22 +++++++++---------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java index 71cd5552e5a..d7fb721623d 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapColumn.java @@ -38,7 +38,7 @@ public abstract class PcapColumn { static { // Basic - columns.put("timestamp", new PcapTimestamp()); + columns.put("packet_timestamp", new PcapTimestamp()); columns.put("packet_length", new PcapPacketLength()); columns.put("type", new PcapType()); columns.put("src_ip", new PcapSrcIp()); diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java index adc4d2dd88b..ed92884ca2c 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapRecordReader.java @@ -67,7 +67,7 @@ public void testCountQuery() throws Exception { @Test public void testDistinctQuery() throws Exception { // omit data field from distinct count for now - runSQLVerifyCount("select distinct type, network, `timestamp`, src_ip, dst_ip, src_port, dst_port, src_mac_address, dst_mac_address, tcp_session, packet_length from dfs.`pcap/tcp-1.pcap`", 1); + runSQLVerifyCount("select distinct type, network, packet_timestamp, src_ip, dst_ip, src_port, dst_port, src_mac_address, dst_mac_address, tcp_session, packet_length from dfs.`pcap/tcp-1.pcap`", 1); } @Test diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java index 5e002b4fdf1..35b84bf98cb 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngRecordReader.java @@ -62,14 +62,14 @@ public void testStarQuery() throws Exception { @Test public void testExplicitQuery() throws Exception { - String sql = "select type, packet_length, `timestamp` from dfs.`pcapng/sniff.pcapng` where type = 'ARP' limit 2"; + String sql = "select type, packet_length, packet_timestamp from dfs.`pcapng/sniff.pcapng` where type = 'ARP' limit 2"; QueryBuilder builder = client.queryBuilder().sql(sql); RowSet sets = builder.rowSet(); TupleMetadata schema = new SchemaBuilder() .addNullable("type", MinorType.VARCHAR) .addNullable("packet_length", MinorType.INT) - .addNullable("timestamp", MinorType.TIMESTAMP) + .addNullable("packet_timestamp", MinorType.TIMESTAMP) .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) @@ -116,14 +116,14 @@ public void testMixedPcapAndPcapngDirectory() throws Exception { @Test public void testExplicitQueryWithCompressedFile() throws Exception { QueryTestUtil.generateCompressedFile("pcapng/sniff.pcapng", "zip", "pcapng/sniff.pcapng.zip"); - String sql = "select type, packet_length, `timestamp` from dfs.`pcapng/sniff.pcapng.zip` where type = 'ARP' limit 2"; + String sql = "select type, packet_length, packet_timestamp from dfs.`pcapng/sniff.pcapng.zip` where type = 'ARP' limit 2"; QueryBuilder builder = client.queryBuilder().sql(sql); RowSet sets = builder.rowSet(); TupleMetadata schema = new SchemaBuilder() .addNullable("type", MinorType.VARCHAR) .addNullable("packet_length", MinorType.INT) - .addNullable("timestamp", MinorType.TIMESTAMP) + .addNullable("packet_timestamp", MinorType.TIMESTAMP) .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) @@ -137,7 +137,7 @@ public void testExplicitQueryWithCompressedFile() throws Exception { @Test public void testCaseInsensitiveQuery() throws Exception { - String sql = "select `timestamp`, paCket_dAta, TyPe from dfs.`pcapng/sniff.pcapng`"; + String sql = "select packet_timestamp, paCket_dAta, TyPe from dfs.`pcapng/sniff.pcapng`"; QueryBuilder builder = client.queryBuilder().sql(sql); RowSet sets = builder.rowSet(); @@ -174,7 +174,7 @@ public void testValidHeaders() throws Exception { RowSet sets = client.queryBuilder().sql(sql).rowSet(); TupleMetadata schema = new SchemaBuilder() - .addNullable("timestamp", MinorType.TIMESTAMP) + .addNullable("packet_timestamp", MinorType.TIMESTAMP) .addNullable("packet_length", MinorType.INT) .addNullable("type", MinorType.VARCHAR) .addNullable("src_ip", MinorType.VARCHAR) @@ -253,7 +253,7 @@ public void testManyInterfaces() throws Exception { */ @Test public void testLinkTypesAndInterfaces() throws Exception { - String sql = "select interface_id, link_type, interface_name, `timestamp`, src_ip, src_port, " + + String sql = "select interface_id, link_type, interface_name, packet_timestamp, src_ip, src_port, " + "src_mac_address, captured_length from dfs.`pcapng/metadata.pcapng`"; RowSet results = client.queryBuilder().sql(sql).rowSet(); @@ -261,7 +261,7 @@ public void testLinkTypesAndInterfaces() throws Exception { .addNullable("interface_id", MinorType.INT) .addNullable("link_type", MinorType.INT) .addNullable("interface_name", MinorType.VARCHAR) - .addNullable("timestamp", MinorType.TIMESTAMP) + .addNullable("packet_timestamp", MinorType.TIMESTAMP) .addNullable("src_ip", MinorType.VARCHAR) .addNullable("src_port", MinorType.INT) .addNullable("src_mac_address", MinorType.VARCHAR) @@ -381,7 +381,7 @@ public void testGroupBy() throws Exception { @Test public void testDistinctQuery() throws Exception { - String sql = "select distinct `timestamp`, src_ip from dfs.`pcapng/sniff.pcapng`"; + String sql = "select distinct packet_timestamp, src_ip from dfs.`pcapng/sniff.pcapng`"; QueryBuilder builder = client.queryBuilder().sql(sql); RowSet sets = builder.rowSet(); @@ -406,14 +406,14 @@ public void testPcapNGFileWithPcapExt() throws Exception { @Test public void testInlineSchema() throws Exception { - String sql = "SELECT type, packet_length, `timestamp` FROM table(dfs.`pcapng/sniff.pcapng` " + + String sql = "SELECT type, packet_length, packet_timestamp FROM table(dfs.`pcapng/sniff.pcapng` " + "(type => 'pcapng', stat => false, sessionizeTCPStreams => false )) where type = 'ARP' limit 2"; RowSet sets = client.queryBuilder().sql(sql).rowSet(); TupleMetadata schema = new SchemaBuilder() .addNullable("type", MinorType.VARCHAR) .addNullable("packet_length", MinorType.INT) - .addNullable("timestamp", MinorType.TIMESTAMP) + .addNullable("packet_timestamp", MinorType.TIMESTAMP) .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) From 8c19ccde5d245f93aa0e728315e42452098ea899 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:40:27 -0400 Subject: [PATCH 19/50] DRILL-8556: Fix minor decoder issues found in review - Keep the IP addresses of a packet whose IPv4 header cannot be parsed, with ports left empty, instead of discarding everything after the link layer - DNS records past the 64-item cap are skipped, so later sections are kept - An invalid HTTP Content-Length is reported once - Compare HTTP chunk sizes without risk of overflow - Give each reader its own copy of the parsed_data schema --- .../drill/exec/store/pcap/decoder/Packet.java | 14 ++++++-- .../store/pcap/protocol/ProtocolColumns.java | 3 +- .../store/pcap/protocol/dns/DnsParser.java | 11 ++++--- .../store/pcap/protocol/http/HttpMessage.java | 11 +++++++ .../protocol/http/HttpSessionDecoder.java | 3 +- .../exec/store/pcapng/PacketDecoder.java | 3 +- .../exec/store/pcapng/PcapngBatchReader.java | 8 +++-- .../store/pcap/TestPacketDecodeErrors.java | 8 ++++- .../pcap/protocol/TestProtocolDecoders.java | 19 +++++++++++ .../pcap/protocol/dns/TestDnsDecoder.java | 33 ++++++++++++++++++- .../pcap/protocol/http/TestHttpParser.java | 11 +++++++ .../protocol/http/TestHttpSessionDecoder.java | 13 ++++++++ .../exec/store/pcapng/TestPcapngDecoding.java | 2 +- 13 files changed, 124 insertions(+), 15 deletions(-) diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java index f263e844868..dcfbc77a5b1 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java @@ -363,6 +363,9 @@ public static String formatFlags(int flags) { } public int getSrc_port() { + if (transportUnknown()) { + return 0; + } if (isPPPoV6Packet()) { return getPort(64); } @@ -370,12 +373,20 @@ public int getSrc_port() { } public int getDst_port() { + if (transportUnknown()) { + return 0; + } if (isPPPoV6Packet()) { return getPort(66); } return getPort(2); } + /** True for an IP packet whose header could not be parsed far enough to find the transport layer. */ + private boolean transportUnknown() { + return (isIpV4Packet() || isIpV6Packet()) && transportOffset < 0; + } + public boolean isCorrupt(){ return isCorrupt; } @@ -518,10 +529,9 @@ private void decodeEtherPacket() { protocol = getByte(raw, etherOffset + 48); } } catch (RuntimeException e) { - // Keep the link-layer fields; treat the rest as undecodable + // Keep what was read (link layer, IP addresses); the transport layer is unknown isCorrupt = true; decodeError = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); - etherProtocol = 0; transportOffset = -1; } // everything is decoded lazily diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java index 0b0797f92fe..b4265786907 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java @@ -66,8 +66,9 @@ public static void addColumns(SchemaBuilder schema, Mode mode, ProtocolDecoders schema.addNullable(DECODE_ERROR, MinorType.VARCHAR); } + /** A copy per reader: map metadata binds its member schema to the parent map. */ private static TupleMetadata dataSchema(Mode mode, ProtocolDecoders decoders) { - return mode == Mode.SESSION ? decoders.sessionDataSchema() : decoders.packetDataSchema(); + return (mode == Mode.SESSION ? decoders.sessionDataSchema() : decoders.packetDataSchema()).copy(); } public ProtocolColumns(RowSetLoader loader, Mode mode, ProtocolDecoders decoders, boolean exposeCredentials) { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsParser.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsParser.java index ad32ca17fdd..797cba0f93a 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsParser.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dns/DnsParser.java @@ -116,12 +116,15 @@ private DnsMessage message(DecoderContext context) { for (int i = 0; i < counts[s]; i++) { if (i == MAX_ITEMS) { context.warn(sections[s] + "s truncated to " + MAX_ITEMS); - return m; } try { - @SuppressWarnings("unchecked") - List list = (List) lists[s]; - list.add(record()); + // Records past the cap are still read, to reach the sections after them + DnsMessage.Record record = record(); + if (i < MAX_ITEMS) { + @SuppressWarnings("unchecked") + List list = (List) lists[s]; + list.add(record); + } confident = true; } catch (IllegalArgumentException e) { if (!confident) { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpMessage.java index 4a6cc015a74..fc5e19ef803 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpMessage.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpMessage.java @@ -37,6 +37,9 @@ public class HttpMessage { public boolean passwordPresent; public String password; private DecoderContext context; + // Content-Length is read once, so an invalid value is reported once + private boolean contentLengthRead; + private Long contentLength; void setContext(DecoderContext context) { this.context = context; @@ -54,6 +57,14 @@ public String header(String name) { /** Content-Length, or null if absent or invalid. */ public Long contentLength() { + if (!contentLengthRead) { + contentLengthRead = true; + contentLength = readContentLength(); + } + return contentLength; + } + + private Long readContentLength() { String value = header("Content-Length"); if (value == null) { return null; diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java index 928a8909587..4edc96edcf4 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/http/HttpSessionDecoder.java @@ -162,7 +162,8 @@ private static long chunkedEnd(byte[] data, int start) { } } } - if (size < 0 || p + size + 2 > data.length) { + // Compared this way round so a huge chunk size cannot overflow + if (size < 0 || size > data.length - p - 2) { return -1; } p += (int) size + 2; diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java index 96ca36dfbde..70f28ddc7e5 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java @@ -83,9 +83,10 @@ public boolean readPcapng(final byte[] raw, final int linkType) { return true; } } catch (RuntimeException e) { + // Still an IP packet: addresses can be read, the transport layer cannot setDecodeError(e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName()); - etherProtocol = 0; transportOffset = -1; + return true; } return false; } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java index fe20a27659b..f71a0980508 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java @@ -333,11 +333,13 @@ private PcapngBlock readPacket(ByteBuffer block, long blockStart) { // Decode once here instead of once per projected column if (!isSkipQuery() || sessionizer != null) { PacketDecoder packet = new PacketDecoder(); - if (packet.readPcapng(row.data, row.linkType)) { + boolean decoded = packet.readPcapng(row.data, row.linkType); + if (packet.getDecodeError() != null) { + row.addError("packet: " + packet.getDecodeError()); + } + if (decoded) { packet.setTimestamp(row.timestamp); row.packet = packet; - } else if (packet.getDecodeError() != null) { - row.addError("packet: " + packet.getDecodeError()); } } return row; diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketDecodeErrors.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketDecodeErrors.java index 13ae67f9f63..523885cbb40 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketDecodeErrors.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketDecodeErrors.java @@ -39,9 +39,13 @@ public void testMalformedIpv4HeaderInPcapng() { byte[] raw = TestPackets.ipv4("10.0.0.1", "10.0.0.2", 17, new byte[8]); raw[0] = 0x43; // IHL 3: shorter than the minimum header PacketDecoder packet = new PacketDecoder(); - assertFalse(packet.readPcapng(raw, 101)); + // Still an IPv4 packet: the addresses are read, the transport layer is not + assertTrue(packet.readPcapng(raw, 101)); assertNotNull(packet.getDecodeError()); assertTrue(packet.getDecodeError(), packet.getDecodeError().contains("header length")); + assertEquals("10.0.0.1", packet.getSourceIpAddressString()); + assertFalse(packet.isUdpPacket()); + assertEquals(0, packet.getSrc_port()); } @Test @@ -58,6 +62,8 @@ public void testMalformedIpv4HeaderInPcap() { packet.decodePcap(record, 0, false, 65535); assertTrue(packet.isCorrupt()); assertNotNull(packet.getDecodeError()); + assertEquals("10.0.0.2", packet.getDestinationIpAddressString()); + assertEquals(0, packet.getDst_port()); } @Test diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java index 5851eea33fe..18735a8b02e 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestProtocolDecoders.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -27,6 +28,7 @@ import java.util.Collections; import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.SchemaBuilder; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.store.pcap.decoder.Packet; import org.apache.drill.test.BaseTest; @@ -122,4 +124,21 @@ public int priority() { ProtocolDecoders decoders = new ProtocolDecoders(Arrays.asList(low, high)); assertEquals("high", decode(decoders, echo("ECHO:hi"), new RowDecoderContext(false)).protocol()); } + + @Test + public void testEachReaderGetsItsOwnDataSchema() { + // Map metadata binds its member schema to a parent, so readers must not share one instance + ProtocolDecoders decoders = ProtocolDecoders.get(); + SchemaBuilder first = new SchemaBuilder(); + ProtocolColumns.addColumns(first, ProtocolColumns.Mode.PACKET, decoders); + SchemaBuilder second = new SchemaBuilder(); + ProtocolColumns.addColumns(second, ProtocolColumns.Mode.PACKET, decoders); + TupleMetadata a = first.buildSchema().metadata(ProtocolColumns.PARSED_DATA).tupleSchema(); + TupleMetadata b = second.buildSchema().metadata(ProtocolColumns.PARSED_DATA).tupleSchema(); + assertNotSame(a, b); + assertNotSame(decoders.packetDataSchema(), a); + assertEquals(decoders.packetDataSchema().size(), a.size()); + // The copy keeps the sparse sizing that keeps batches within their memory budget + assertEquals(1, a.metadata("dns").tupleSchema().metadata("answers").expectedElementCount()); + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dns/TestDnsDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dns/TestDnsDecoder.java index f12c67aa615..bfee46ad1cd 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dns/TestDnsDecoder.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dns/TestDnsDecoder.java @@ -121,7 +121,7 @@ public void testTruncatedAnswerIsMalformedDns() { @Test public void testCompressionPointerLoop() { - // Answer name points to itself + // Answer name points forward, past itself byte[] loop = ByteBuffer.allocate(12).put((byte) 0xC0).put((byte) 29).putShort((short) 1).putShort((short) 1) .putInt(0).putShort((short) 0).array(); byte[] m = message(1, 0x8180, 1, 1, 0, 0, concat(question("a.b", 1), loop)); @@ -157,4 +157,35 @@ public void testRecordTypes() { assertEquals("10 example.com", m.answers.get(0).data); assertFalse(m.truncated); } + + @Test + public void testRecordCapKeepsLaterSections() { + // Records past the cap are skipped, not the sections after them + ByteArrayOutputStream body = new ByteArrayOutputStream(); + byte[] q = question("a.b", 1); + body.write(q, 0, q.length); + for (int i = 0; i < 71; i++) { + byte[] a = answerA(1, new byte[4]); + body.write(a, 0, a.length); + } + Context context = new Context(); + DnsMessage m = DnsParser.parse(message(1, 0x8180, 1, 70, 0, 1, body.toByteArray()), 0, context); + assertEquals(64, m.answers.size()); + assertEquals(1, m.additionals.size()); + assertEquals(1, context.warnings.size()); + } + + @Test + public void testCompressionPointerToItself() { + // The answer name (at offset 21) is a pointer to offset 21 + byte[] self = ByteBuffer.allocate(12).put((byte) 0xC0).put((byte) 21).putShort((short) 1).putShort((short) 1) + .putInt(0).putShort((short) 0).array(); + byte[] m = message(1, 0x8180, 1, 1, 0, 0, concat(question("a.b", 1), self)); + try { + DnsParser.parse(m, 0, new Context()); + fail("expected malformed DNS"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("pointer")); + } + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpParser.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpParser.java index eaea50c42dd..a6d908b9d25 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpParser.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpParser.java @@ -126,4 +126,15 @@ public void testHeaderCap() { public void testChunked() { assertTrue(parse("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", false).isChunked()); } + + @Test + public void testInvalidContentLengthWarnsOnce() { + // Parsing and writing both read the length; the problem is reported once + byte[] b = bytes("HTTP/1.1 200 OK\r\nContent-Length: abc\r\n\r\n"); + Context context = new Context(false); + HttpMessage m = HttpParser.parseMessage(b, 0, b.length, context); + assertNull(m.contentLength()); + assertNull(m.contentLength()); + assertEquals(1, context.warnings.size()); + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java index b715a2cafac..78fac4b4f78 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/http/TestHttpSessionDecoder.java @@ -90,4 +90,17 @@ public void testInterimResponsesDoNotShiftPairing() { assertEquals(Integer.valueOf(201), responses.get(0).statusCode); assertEquals(Integer.valueOf(200), responses.get(1).statusCode); } + + @Test + public void testHugeChunkSizeEndsTheStream() { + // A chunk claiming nearly Long.MAX_VALUE bytes cannot be in the data; parsing stops there + String client = "GET /a HTTP/1.1\r\n\r\nGET /b HTTP/1.1\r\n\r\n"; + String server = "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n7fffffffffffffff\r\nabc\r\n0\r\n\r\n" + + "HTTP/1.1 204 No Content\r\n\r\n"; + TestHttpParser.Context context = new TestHttpParser.Context(false); + List requests = HttpSessionDecoder.parseStream(TestHttpParser.bytes(client), true, + Collections.emptyList(), context); + List responses = HttpSessionDecoder.parseStream(TestHttpParser.bytes(server), false, requests, context); + assertEquals(1, responses.size()); + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java index a076e96f41a..beb099a006d 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java @@ -70,7 +70,7 @@ public void testPacketErrorsDoNotStopTheFile() throws Exception { .addNullable("decode_error", MinorType.VARCHAR) .buildSchema(); RowSet expected = new RowSetBuilder(client.allocator(), schema) - .addRow(0, null, "packet: Invalid IPv4 header length 12") + .addRow(0, "10.0.0.1", "packet: Invalid IPv4 header length 12") .addRow(0, "10.0.0.1", null) .addRow(5, null, "file: packet references undefined interface 5") .addRow(1, "10.0.0.1", "file: interface 1 has unsupported if_tsresol 127; timestamps assume microseconds") From d29f569a0895c22a08b2cf66b60441538300c87e Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:40:27 -0400 Subject: [PATCH 20/50] DRILL-8556: Document the packet_timestamp column --- contrib/format-pcapng/README.md | 2 ++ docs/dev/PcapProtocolDecoders.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/format-pcapng/README.md b/contrib/format-pcapng/README.md index 6f876e64589..d4ea28a3d22 100644 --- a/contrib/format-pcapng/README.md +++ b/contrib/format-pcapng/README.md @@ -37,6 +37,8 @@ exposeCredentials|false|include cleartext passwords found by protocol decoders i PCAP-NG files are streamed one block at a time, so file size is not limited by memory. In addition to the packet columns above, packet queries return the following metadata. +The time of each packet is in `packet_timestamp`, the same column name as in PCAP files. (It was named +`timestamp` in PCAP-NG files before; that name is a reserved word that had to be quoted in every query.) Timestamps honor each interface's `if_tsresol` and `if_tsoffset`. Column|Description diff --git a/docs/dev/PcapProtocolDecoders.md b/docs/dev/PcapProtocolDecoders.md index 4825c1cb713..718747c5029 100644 --- a/docs/dev/PcapProtocolDecoders.md +++ b/docs/dev/PcapProtocolDecoders.md @@ -175,7 +175,7 @@ Structural damage that makes the rest of the file unreadable (invalid block leng Session stream gap that stops a decoder early | Fields parsed before the gap are kept; `decode_error` = `: stopped at missing data in stream at byte ` An **error row** has `decode_error` set and every other column null. To allow this, the PCAP-NG packet -columns that are currently `REQUIRED` (`timestamp`, `packet_length`, `captured_length`, `interface_id`, +columns that are currently `REQUIRED` (`packet_timestamp`, `packet_length`, `captured_length`, `interface_id`, `link_type`) become nullable. Queries that count packets can exclude error rows with `WHERE decode_error IS NULL` or count them with `WHERE decode_error IS NOT NULL`. From 9c9c0dd499e3ad0c5194b53fbe3096d748584e47 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:49:17 -0400 Subject: [PATCH 21/50] DRILL-8556: Add link and IP payload accessors for protocol decoders getLinkPayload returns the bytes after the link-layer header (the ARP message of an ARP frame); getIpPayload returns the bytes after the IP header (the ICMP message of an ICMP packet). ARP frames read from PCAP-NG now record where their link payload starts on every link type. --- .../drill/exec/store/pcap/decoder/Packet.java | 26 ++++++ .../exec/store/pcapng/PacketDecoder.java | 3 +- .../exec/store/pcap/TestPacketPayloads.java | 92 +++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketPayloads.java diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java index dcfbc77a5b1..66c439f3abe 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java @@ -420,6 +420,32 @@ public byte[] getData() { return payloadStart < payloadEnd ? Arrays.copyOfRange(raw, payloadStart, payloadEnd) : null; } + /** + * The bytes after the link-layer header, up to the end of the captured data: + * for example the whole ARP message of an ARP frame. + * + * @return null if there are none + */ + public byte[] getLinkPayload() { + int start = etherOffset + PacketConstants.IP_OFFSET; + int end = Math.min(getFrameEnd(), raw.length); + return start >= 0 && start < end ? Arrays.copyOfRange(raw, start, end) : null; + } + + /** + * The bytes after the IP header and any IPv6 extension headers, bounded by the + * IP length: for example the whole ICMP message, or a UDP header and its data. + * + * @return null if this is not an IP packet whose header could be parsed + */ + public byte[] getIpPayload() { + if (transportOffset < 0) { + return null; + } + int end = Math.min(Math.min(getIpPacketEnd(), getFrameEnd()), raw.length); + return transportOffset < end ? Arrays.copyOfRange(raw, transportOffset, end) : null; + } + /** * @return index just past the captured bytes of this packet */ diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java index 70f28ddc7e5..373007cd6ec 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PacketDecoder.java @@ -58,7 +58,8 @@ public boolean readPcapng(final byte[] raw, final int linkType) { dstMacOffset = -1; int networkOffset = findNetworkLayer(linkType); if (networkOffset >= 0 && isArpPacket()) { - // Typed, but has no IP fields + // Typed, but has no IP fields; the link payload starts at the ARP message + etherOffset = networkOffset - PacketConstants.IP_OFFSET; return true; } if (networkOffset < 0 || networkOffset + 20 > raw.length) { diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketPayloads.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketPayloads.java new file mode 100644 index 00000000000..5280ace8318 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketPayloads.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.time.Instant; + +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.TestPackets; +import org.apache.drill.exec.store.pcapng.PacketDecoder; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestPacketPayloads extends BaseTest { + // An ARP request: Ethernet/IPv4, who has 10.0.0.2, tell 10.0.0.1 + private static final byte[] ARP = {0, 1, 8, 0, 6, 4, 0, 1, 2, 0, 0, 0, 0, 1, 10, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 10, 0, 0, 2}; + + private static byte[] concat(byte[]... parts) { + ByteBuffer out = ByteBuffer.allocate(java.util.Arrays.stream(parts).mapToInt(p -> p.length).sum()); + for (byte[] p : parts) { + out.put(p); + } + return out.array(); + } + + @Test + public void testIcmpMessageIsTheIpPayload() { + byte[] icmp = {8, 0, 0, 0, 0, 1, 0, 1, 'h', 'i'}; + PacketDecoder packet = TestPackets.decode(TestPackets.ipv4("10.0.0.1", "10.0.0.2", 1, icmp), Instant.EPOCH); + assertTrue(packet.isIcmpPacket()); + assertArrayEquals(icmp, packet.getIpPayload()); + } + + @Test + public void testArpOverEthernet() { + byte[] ethernet = {2, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 1, 8, 6}; + PacketDecoder packet = new PacketDecoder(); + assertTrue(packet.readPcapng(concat(ethernet, ARP), 1)); + assertTrue(packet.isArpPacket()); + assertArrayEquals(ARP, packet.getLinkPayload()); + } + + @Test + public void testArpOverLinuxCookedCapture() { + // 16-byte SLL header, protocol 0x0806 at offset 14 + byte[] sll = {0, 0, 0, 1, 0, 6, 2, 0, 0, 0, 0, 1, 0, 0, 8, 6}; + PacketDecoder packet = new PacketDecoder(); + assertTrue(packet.readPcapng(concat(sll, ARP), 113)); + assertArrayEquals(ARP, packet.getLinkPayload()); + } + + @Test + public void testNoIpPayloadWhenTheHeaderIsMalformed() { + byte[] raw = TestPackets.ipv4("10.0.0.1", "10.0.0.2", 1, new byte[8]); + raw[0] = 0x43; + PacketDecoder packet = new PacketDecoder(); + packet.readPcapng(raw, 101); + assertNull(packet.getIpPayload()); + } + + @Test + public void testArpInPcapRecord() { + byte[] frame = concat(new byte[] {2, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 1, 8, 6}, ARP); + byte[] record = new byte[16 + frame.length]; + ByteBuffer.wrap(record).order(ByteOrder.LITTLE_ENDIAN).putInt(0).putInt(0).putInt(frame.length).putInt(frame.length); + System.arraycopy(frame, 0, record, 16, frame.length); + Packet packet = new Packet(); + packet.decodePcap(record, 0, false, 65535); + assertArrayEquals(ARP, packet.getLinkPayload()); + } +} From 1061de1ab44381bfcac5626699581d4c76946230 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:56:34 -0400 Subject: [PATCH 22/50] DRILL-8556: Read compressed and damaged captures reliably - PCAP: refill the buffer by appending after the unread bytes until it is full. Topping up at the read position overwrote unread data whenever a read came back short, so gzip-compressed captures could not be read at all - PCAP: read the file header fully instead of with a single read - Each PCAP packet copies its record; packets kept for sessionization pointed into the reader's buffer and changed when it was reused, corrupting session data in captures larger than the buffer - A read that fails partway through a file (for example a corrupted gzip stream) now returns the packets read before it and an error row, in both readers, instead of ending silently (PCAP) or failing the query (PCAP-NG) --- .../exec/store/pcap/PcapBatchReader.java | 65 ++++++++++-------- .../drill/exec/store/pcap/decoder/Packet.java | 11 +-- .../store/pcap/decoder/PacketDecoder.java | 4 +- .../exec/store/pcapng/PcapngBatchReader.java | 9 ++- .../exec/store/pcap/TestPacketPayloads.java | 16 +++++ .../exec/store/pcap/TestPcapDecoding.java | 19 +++++ .../exec/store/pcapng/TestPcapngDecoding.java | 11 +++ .../test/resources/fixtures/pcap_fixtures.py | 16 +++++ .../src/test/resources/pcap/http.pcap.gz | Bin 0 -> 8229 bytes .../test/resources/pcap/http_corrupt.pcap.gz | Bin 0 -> 8229 bytes .../resources/pcapng/sniff_corrupt.pcapng.gz | Bin 0 -> 8326 bytes 11 files changed, 114 insertions(+), 37 deletions(-) create mode 100644 contrib/format-pcapng/src/test/resources/pcap/http.pcap.gz create mode 100644 contrib/format-pcapng/src/test/resources/pcap/http_corrupt.pcap.gz create mode 100644 contrib/format-pcapng/src/test/resources/pcapng/sniff_corrupt.pcapng.gz diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java index 4ca783ddb9f..e9e3f117c7c 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java @@ -92,6 +92,8 @@ public class PcapBatchReader implements ManagedReader { // Set when a damaged record makes the rest of the file unreadable private boolean finished; private int packetCount; + // Why the stream stopped being readable; reported after the packets already buffered + private String readError; // Snapshot length from the file header, capped at MAX_SNAPLEN private int maxLength; @@ -170,7 +172,8 @@ private void openFile() { int snaplen = decoder.getMaxLength(); maxLength = snaplen <= 0 || snaplen > MAX_SNAPLEN ? MAX_SNAPLEN : snaplen; buffer = new byte[BUFFER_SIZE + maxLength]; - validBytes = fsStream.read(buffer); + validBytes = 0; + fill(); } catch (IOException e) { decoder = null; fileError = "file: cannot read: " + ProtocolDecoders.describe(e); @@ -221,10 +224,15 @@ private boolean parseNextPacket(RowSetLoader rowWriter) { Packet packet = new Packet(); if (offset >= validBytes) { + if (readError != null) { + protocolColumns.writeErrorRow("file: read failed after packet " + packetCount + ": " + readError); + readError = null; + finished = true; + } return false; } - if (validBytes - offset < maxLength) { - getNextPacket(rowWriter); + if (validBytes - offset < maxLength && readError == null && !finished) { + refill(); } int old = offset; @@ -234,8 +242,10 @@ private boolean parseNextPacket(RowSetLoader rowWriter) { // The record's length cannot be trusted, so neither can anything after it finished = true; if (protocolColumns != null) { - protocolColumns.writeErrorRow("file: invalid packet record after packet " + packetCount + ": " - + ProtocolDecoders.describe(e)); + String error = "file: invalid packet record after packet " + packetCount + ": " + ProtocolDecoders.describe(e); + // Damage from a failed read shows up as a bad record; report both in the one row + protocolColumns.writeErrorRow(readError == null ? error : error + "; file: read failed: " + readError); + readError = null; } return false; } @@ -259,35 +269,36 @@ private boolean parseNextPacket(RowSetLoader rowWriter) { return true; } - private boolean getNextPacket(RowSetLoader rowWriter) { - Packet packet = new Packet(); - try { - if (validBytes == buffer.length) { - // shift data and read more. This is the common case. - System.arraycopy(buffer, offset, buffer, 0, validBytes - offset); - validBytes = validBytes - offset; - offset = 0; + /** + * Moves the unread bytes to the front of the buffer and fills the rest. + */ + private void refill() { + System.arraycopy(buffer, offset, buffer, 0, validBytes - offset); + validBytes -= offset; + offset = 0; + fill(); + } + /** + * Reads until the buffer is full or the file ends. Compressed streams return short + * reads, so one read is not enough. A read failure keeps the bytes read before it + * and is reported once those packets have been returned. + */ + private void fill() { + try { + while (validBytes < buffer.length) { int n = fsStream.read(buffer, validBytes, buffer.length - validBytes); - if (n > 0) { - validBytes += n; - } - logger.debug("read {} bytes, at {} offset", n, validBytes); - } else { - // near the end of the file, we will just top up the buffer without shifting - int n = fsStream.read(buffer, offset, buffer.length - offset); - if (n > 0) { - validBytes = validBytes + n; - logger.debug("Topped up buffer with {} bytes to yield {}", n, validBytes); + if (n < 0) { + break; } + validBytes += n; } - } catch (Exception e) { - // Exception denotes EOF or unreadable file - return false; + } catch (IOException e) { + readError = ProtocolDecoders.describe(e); } - return true; } + private void addDataToTable(Packet packet, int networkType, RowSetLoader rowWriter) { rowWriter.start(); List errors = new ArrayList<>(); diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java index 66c439f3abe..94fa725a8c8 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java @@ -87,11 +87,14 @@ public boolean readPcap(final InputStream in, final boolean byteOrder, final int @SuppressWarnings("WeakerAccess") public int decodePcap(final byte[] buffer, final int offset, final boolean byteOrder, final int maxLength) { - raw = buffer; - etherOffset = offset + PacketConstants.PCAP_HEADER_SIZE; - decodePcapHeader(raw, byteOrder, maxLength, offset); + decodePcapHeader(buffer, byteOrder, maxLength, offset); + // Copy the record: the reader reuses its buffer, and packets kept for + // sessionization must not change when it does + int end = offset + PacketConstants.PCAP_HEADER_SIZE + originalLength; + raw = Arrays.copyOfRange(buffer, offset, end); + etherOffset = PacketConstants.PCAP_HEADER_SIZE; decodeEtherPacket(); - return offset + PacketConstants.PCAP_HEADER_SIZE + originalLength; + return end; } public String getPacketType() { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketDecoder.java index 1794eea4c98..c78604982d9 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketDecoder.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketDecoder.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.store.pcap.decoder; import com.google.common.base.Preconditions; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,7 +75,8 @@ public class PacketDecoder { public PacketDecoder(final InputStream input) throws IOException { this.input = input; byte[] globalHeader = new byte[GLOBAL_HEADER_SIZE]; - int n = input.read(globalHeader); + // Read fully: a compressed stream may return fewer bytes per read + int n = IOUtils.read(input, globalHeader); if (n != globalHeader.length) { throw new IOException("Can't read PCAP file header"); } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java index f71a0980508..e5648dc59f2 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcapng/PcapngBatchReader.java @@ -180,11 +180,10 @@ public boolean next() { try { block = nextRow(); } catch (IOException e) { - throw UserException - .dataReadError(e) - .message("Failed to decode the pcapng file. " + e.getMessage()) - .addContext(errorContext) - .build(logger); + // The stream cannot be read further: report it, then finish like end of file + finished = true; + protocolColumns.writeErrorRow("file: read failed at byte " + position + ": " + ProtocolDecoders.describe(e)); + continue; } if (block == null) { // At end of file, more batches are needed only for open sessions that did not fit diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketPayloads.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketPayloads.java index 5280ace8318..1d3c634e75d 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketPayloads.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPacketPayloads.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.store.pcap; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -89,4 +90,19 @@ public void testArpInPcapRecord() { packet.decodePcap(record, 0, false, 65535); assertArrayEquals(ARP, packet.getLinkPayload()); } + + @Test + public void testPcapPacketOwnsItsBytes() { + // The reader reuses its buffer; packets kept for sessionization must not change with it + byte[] frame = concat(new byte[] {2, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 1, 8, 0}, + TestPackets.ipv4("10.0.0.1", "10.0.0.2", 17, new byte[] {0, 7, 0, 9, 0, 8, 0, 0})); + byte[] record = new byte[16 + frame.length]; + ByteBuffer.wrap(record).order(ByteOrder.LITTLE_ENDIAN).putInt(0).putInt(0).putInt(frame.length).putInt(frame.length); + System.arraycopy(frame, 0, record, 16, frame.length); + Packet packet = new Packet(); + packet.decodePcap(record, 0, false, 65535); + java.util.Arrays.fill(record, (byte) 0); + assertEquals("10.0.0.1", packet.getSourceIpAddressString()); + assertEquals(7, packet.getSrc_port()); + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java index 0c80648bf49..3a32ff38894 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/TestPcapDecoding.java @@ -128,4 +128,23 @@ public void testSessionModeReportsPacketErrors() throws Exception { "where decode_error is not null"; assertEquals("packet: Invalid IPv4 header length 12", client.queryBuilder().sql(sql).singletonString()); } + + @Test + public void testReadFailureMidFileIsReported() throws Exception { + // A corrupted gzip file fails while decompressing, partway through the capture + String file = "dfs.`pcap/http_corrupt.pcap.gz`"; + long packets = client.queryBuilder().sql("select count(*) from " + file + " where decode_error is null").singletonLong(); + String error = client.queryBuilder().sql("select decode_error from " + file + " where decode_error is not null").singletonString(); + assertTrue(packets > 0); + assertTrue(error, error.contains("file: read failed")); + } + + @Test + public void testCompressedCapture() throws Exception { + // A decompressing stream returns short reads; every packet must still be read + String sql = "select count(*) from dfs.`pcap/http.pcap.gz` where decode_error is null"; + assertEquals(43, client.queryBuilder().sql(sql).singletonLong()); + sql = "select count(*) from dfs.`pcap/http.pcap.gz` where decode_error is not null"; + assertEquals(0, client.queryBuilder().sql(sql).singletonLong()); + } } diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java index beb099a006d..3572824f4c6 100644 --- a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcapng/TestPcapngDecoding.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.store.pcapng; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.nio.file.Paths; @@ -157,4 +158,14 @@ public void testSessionModeReportsPacketErrors() throws Exception { .build(); new RowSetComparison(expected).verifyAndClearAll(results); } + + @Test + public void testReadFailureMidFileIsReported() throws Exception { + // A corrupted gzip file fails while decompressing, partway through the capture + String file = "dfs.`pcapng/sniff_corrupt.pcapng.gz`"; + long packets = client.queryBuilder().sql("select count(*) from " + file + " where decode_error is null").singletonLong(); + String error = client.queryBuilder().sql("select decode_error from " + file + " where decode_error is not null").singletonString(); + assertTrue(packets > 0); + assertTrue(error, error.startsWith("file: read failed at byte ")); + } } diff --git a/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py index 14d63982951..58404a4236c 100644 --- a/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py +++ b/contrib/format-pcapng/src/test/resources/fixtures/pcap_fixtures.py @@ -180,6 +180,21 @@ def oversized(): frames = [ETHERNET + ipv4(17, udp(4000, 7, b'ECHO:ok'), '10.0.0.1', '10.0.0.2', ihl=ihl) for ihl in (5, 3, 5)] pcap_file(os.path.join(PCAP, 'malformed.pcap'), frames) +def compressed(): + # Valid and corrupted gzip copies of real captures; mtime=0 keeps the output stable + import gzip + for name, folder in (('http.pcap', PCAP), ('sniff.pcapng', PCAPNG)): + data = gzip.compress(open(os.path.join(folder, name), 'rb').read(), mtime=0) + if name == 'http.pcap': + write(os.path.join(folder, name + '.gz'), data) + # Flip bytes in the middle of the compressed stream: decompression fails there + corrupt = bytearray(data) + middle = len(corrupt) // 2 + for i in range(middle, middle + 40): + corrupt[i] ^= 0xFF + stem = name.split('.')[0] + write(os.path.join(folder, stem + '_corrupt.' + name.split('.')[1] + '.gz'), bytes(corrupt)) + if __name__ == '__main__': echo() errors() @@ -187,3 +202,4 @@ def oversized(): classic() dns() oversized() + compressed() diff --git a/contrib/format-pcapng/src/test/resources/pcap/http.pcap.gz b/contrib/format-pcapng/src/test/resources/pcap/http.pcap.gz new file mode 100644 index 0000000000000000000000000000000000000000..d4c9efbda69b565c6a8f5626bc3a3e023ffd3e06 GIT binary patch literal 8229 zcmV+=Ally_iwFP!000026YYHqd=%B$_do(_BpSRG(HD-3ghY0Cc5}bO<{nANN)idL z`lGYElieY+GsDbmHj6i)P(*pH_}YqiL(p0jFGcHx7Zt?Uw%*Xz3tCHS3;kLFMWs^S zs=UuRGqd+hHUYn{-|uJIkhz`zdCob{^M9Um&NJiNySCmpmdd7#pX0|VY7G3HHfPi9 zXI{vr>fjjvesdg=>-hIns)?c;XS!xnOUJK0u=bk`hg@&&`$Mjka#uX|&Dj`58f&T8 zHG!hWW{-Pv%$V%47Xw-|n-xXVa5PK}jiP2#^!U%M7zra%?p*7{cIGBPJV!d~yf?4- zkf8bE(aHd$ES+oxXEqYq4kyWW4j`^^=S|-p@llk4a_(4){3fG}J>|?MjB-m}T9irC zw{PyHsFtQ~MEk_Bz>AE}9#DfkYVGcJ(@wiHH>Xup)G8DXhwZEyU}cu!?OrjMo71DP zvd!gZ1-RQGF5!5dp-b%!R4|_t086PxJ=I7*MsvGSsohzPSi#oQRgL6|pCl>nnu^Dcf8owMAFa4ZQ?>rWGpY6CDDs<( zGH%U|dohaDITXsw9EvjZ3c;YG&N;a`jnH9Ls5=m{B4;t`U?TWV3358BoaI%{BGl5+ zotx9e$^*F7E{X95pc~p5MYVN6*Y|O(4@!qvD-?s85T5M#xghIGl{F zq^z>cR?ZfemUtWvOLVB_!BDeG74k$>Rw4bbLKoWFS;4Ocs*tn7QSL|_CzT7bVo0rZ zmReDe8LV|Ck6Y~F7+!ZoVs-T<`ii`oiqyt51dmwFEKq9%O;!M@*5b4~ETDp#uGLz4 zx|?kkmRWT8@n4(Z)M z7@h~%(`Iu)Nt9I-lKBElFo++a4gI$j+Z=XWrm$KAqzNK4h&E)Y@rb^NUY(B{Kwh3v zppmL5SqxDb4+s(Fd};vt-Qk#SL0*>U6;M3V_YUn`VthX1+#||9Rwjo^C>UhqNUcSO z!+^4Jh);ZI04&)LV;baqKAyEitHP~=Cm0lv*}hr}ZKjti*x>EawlXVEc?paQdLTVhrz%X-5;MiGY(FZgP8lYHoN#Y+5fTm-L(;oARLLcWR zb`XM7V{|R1O*k#y`aPnmiotX>5i7PBZSsqr5IiX)TRE7xc7qr=K~J!fXw7o!HQY&< z6p6uTtfNVs&fsDY(E{2DDG}fO3RtvdKDTolQ)E8DrSV$MsavfQ87?E{yQ3gK4!7r z1I@0>Mgx!>8A79CR|sGYE#ZGsMpj@3(hlr2ndMnVVZmZL`}#O92QF!zekq-V8qSac zP7|>mLD&xF+ldd68;-HIjvft9Y78_(1xAI&vwm#g?f4IxE2^AVZ(~BIU1YPNdKNqn za`{k6S;b(P1KDhey~+4Iyy)%6-KltGPKswFdJ!|gXjj2616qz>6#YEAknt__@*E5l zsznwlBE!A;ilb{2nhTe$3tOX!&z047R8ILwY8maZ(LPm+9c|~577iwM_ zKT-xi#lmWbLcOj@88Ws-^zFSOw;4V$W$bzbn(Hd?rL6-=D z>oI`@9M1TP>{<;g=z zL#>PoFzuWW8l;<9pC~g{Ql?Af8JSa%7p&03Lb(uJ6a@sv3KJ@@p=g{NW<3bg8!iS= zoT{K8C(EL&*me60l+<-;qBVeMAbyD^KAvt?6GhtH&i!9p0V2y-&4GLppl?0`rstT!ZcY6LXF+s~>tFC(cT znT^&wEOtyRJY&-i={{9TY_+AXg#~jHBIeBmFo@CYi=^cCUCIDWnwP~b33E%*_0Aj- zo-i-%5IBcGnvgO;JQGodXgqGG5(25Ab{e=6_XY%Z{d7h_ct}qLsU0Nhf z+d-lIEEyL`78k+no>*od2Zn2x0@5tMEQX|oU{OV-r50Tcn{k;q{{X+u37Xg!6*XZ< zu()xurB_7E)JuzU&6s*4VQPCKQ!jpoF!iMK(MP2e6(>T9A}5J4gtEl|hR|aSGkRUt z;Qc7yIcHt{_GRJ{@2&AbJQCrkKoESBSIia}dWD&Fh z_U+MhW;J9+)&|yzP_}__BE)TAjGn$@gLtw?+(7)7qjH0(O(OGF5}6-Njm*Or;>eu& zyB3*alm$>~K{}K{PO6kD&wGrTHb#34eooPY&WR~OXBLQy z9~*-!Qu@y@b1a*PW3eSVmhgqf6kyXJMWqbO5Z0Qh{~5-LPL3DKsFKWmQ3BkkJd&xs zXW*AiY&|pY6dk>j;=hzQ zmXA{%%QZ`}W4UrIMPZaSOO_em-%t2HDWl<C)-Tgn>p9Uqm7Dc%`7DY zrXAW!G9y|;hfWU94yE2?g+yC$IXS$lXlN&kT4QkE9=m2*M`i>YLpu?UjiHTM?)!I#J2{n1Vc2mDMp6O?N zf|3P|pmy|PAb9F(DtphT)q$jQFK){f%d@Fdz<1!^l$^C=XLexUb)SOUflK7hq}*jM zPARbWcdc+wvQ(Vz({tAJm2$?X%+j_;YQ2mtd3ywJN#x6n;1h$jPDe#aNl9@@v7@M> zG@lQu_)1YpS-GRgk*{q&)JAt6^2LxUh16QS1d<<;`C6>6(~Dixi<;s8$?OQ15>n{$ zjqQk9Z7(7}wvSM&Z6f5yHYIAc9f|ztMntW)3y~kc=b>Lq$;~s}P53TR?_V@gFJ4!0 z&KEX|r;eKL;>{%?-Ct5ex@G%tNH^c6g>+wqbId}z2S`XaGclxFw$BLZre2Ffx)HNl z=J8q*;HW~jVSH}}QJbmX%`9+>a~vlpc1!OqhuD?RXphfqOFg#5SI-HI96|BXfI_zD zKq_CvriwP`a|t{#lJw9qY4sQo0VW7+ge0Q^JJdz?2L(}O@ur>m669!XuQ@k%?m%3mD;GOfjVO}YG^poueFOBZ5Akn+GQ;TM^7Y-Pa+~|i!LBq zyz3bRPX&Ba257j zi{=g{WBM35%n3MSQ?ZM9w8=R(MoV2IId)}8+%qifq=%UKjpM29i?#0{Z#5jWGCwtvzA;q4^M zS&2yc)saDkgtSr_mq(`*c+;y{X4(3#MngNv!06XTt{{C-(4+W>#9lu~Ysw5qvlfR~ zuyI51e1&9wkUV8+D5XwYbqhphqKLX#Z$JQ=@B{e~k@f4t>Cq5!`$=id@9CY7JKDwo z&qkK8a%?)NC4*s^8UHAIz12-5?v_JkC8$T>GL<*xBSDxjgb6Au;2can4MrE+6S3jE z%UGf}gpPJ+8Lf@>e61Lo59LrniQVC|;t;kFx1>f1NvsTif-o+1&Kg=ffWPE=M#H1G zt<`Po86l$vq|hL=gan9qAnc)jxF-3?4Z20`r(Llq8GHSnIYV0z0b`VY97_0DbDX%C z$OKFkAY${ee4lC;cyqwHr0Zewt0m8wfO-{sNM!^QX6S@@n!}2mPP;?<;~bSi4$nMo^&P}KU2VOiaQ-2l$NV0Ke$&%H>fsW? zDd{kaBLb&%R@^dJNZXl! znivq4Pp9XehIUsRR6<7qAkGDJDFL31pcVwPSzZ(%*st*_p>Q9%f>EzRKCsL%4!OMj zSm029ot8B?1nU|7+C>*vI`oaX0%M=9++I{T^4?C7fWD5%N6&M2klCOnzaf3J$40CZ z9l%pE)6aH^eX1T1YZpfAOL<((7SvIQrmzC-6~%s5Cb=>=HZBGlB3GjJS-@$@jNk(B zyb3AWLP23hZl{Cu%A#Vlz*wFsXcJVHx1!=A8^o=n7=&SRYb01StN`au=u z*Mupw#QfL{c_Fz43JkMPchlUa-tKv>(FGXbfN(_DjB`W@OOsyFu8ug?02gXRba+Ot zp~QGj!c6U4cg9ffOXg7pD=R3feEidGcyUr^>X~?Pvg}24%$>Jn-}p5YH3Lw-{KRyN z&P}|=zErPqYaaF*En_s4{DCa$HSid(rFxCLanG-O1IzOBe@iRNq$T^dyr1T+OLwum z3P~dPyhuvmNLCRUpTf!mILkdgC0Qz#?1Ml@R@C6Gwao2oLyb+BBX^ezej8ESx1B}XA z(Vz{Nu!;RJ3y-67kJN4Bc`%dln;)OrFGD$+dTlc>H`W?)XJr52X zeR}KFtEYT+9ev-rBUfMAS$XK_bwbH?TX*d%d+mXsKJu}2e)`L=Hh&%*c<7a1yj8s7!Ux~_>(RT4)?RX(^|w!~ zXpT&t%-#R?m%deNtG3zwfRyS>Z#_X=)uweRYiu>AKwTj%Rt z{>mIzVCTd!t1dh1{G0Dzw?(=*w5is4cd+fz52cdboZp? zU)HR7{`!^$m6uLB@Xp~`)$7CIFZSKOAl$d%jO;r$qPntQ{AmIENbe25ZCyBfUGK*W zI$pi_v8|KO{q1Sfsy1&q`;|X6-uWNbb^dv2SGTNwd4{#*!B56q-U{4*|Ux@Z3z z@1CE3e{WW8-mUJ+J6~r0_WqQbLr=YB`>69>{-7;*aLSfFfk*d9<$DS`XFbj2Wmo<2 zO zrT)+IxYb*4Y2CeH((DU&Y(Mw(UrkxxPTkdg^R$Z^7OnE^;}7x69(ZIM^FrRU8>hbf z^4$-vw)$l0DiU`w~psNvwK2#_h}m<-~Y#* zAC74gUX#A~_4)QsANt;*KmGiFX58io2-1wn|J8lT+q3DT;k(z)F0FZGTfy#It}dOu z5dH9v^t~Hzw{IW&+O_hH-#ngu+dn_e`nYz^7Z?2Vt4mMcnYGyK>0R;Wwb_^b=#{Ug zqWyJe&zrjU>&Z3Qn}7e5pAk-NV)|dh(OJ$?FgA#oWH(tZeFC;FfpO`I%p>+>N>Ysk0KfeZgdd z+xHeyRLgqrC-j5QOnA6@|06%SarH~@Ul{u2;pu<9^61Xfue*8gKbNmsHS^Z#vmbar z;@-V=$L-=TAD*6Dc+12GF5cmOZ>H_ek5^oJW8|_c4-ecaFIyda{)R=T9S!81d+hxy zKE40(YxmneV0RvwU3kuwd3)c3|G#?p-5Xjr&;M^`$KoZ=o&R~~?rrZJU0uKPoZOo( z@UL}!HF)0ls{hjQ*7maX^ouWyx%$Jb%1LKv9t^kA#1D31KX`v#;&vZ81})v@&a+f3 z7^|TKt{QJPd-@rfJryL*o^H)ER?B{HjH3GeitJ5TZ=&Zlxf(l~j75-WW<-W-H=Pjw zK=44K5&d}NF&Pe`;+B}m%fa3OCCqxX)uu#&O_rC2kE7h}ji{Stf<$0^GLncmejnUt zBSCRAi$I}e#!}Y|oS0G^^v(SVb#YL zt~A3lSTs~1vx`HV%qqwavo3F;*_61C;T1LjV?+WWJddXR@XwO?5ou)wnL{o3a{7?@ z5i8xi&W1VKsA}UJJsr8J$fCDDrg-|R3`Pgis=_4LiFSD? z!PC~<)zFCw@RkfK^Dq}>cpob>+6Dt&*PPGDER79)1dE(;;wxTJ+rI1%3H^B00k1U~ z$5_s)A;zl6?$tzzj#UgI6fRF@2t!mhx31 z`2KP-@7bF=@3~#Y^B&jT8sEQNbgCKOZzg;{Ba!cKSB=%Og)1p)NR{krX(MCWCjUQN zt1h#bj?RRQB`9O*O7U|<3ZOT3&;}wfycA%JFD$gQksT33#WFJk9vXy8#My1qwNWmk zM0bL;j6F;)3kNthOme(Dky_k)`Q$ST(Vdk6yscWNtwFg>JoNBP)D;qtwy+XWR5qyX zv*>A&G+A01g-wTqgp`EnsOa6VCtW(6w8M!)l8@2cBPs|9FuvHAv%buhgiNLHa=Yd= zoXqv0STaI_8$bmWR#aK|f6L7k*-PzCWHV|UrKRZOl(g2{m3Rewsufif4!hy?~HCz-Q(Am@X z2xjW<)210m4-ux$Ph{%5_83e(elkUoe$|Btfk)StaiUpLSW4($b)k_ssr7P)c1F69 zWTfzl7)wjBCw8Xs{SqzbrL7M|#1QHi#4r;3V3Z=rlbQyDlUIx#6|AgQBxy-43Qk=K zLPY?jit(R{z!z1qDyrNN#$Vy;zAE0(EOlmg3AGuyCz7E6eWIx1Od2wXS2(XiO?Y=F z`ii`6Tu57k1!+Bzob>9OYMH5aM>-qbA+Hl1&Q}u1$Xm8&5io9{w3!>@cSW(BcqIU?Fmz zc)HyX-<%=ocK9s}CP*^2w3id&Ucx}gg`iWRw`GpSoTR_MfJhEJ0%sCSxP$J6xlerG z*zL4JkU=%H1Eah~j}qfInA#*VpHCw5-={|A8*6Z6UiOX_nV;5Wnz!^Y;rr={e7~{A z;QLCRqS&|>$CEy;6yG22#%AL)h`2}aZd#P`a9;Ba1+|1YADh@aTnb}B?~pG!O@r>x zCXH#7nCy?+M?LuIKM`bPS-N22goOmF^kaV_14pYeAOsn?AGZyMpRz!Nf%5es1%MR= zbpXxdBP%@MwdZIlHl{!%)L0x%34{X~tUyZ$&(RaqOaY=1N}E6|)E)BhoL3VN>_nHu z$g(npn$Pe;NJS0sY(E4%^Ex~9(j=Y6ylg8Q=2=y>k&qv_K?|WG_8dHE8>R&u2n;II zA~RBe^BSE;f+M%PK`We4;!?H`dXAB0dybwyX<9*I)};=wZ&I2T#2TEknQ6~4*1k;3 z>JP;?#=HQ2L_yu63_+j|xD79E;BXW#50C-Izv&7qq!Ssi4H&x65(IO9O@Z{fV^cxX z`rQovu4q5HT;%aiIVbp`s2&)bk6+SE$EX&Eq{9@1CtNKsh{pE0HTE&0=m#oF<0a;Msz*l zWT-%qaMA!wyTFmKvWb7f@gXY{j?x{iVlcG{-}e%}|1g#BAKZZX-nkea0+b7T$C&Iu zuQ`Z!DK;PfZU_3i9q8|NpugLJ{%!~Q|Cb$TgWF9!!~(K6TVb|0i%~w~#!`Es*xoUz zGlDU1?%0B71oR&hX9N#!F!p9^Z-*HH_*Z?x7MqVD#v?-f7>)U1GMg0HCoi{zz-uTb z3odEzM|Vju8h!-`FZtN@RlsJLlX^v2*0T2$8o%?S9VsEd-w+hfT!}l!A1lPp3Kigp zp~h1Ov)vlnQx|On{uX3%I|~yRmRKyux3jLKHwu429|aS?pzi|L#pCbjV;>Zo2hrL` z(`$&ctjocp{pHwkW;$~vi7NJ_MirY|-FX+EgQJR4K)IWfjjvesdg=>-hIns)?c;XS!xnOUJK0u=bk`hg@&&`$Mjka#uX|&Dj`58f&T8 zHG!hWW{-Pv%$V%47Xw-|n-xXVa5PK}jiP2#^!U%M7zra%?p*7{cIGBPJV!d~yf?4- zkf8bE(aHd$ES+oxXEqYq4kyWW4j`^^=S|-p@llk4a_(4){3fG}J>|?MjB-m}T9irC zw{PyHsFtQ~MEk_Bz>AE}9#DfkYVGcJ(@wiHH>Xup)G8DXhwZEyU}cu!?OrjMo71DP zvd!gZ1-RQGF5!5dp-b%!R4|_t086PxJ=I7*MsvGSsohzPSi#oQRgL6|pCl>nnu^Dcf8owMAFa4ZQ?>rWGpY6CDDs<( zGH%U|dohaDITXsw9EvjZ3c;YG&N;a`jnH9Ls5=m{B4;t`U?TWV3358BoaI%{BGl5+ zotx9e$^*F7E{X95pc~p5MYVN6*Y|O(4@!qvD-?s85T5M#xghIGl{F zq^z>cR?ZfemUtWvOLVB_!BDeG74k$>Rw4bbLKoWFS;4Ocs*tn7QSL|_CzT7bVo0rZ zmReDe8LV|Ck6Y~F7+!ZoVs-T<`ii`oiqyt51dmwFEKq9%O;!M@*5b4~ETDp#uGLz4 zx|?kkmRWT8@n4(Z)M z7@h~%(`Iu)Nt9I-lKBElFo++a4gI$j+Z=XWrm$KAqzNK4h&E)Y@rb^NUY(B{Kwh3v zppmL5SqxDb4+s(Fd};vt-Qk#SL0*>U6;M3V_YUn`VthX1+#||9Rwjo^C>UhqNUcSO z!+^4Jh);ZI04&)LV;baqKAyEitHP~=Cm0lv*}hr}ZKjti*x>EawlXVEc?paQdLTVhrz%X-5;MiGY(FZgP8lYHoN#Y+5fTm-L(;oARLLcWR zb`XM7V{|R1O*k#y`aPnmiotX>5i7PBZSsqr5IiX)TRE7xc7qr=K~J!fXw7o!HQY&< z6p6uTtfNVs&fsDY(E{2DDG}fO3RtvdKDTolQ)E8DrSV$MsavfQ87?E{yQ3gK4!7r z1I@0>Mgx!>8A79CR|sGYE#ZGsMpj@3(hlr2ndMnVVZmZL`}#O92QF!zekq-V8qSac zP7|>mLD&xF+ldd68;-HIjvft9Y78_(1xAI&vwm#g?f4IxE2^AVZ(~BIU1YPNdKNqn za`{k6S;b(P1KDhey~+4Iyy)%6-KltGPKswFdJ!|gXjj2616qz>6#YEAknt__@*E5l zsznwlBE!A;ilb{2nhTe$3tOX!&z047R8ILwY8maZ(LPm+9c|~577iwM_ zKT-xi#lmWbLcOj@88Ws-^zFSOw;4V$W$bzbn(Hd?rL6-=D z>oI`@9M1TP>{<;g=z zL#>PoFzuWW8l;<9pC~g{Ql?Af8JSa%7p&03Lb(uJ6a@sv3KJ@@p=g{NW<3bg8!iS= zoT{K8C(EL&*me60l+<-;qBVeMAbyD^KAvt?6GhtH&i!9p0V2y-&4GLppl?0`rstT!ZcY6LXF+s~>tFC(cT znT^&wEOtyRJY&-i={{9TY_+AXg#~jHBIeBmFo@CYi=^cCUCIDWnwP~b33E%*_0Aj- zo-i-%5IBcGnvgO;JQGodXgqGG5(25Ab{e=6_XY%Z{d7h_ct}qLsU0Nhf z+d-lIEEyL`78k+no>*od2Zn2x0@5tMEQX|oU{OV-r50Tcn{k;q{{X+u37Xg!6*XZ< zu()xurB_7E)JuzU&6s*4VQPCKQ!jpoF!iMK(MP2e6(>T9A}5J4gtEl|hR|aSGkRUt z;Qc7yIcHt{_GRJ{@2&AbJQCrkKoESBSIia}dWD&Fh z_U+MhW;J9+)&|yzP_}__BE)TAjGn$@gLtw?+(7)7qjH0(O(OGF5}6-Njm*Or;>eu& zyB3*alm$>~K{}K{PO6kD&wGrTHb#34eooPY&WR~OXBLQy z9~*-!Qu@y@b1a*PW3eSVmhgqf6kyXJMWqbO5Z0Qh{~5-LPL3DKsFKWmQ3BkkJd&xs zXW*AiY&|pY6dk>j;=hzQ zmXA{%%QZ`}W4UrIMPZaSOO_em-%t2HDWl<C)-Tgn>p9Uqm7Dc%`7DY zrXAW!G9y|;hfWU94yE2?g+yC$IXS$lXlN&kT4QkE9=m2*M`i>YLpu?UjiHTM?)!I#J2{n1Vc2mDMp6O?N zf|3P|pmy|PAb9F(DtphT)q$jQFK){f%d@Fdz<1!^l$^C=XLexUb)SOUflK7hq}*jM zPARbWcdc+wvQ(Vz({tAJm2$?X%+j_;YQ2mtd3ywJN#x6n;1h$jPDe#aNl9@@v7@M> zG@lQu_)1YpS-GRgk*{q&)JAt6^2LxUh16QS1d<<;`C6>6(~Dixi<;s8$?OQ15>n{$ zjqQk9Z7(7}wvSM&Z6f5yHYIAc9f|ztMntW)3y~kc=b>Lq$;~s}P53TR?_V@gFJ4!0 z&KEX|r;eKL;>{%?-Ct5ex@G%tNH^c6g>+wqbId}z2S`XaGclxFw$BLZre2Ffx)HNl z=J8q*;HW~jVSH}}QJbmX%`9+>a~vlpc1!OqhuD?RXphfqOFg#5SI-HI96|BXfI_zD zKq_CvriwP`a|t{#lJw9qY4sQo0VW7+ge0Q^JJdz?2L(}O@ur>m669!XuQ@k%?m%3mD;GOfjVO}YG^poueFOBZ5Akn+GQ;TM^7Y-Pa+~|i!LBq zyz3bRPX&Ba257j zi{=g{WBM35%n3MSQ?ZM9w8=R(MoV2IId)}8+%qifq=%UKjpM29i?#0{Z#5jWGCwtvzA;q4^M zS&2yc)saDkgtSr_mq(`*c+;y{X4(3#MngNv!06XTt{{C-(4+W>#9lu~Ysw5qvlfR~ zuyI51e1&9wkUV8+D5XwYbqhphqKLX#Z$JQ=@B{e~k@f4t>Cq5!`$=id@9CY7JKDwo z&qkK8a%?)NC4*s^8UHAIz12-5?v_JkC8$T>GL<*xBSDxjgb6Au;2can4MrE+6S3jE z%UGf}gpPJ+8Lf@>e61Lo59LrniQVC|;t;kFx1>f1NvsTif-o+1&Kg=ffWPE=M#H1G zt<`Po86l$vq|hL=gan9qAnc)jxF-3?4Z20`r(Llq8GHSnIYV0z0b`VY97_0DbDX%C z$OKFkAY${ee4lC;cyqwHr0Zewt0m8wfO-{sNM!^QX6S@@n!}2mPP;?<;~bSi4$nMo^&P}KU2VOiaQ-2l$NV0Ke$&%H>fsW? zDd{kaBLb&%R@^dJNZXl! znivq4Pp9XehIUsRR6<7qAkGDJDFL31pcVwPSzZ(%*st*_p>Q9%f>EzRKCsL%4!OMj zSm029ot8B?1nU|7+C>*vI`oaX0%M=9++I{T^4?C7fWD5%N6&M2klCOnzaf3J$40CZ z9l%pE)6aH^eX1T1YZpfAOL<((7SvIQrmzC-6~%s5Cb=>=HZBGlB3GjJS-@$@jNk(B zyb3AWLP23hZl{Cu%A#Vlz*wFsXcJVHx1!=A8^o=n7=&SRYb01StN`au=u z*Mupw#QfL{c_Fz43JkMPchlUa-tKv>(FGXbfN(_DjB`W@OOsyFu8ug?02gXRba+Ot zp~QGj!c6U4cg9ffOXg7pD=R3feEidGcyUr^>X~?Pvg}24%$>Jn-}p5YH3Lw-{KRyN z&P}|=zErPqYaaF*En_s4{DCa$HSid(rFxCLanG-O1IzOBe@iRNq$T^dyr1T+OLwum z3P~dPyhuvmNLCRUpTf!mILkdgC0Qz#?1Ml@R@C6Gwao2oLyb+BBX^ezej8ESx1B}XA z(Vz{Nu!;RJ3y-67kJN4Bc`%dln;)OrFGD$+dTlc>H`W?)XJr52X zeR}KFtEYT+9ev-rBUfMAS$XK_bwbH?TX*d%d+mXsKJu}2e)`L=Hh&%*c<7a1yj8s7!Ux~_>(RT4)?RX(^|w!~ zXpT&t%-#R?m%deNtG3zwfRyS>Z#_X=)uweRYiu>AKwTj%Rt z{>mIzVCTd!t1dh1{G0Dzw?(=*w5is4cd+fz52cdboZp? zU)HR7{`!^$m6uLB@Xp~`)$7CIFZSKOAl$d%jO;r$qPntQ{AmIENbe25ZCyBfUGK*W zI$pi_v8|KO{q1Sfsy1&q`;|X6-uWNbb^dv2SGTNwd4{#*!B56q-U{4*|Ux@Z3z z@1CE3e{WW8-mUJ+J6~r0_WqQbLr=YB`>69>{-7;*aLSfFfk*d9<$DS`XFbj2Wmo<2 zO zrT)+IxYb*4Y2CeH((DU&Y(Mw(UrkxxPTkdg^R$Z^7OnE^;}7x69(ZIM^FrRU8>hbf z^4$-vw)$l0DiU`w~psNvwK2#_h}m<-~Y#* zAC74gUX#A~_4)QsANt;*KmGiFX58io2-1wn|J8lT+q3DT;k(z)F0FZGTfy#It}dOu z5dH9v^t~Hzw{IW&+O_hH-#ngu+dn_e`nYz^7Z?2Vt4mMcnYGyK>0R;Wwb_^b=#{Ug zqWyJe&zrjU>&Z3Qn}7e5pAk-NV)|dh(OJ$?FgA#oWH(tZeFC;FfpO`I%p>+>N>Ysk0KfeZgdd z+xHeyRLgqrC-j5QOnA6@|06%SarH~@Ul{u2;pu<9^61Xfue*8gKbNmsHS^Z#vmbar z;@-V=$L-=TAD*6Dc+12GF5cmOZ>H_ek5^oJW8|_c4-ecaFIyda{)R=T9S!81d+hxy zKE40(YxmneV0RvwU3kuwd3)c3|G#?p-5Xjr&;M^`$KoZ=o&R~~?rrZJU0uKPoZOo( z@UL}!HF)0ls{hjQ*7maX^ouWyx%$Jb%1LKv9t^kA#1D31KX`v#;&vZ81})v@&a+f3 z7^|TKt{QJPd-@rfJryL*o^H)ER?B{HjH3GeitJ5TZ=&Zlxf(l~j75-WW<-W-H=Pjw zK=44K5&d}NF&Pe`;+B}m%fa3OCCqxX)uu#&O_rC2kE7h}ji{Stf<$0^GLncmejnUt zBSCRAi$I}e#!}Y|oS0G^^v(SVb#YL zt~A3lSTs~1vx`HV%qqwavo3F;*_61C;T1LjV?+WWJddXR@XwO?5ou)wnL{o3a{7?@ z5i8xi&W1VKsA}UJJsr8J$fCDDrg-|R3`Pgis=_4LiFSD? z!PC~<)zFCw@RkfK^Dq}>cpob>+6Dt&*PPGDER79)1dE(;;wxTJ+rI1%3H^B00k1U~ z$5_s)A;zl6?$tzzj#UgI6fRF@2t!mhx31 z`2KP-@7bF=@3~#Y^B&jT8sEQNbgCKOZzg;{Ba!cKSB=%Og)1p)NR{krX(MCWCjUQN zt1h#bj?RRQB`9O*O7U|<3ZOT3&;}wfycA%JFD$gQksT33#WFJk9vXy8#My1qwNWmk zM0bL;j6F;)3kNthOme(Dky_k)`Q$ST(Vdk6yscWNtwFg>JoNBP)D;qtwy+XWR5qyX zv*>A&G+A01g-wTqgp`EnsOa6VCtW(6w8M!)l8@2cBPs|9FuvHAv%buhgiNLHa=Yd= zoXqv0STaI_8$bmWR#aK|f6L7k*-PzCWHV|UrKRZOl(g2{m3Rewsufif4!hy?~HCz-Q(Am@X z2xjW<)210m4-ux$Ph{%5_83e(elkUoe$|Btfk)StaiUpLSW4($b)k_ssr7P)c1F69 zWTfzl7)wjBCw8Xs{SqzbrL7M|#1QHi#4r;3V3Z=rlbQyDlUIx#6|AgQBxy-43Qk=K zLPY?jit(R{z!z1qDyrNN#$Vy;zAE0(EOlmg3AGuyCz7E6eWIx1Od2wXS2(XiO?Y=F z`ii`6Tu57k1!+Bzob>9OYMH5aM>-qbA+Hl1&Q}u1$Xm8&5io9{w3!>@cSW(BcqIU?Fmz zc)HyX-<%=ocK9s}CP*^2w3id&Ucx}gg`iWRw`GpSoTR_MfJhEJ0%sCSxP$J6xlerG z*zL4JkU=%H1Eah~j}qfInA#*VpHCw5-={|A8*6Z6UiOX_nV;5Wnz!^Y;rr={e7~{A z;QLCRqS&|>$CEy;6yG22#%AL)h`2}aZd#P`a9;Ba1+|1YADh@aTnb}B?~pG!O@r>x zCXH#7nCy?+M?LuIKM`bPS-N22goOmF^kaV_14pYeAOsn?AGZyMpRz!Nf%5es1%MR= zbpXxdBP%@MwdZIlHl{!%)L0x%34{X~tUyZ$&(RaqOaY=1N}E6|)E)BhoL3VN>_nHu z$g(npn$Pe;NJS0sY(E4%^Ex~9(j=Y6ylg8Q=2=y>k&qv_K?|WG_8dHE8>R&u2n;II zA~RBe^BSE;f+M%PK`We4;!?H`dXAB0dybwyX<9*I)};=wZ&I2T#2TEknQ6~4*1k;3 z>JP;?#=HQ2L_yu63_+j|xD79E;BXW#50C-Izv&7qq!Ssi4H&x65(IO9O@Z{fV^cxX z`rQovu4q5HT;%aiIVbp`s2&)bk6+SE$EX&Eq{9@1CtNKsh{pE0HTE&0=m#oF<0a;Msz*l zWT-%qaMA!wyTFmKvWb7f@gXY{j?x{iVlcG{-}e%}|1g#BAKZZX-nkea0+b7T$C&Iu zuQ`Z!DK;PfZU_3i9q8|NpugLJ{%!~Q|Cb$TgWF9!!~(K6TVb|0i%~w~#!`Es*xoUz zGlDU1?%0B71oR&hX9N#!F!p9^Z-*HH_*Z?x7MqVD#v?-f7>)U1GMg0HCoi{zz-uTb z3odEzM|Vju8h!-`FZtN@RlsJLlX^v2*0T2$8o%?S9VsEd-w+hfT!}l!A1lPp3Kigp zp~h1Ov)vlnQx|On{uX3%I|~yRmRKyux3jLKHwu429|aS?pzi|L#pCbjV;>Zo2hrL` z(`$&ctjocp{pHwkW;$~vi7NJ_MirY|-FX+EgQJR4K)IWE(wp?UjEBs<4T6p`ehATJh`;er?=6vR&;B>M-Ehmq+tDxFDXx{||{k|+_G z&h=z*$r$M6Q^CJDdL47u?XUUc$j=RW^U9(#^SwbSFs(wK>KI`F-m$n@pGq#a>`DcguTq!@g z$zyrm<;EtVvIqOy_qp^afR@yy%dMvuE^BDhr!l%jN28iLHKl4Am;dI0&`p%KkUmH! znq9FS(g)~#p3sUETKHZ~qt&egy&#=9jaC1;Io}_B>-yy>qQFrF10U?DD({e+vF1eK zQcI;#yK>paty#$7ze*0Y?$U#^Y5tV5kk-b7k9V>kVsV>5qliG`W9vsk(PNiF! z`}hY9b|W*Gd@6&@)_iGhbUuw{W$xE|pc~mCHdZ8a6G>u)(m1i)0h}G?Mg|YU$CUCI za77SBiB?7_z#Ve2IEEsXN4Uj`<#7_B7{BNSZVnFe3Z(S*_8JHt^2RwP_l=ZGqs84g zOe&4ag!t(+9@$qCqa1H#Ze_0K7xEoR;JZ920qY0a@XKVvUad^a4X?CJyfqpf+#RuT ztNN6nzS}Z*(u~-aq~a<~ziU25vyq!(e2}8*C)M!v`#05nwt@a#K0)2Tz2jm3QmsM% zZi4-L(iHY@GW+}NUl!iKJO-6c=NszZ5CY%Hy=P-RK)Z)_{+M-UU&!-WrX~n#ikOfP zRDEo2WND{PQ>57@K64BNhu4tBR^iP&2bg1|4XbMJvJ zgrQ|X=giL^BOSNZ>fGew7Js&IGK$mFIouAAj#r$T5|DiYD82$9#sOjlg-&D8 z!X$Bm7?}`E#p*_*Fo{eS`1a>G|U4~Y_p$P`0Z;oUTF zJAt9sVd?B2x6Ox9y3#nSAby=>rOWvr;sWHQSS71a+$kKO=WC1 z{cfZ%*oq16H4ycaN}~k|iLe_cl`l5e_D-%Vz z3uLk=i4f2!jp-&8DgfA6g>!hMC77WkB;59vpl zm{<$}pDE;F6uyuJCMFJWFW5A$h{B}vSu`Ps$rtf>-)v@T_oG!FT17ZyKoJ~kb0#&g z>L>r2)~${b^AvVF~*>?Fp-E(VX=W#WzyI@3KyfZD0Ga&q0t0fKKOjI z?=e|pdrS#&Sqp3q_7I$hUv(}wAv3 z;276mm#|Js!mfh$Rn59YbzU1^mpJ2S1lJ`3%wU%Lc@=_^5T~KYLAuSay$25T5AHY+ z2cv$#vwt9k$6#yYF#Y4Yd}vfYgT>T8i)y|`+GX5t{8X#MBz=N^d6 zB&{igsa{`wu@S-qu+tK3jEp|@P_JEDr(GV_exvzd-P$E_ZNl^MTz~Vh zj`dICIOutBaV_Iuq8N^Yg#J)hJs&%qez6J7Ug;ZNy!S@b!2laFVzik!4v0G1oW4Y> zqcz5X^U@t2h?@!32*T8igG~@7fL#(`gF1AWQQO*FKc5u6wK;^;%w_&s-P)Y`7+5VE z^-}-`YSRZX>VOb!NHuIrR0n%dU<&pZX#Z8c9&D^OZ;7`?!&?s8lB)AhiD&jzeF9hX z@=wMj6xvAJ*FTnxgngjj&wpG?{q{A`S#?`q&3RA{*fZ+}`_SWhz@G7C*hD))nY6s7 zoydB;Uu`Fr+d(^V9n6>3rqE8T-45-<0qT#rj!z5aYv}&Oc07t7PECg|B zv>y7dNK=G_G(leM!ue5irRE*>K`TNZ*qW}y{srxSt8Ig(4}fhTh2|kDU2fWRht>WZ zl``5;ZW>WZF8#Ru(h99PCJrn}tHo#NR|dyKm)$>y&mbf4o!e>^2F?4xYJEUIn=4xB z_6~hvu0Ru!iXso<2@M1_9uq{j%M0Rr-EkFm2lPMuYUb2x{d3_bquWh9^!2a$xOw8; zUp?lWeSiGji{i#@WA4rCnPQdD$`qMvsrEO!txG!JepBBocE%_|-k#reWQmK?E&c*^ zYtYi0bF2Q|9A}wY`nt(!*(+r^oeFM3M?bK47~Xr>!OQYdN81<;OWcU%o^QV9s;zk5 z25(F;p|I>iv2oYnNs=WVoqf8bmrvZ-bLFjmzVGr^k$*{=4&?A>K&jNS}y>&&QA2L|0_=|M29r#jd1xBi`A0 zTv_(G{Ex;hUWf+|?Q_%nUjCW8W?lalcJsL>7QMaos&kI}^`o0i4}`xvZPACqu2iTOM5$u#7j>#F^fo= zc<`Eq*9pn>k1g-K-Ik^36I2nh{=o%w$JoX*T*~9$EV-2ZvWlck!th0LT={ppaQqe2_K`TT9Gs7SHwB4>l87xWQxN_-`$F#+&vx#gqGf3X{$p| zj`X}3(WE5&U7M33?-r&#-q2~T(>m{ohetVdA5eMXexN$3j3DC)`WD`46*dI4;a4qr zwH5&R8|5bW9k&A=644UijT{+t=tmbe`j*xnadZ??!SME z^XBGU6-k-E{d6wb=Uje>>}^vEAFml_AEcgWI=I)ybA|VM1#NO84GRo&wfUD(`Pgfg z@AkFpj{Tx~oH%t`%H~r+=_^}Xw>^H9x@6h2r&Fl7QdGv;S(yjKt zZq7*^^em`}Ya_>gPlsGUEu*7Whwf@^pWb3UyLE4A&Y-KGA}*Y8xCUE>UTR_oVR0Tm-Jiyz0B$0RIKMwo`GZnbRo`}_*t@`vOx5f`gij1>n}|z)}uA`V&h|-Ug+5cGk!8rFKV|7vZ#4IQ@IH?jo+udtF{Ys z9ifbLdO@`NEq1Q1EoQol$4tY1j+p5cf$yt5)?&q=U99C>Haj03!N>VN?kt0R`!oVE z#}UZ))n5SLyFa*oCw$`!OO(-HcK>i*^<{1UaGl;9Z#p_gI=!QMj(->gd7ck>E`~fW z^M!N#3a1})jwi+v%E=ku9!n(DF$S7m57^V{7NgnUxPKmMN#VB&;gC_gn;Q<^LPfyQa z@IfYF_$MC!B;lVl&1bX$_%Xk4|0?`|Z-hUK;J?^=@4&*rG5qT1xX{7sdXh`VFb~BE zZh5`pJ(_(xIJ3;$W#4V+tE@|^_Eln=iRPnOcE-kNTOZPut|ym_7!uKEw0!RFkA2=p zcjVkz^G@xb{n~HmI}%7%>Hcv^E0_Vq*ve(8MJMZ<{vdDrByK<^e$K<4=G7 z=pXY#NliYQrfYdF zCnfL7n64ZyH7^;HpHbpMMl;tW`(2f_)R%Tz@@QilZG zXjiqQ!oQ8*`8CvLF`bn|Hwv-~)qSrogNJpHLDyY)O^*)O23@A#44aH}^Q1TCE=o$w+37vm&g;+>ZX%_S{;#HR4eg z{Wn_ud&;5`k=k_%d1^lE_r$x5HC02?J(i6cr#io`hnts6_tYnMefx$l@9a0?;o-t& zsVNz}OXy+mJCkQ*cD%kP4kM-HY@BBAy?#uz@y4MKI@w6#UeQ)>FfA$Ck|*$e6!Pir zm~DgX$3-a*+a*?R9vE_r=5``?3-Y{V|(=t+G1_O+pf}fb^$T3FS+*Xd;KwwHEQ{v7D=eEXq?^H);bvw{lp0W z9a=w8JVX@Up_iLD>-JIw zsttE8tVFc6z5sFOwJpB*PdUUr&7`ilJ^YkhFULGH!?u~Gk{Z)uB9OrJfE!sk! zA%0{M;GD=0uj{C>xcE_ogFI)W5w3^S`9jEO$)vh8N|4UBUVi@;zYkJ$s$oK_t3|tY!*th30q#eNhJ}Bf&P&Ji6YI!MU)!lyM^^O5-xKUNzUvo63q>@+VhhV$WAx|RmUl4+|pb*oD{Y_h9nUD8(bRKSDj zy?w;GBxR^i@Kst`Tf=opvyuPPJ#n>fcB|u#xrRM)4SV7m_QW;RB~{zTe>?B0S(l`> zjl{a7+H9k7rrjyxKDnym)&QP(8}`~~;pySkbiS);nTq{Gu1Vz7o7 ztRV(#h{1lIx}*(jch#&*(%3=coPY{w2Ps}eJ7}CapkG70)evtr#9QBUyj5M7w2iP{ zsNA&+n*)yLpf&3U&_D3y-skwR`Z_1_i(-TcJTJ9gjVR5t!e7@LeYFG6|Nb= z^+FHCccPumuE&GAjy?IoZtTKXyNgFprM%jZKB+MAx=8`$e3|#EFzi=mUfSBziOIH$ z`KDK!ZhSeqs@J)7s=W@oB_-or=JgMX3>)%Wu=BXpk4E|wW&4fW`smczedMJx@=i$( zR>q}BQ_n8)i|86~^3ePAqc`u3ytfo(k9rY2&`xSJHgs%Zp3$MqDVrC4xO=uqxXUwz zhiB-b03#RMX{nR;-|{zKN9!i>A3{=P+EwuP5537-G1WT!alcOMb5~{C^suCtvyL$B zC(jBAUMN3%q^J4nX72~~&fYqI@2gm(t7E8ZztyfiyU$!X=v>&$hn)`j`d;#dmGp?uK*g&&Rk<-u`nuzGwnu2-gKOnQ z$`KzQx0@4wgjOY@bnn6*opJJ`MLasOjPu4Amyf=j#@8yRCTl#9=O)pA!8KitoGRg( zuEN4d2<0?YQ}5JKKEXpPry-||d(N_toDrsCuVwx-dScbJI|pvP?e14<@320v(VKF5 z*E1Q)Pn(zeM!SEI7*kVNJGTWmMdaS1erUhv#=si|!v++DmeH;a@9uQUsI)3M@8SHT zk@?x^#q_@}Syz>YUR>QMAZ^^dE!Hhp&$O3RbhsRU{z2u`A|98@+xdR8XR&=Q^jK*bGwG35PQ<#jA|R)) zy;JkDcToBt%!YiCTumMv28T3Q*JW6G&l7k*-(@7ASmPy=3_khpxu4 z*ueX+8=-qKSh6@vThpc8w_yz+4Yd8*z6}kH?@Rt08Z!*Y<9~z&JjQRd1K52aF8EdJoK3Dz;>WeHjxcwS(N3Mti5!aZQ*F$b8PYkUorh;? zyc$dR$AFJwiJ&BSR@D?QBy`YJ|D__FdS!MU2oM~H-sX#uIq|M8bgVy#334N4%@V7cP4NOSx4g-6JKbhxh_YC86aZ9cL2XApg(69J^E!_Z* zjlj7@$V2#g5fmmX^yh2co?+scP&Ui0CB}HP1?tA!eLqKxQAFT-&T1952ekKS{kDLI z{n<|OINuuswtxu3wy*-&{=@K^XNYGAvT&thl0zg2BE!H;=m02ZFnDYpo6F=dxhx)! z2ZBE~gTbJ4`5?^aF}XYz4+MvFSF!_)$Uy`Oo*Bt-CDTH|DbNeDD8YmnL9`g0c5ncv zWKuZ}mdmEmz#+U8D-#ICvG6%M3&e>mKAT1lol<0gpNdVj@^b+4bMO}A2i8@n`I#E~ zeepw#(VVucYKhU7Zh|oy)ln}-dtb-+2l@6GZPFgqe-fj82>fqo%rNlYpg3xN!+V3O zYZMsy-yWAus#{zp=F|iB#Ja(L=T;Bc6TS>vMS%VG*l}zfY-`7q`qzT+F}38J0FJ4D zVBOVm?U1-a8$6_;C|IK{GWztv-4wu6Toq`yef5RTu#!s*c z^0UtZ@}r%97mRuHeepxIL*1U7{7-AxiLeJh)4?vp&t`2(jBB@Kv3fSBYZ0Ipb}|8W zl*w5Ps4)_y^BV%vR@%(NI-IJv6wmgJi%NXw=t zfR-#RE#nhNgCwtiD_2sB%N>tFKkt2ESelFFiTk+5=oe zHP=WGx0+^s8AY}}tN(sO%f4~sc*6S)@nbL#wXS}@A%yiM8LcM3PFQ>no2-d})cr!6 zow=8}4DXki>o;J(Y#RgO@b21v8PUF-reAOm!3h)dXm09%A`aJ0#N*T4!bME38(l~f zb`;0Rn6Wgi03o+cY;-+g6>)8G4%gmtwSicfar|5@v2vFlc&yC-@ndD8Jg%HMhd~!& zf>s`ZgOTnzD{-F3o>&Owk#iOB+)gWxq2~ESd2}Gi1K2Q@8-}@IVrDpy37^+dtcd)w zFaic6Ri6Md4NZ`?O2YfbS~D+Th2R)|)wHSe)Q~o#yv$Na+odd^d#mqgbJ7d{ZpcYD zcE33jXKsF(m%x?eV+<3b_03EozJ#&sh`RTwC*?+2J(&FJc_Uo?Qc>;oBlmvQPmizbDW4fOjNuObZoRIL~x?49Db=!xSV8dC{z+hzlP zpK5XUEqg=cw|W0asa*e^p?x4!b!dC?GJ)+m>?iu(-=)W7kgI=O zU1NVf7|_dt7nf@4<>@yw)q1&Z7u3sp!GQNvonGD@^xx~{+^JXeb^QK?n?xP|Jg*@zS5eu+!`j}3jMyZg zUwj8Lxb}rtEyGK-CWi?+ZbQfqZ!W!wJq7KjhI815>zKnnO!oir`UCbuBTZhQxGe3K$sCYQ!ZL}C#+d;(ddjFt%nG9$zkzc+nw9A>O(=YZ!(pk19VZUoxI Q55)2R06%l~Pq=~r0C@P9EdT%j literal 0 HcmV?d00001 From 4ac2c40c137f6f9ffd78225c909c37deeb6b294e Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:56:54 -0400 Subject: [PATCH 23/50] DRILL-8556: Add ICMP and ICMPv6 protocol decoder --- .../store/pcap/protocol/icmp/IcmpDecoder.java | 352 ++++++++++++++++++ .../store/pcap/protocol/icmp/IcmpMessage.java | 39 ++ ....store.pcap.protocol.PacketProtocolDecoder | 1 + .../pcap/protocol/icmp/TestIcmpDecoder.java | 236 ++++++++++++ 4 files changed, 628 insertions(+) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/icmp/IcmpDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/icmp/IcmpMessage.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/icmp/TestIcmpDecoder.java diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/icmp/IcmpDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/icmp/IcmpDecoder.java new file mode 100644 index 00000000000..2cd404682d7 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/icmp/IcmpDecoder.java @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.icmp; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** + * ICMP (IPv4 protocol 1) and ICMPv6 (IPv6 next header 58). The message is the IP payload, after any + * IPv6 extension headers. Checksums are not verified. + */ +public class IcmpDecoder implements PacketProtocolDecoder { + private static final int HEADER = 4; + /** Most extension headers walked in an embedded IPv6 packet. */ + private static final int MAX_EXTENSION_HEADERS = 8; + + /** Name and length of the fixed part, by type. */ + private static final class Type { + final String name; + final int fixedLength; + + Type(String name, int fixedLength) { + this.name = name; + this.fixedLength = fixedLength; + } + } + + private static final Map V4_TYPES = new HashMap<>(); + private static final Map V6_TYPES = new HashMap<>(); + private static final Map V4_CODES = new HashMap<>(); + private static final Map V6_CODES = new HashMap<>(); + + static { + v4(0, "echo_reply", 8); + v4(3, "destination_unreachable", 8); + v4(4, "source_quench", 8); + v4(5, "redirect", 8); + v4(8, "echo_request", 8); + v4(9, "router_advertisement", 8); + v4(10, "router_solicitation", 8); + v4(11, "time_exceeded", 8); + v4(12, "parameter_problem", 8); + v4(13, "timestamp_request", 20); + v4(14, "timestamp_reply", 20); + v4(15, "information_request", 8); + v4(16, "information_reply", 8); + v4(17, "address_mask_request", 12); + v4(18, "address_mask_reply", 12); + v4(42, "extended_echo_request", 8); + v4(43, "extended_echo_reply", 8); + V4_CODES.put(3, new String[] {"net_unreachable", "host_unreachable", "protocol_unreachable", + "port_unreachable", "fragmentation_needed", "source_route_failed", "destination_network_unknown", + "destination_host_unknown", "source_host_isolated", "network_administratively_prohibited", + "host_administratively_prohibited", "network_unreachable_for_tos", "host_unreachable_for_tos", + "communication_administratively_prohibited", "host_precedence_violation", "precedence_cutoff_in_effect"}); + V4_CODES.put(5, new String[] {"redirect_for_network", "redirect_for_host", "redirect_for_tos_and_network", + "redirect_for_tos_and_host"}); + V4_CODES.put(11, new String[] {"ttl_exceeded_in_transit", "fragment_reassembly_time_exceeded"}); + V4_CODES.put(12, new String[] {"pointer_indicates_error", "missing_required_option", "bad_length"}); + + v6(1, "destination_unreachable", 8); + v6(2, "packet_too_big", 8); + v6(3, "time_exceeded", 8); + v6(4, "parameter_problem", 8); + v6(128, "echo_request", 8); + v6(129, "echo_reply", 8); + v6(130, "multicast_listener_query", 24); + v6(131, "multicast_listener_report", 24); + v6(132, "multicast_listener_done", 24); + v6(133, "router_solicitation", 8); + v6(134, "router_advertisement", 16); + v6(135, "neighbor_solicitation", 24); + v6(136, "neighbor_advertisement", 24); + v6(137, "redirect", 40); + v6(143, "multicast_listener_report_v2", 8); + V6_CODES.put(1, new String[] {"no_route_to_destination", "administratively_prohibited", + "beyond_scope_of_source_address", "address_unreachable", "port_unreachable", + "source_address_failed_policy", "reject_route_to_destination", "error_in_source_routing_header"}); + V6_CODES.put(3, new String[] {"hop_limit_exceeded_in_transit", "fragment_reassembly_time_exceeded"}); + V6_CODES.put(4, new String[] {"erroneous_header_field", "unrecognized_next_header", "unrecognized_ipv6_option"}); + } + + private static void v4(int type, String name, int fixedLength) { + V4_TYPES.put(type, new Type(name, fixedLength)); + } + + private static void v6(int type, String name, int fixedLength) { + V6_TYPES.put(type, new Type(name, fixedLength)); + } + + @Override + public String protocol() { + return "icmp"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("version", MinorType.INT) + .addNullable("type", MinorType.INT) + .addNullable("code", MinorType.INT) + .addNullable("type_name", MinorType.VARCHAR) + .addNullable("code_name", MinorType.VARCHAR) + .addNullable("identifier", MinorType.INT) + .addNullable("sequence", MinorType.INT) + .addNullable("mtu", MinorType.INT) + .addNullable("gateway", MinorType.VARCHAR) + .addNullable("target_address", MinorType.VARCHAR) + .addNullable("destination_address", MinorType.VARCHAR) + .addNullable("original_src_ip", MinorType.VARCHAR) + .addNullable("original_dst_ip", MinorType.VARCHAR) + .addNullable("original_protocol", MinorType.INT) + .addNullable("original_src_port", MinorType.INT) + .addNullable("original_dst_port", MinorType.INT); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isIcmpPacket(); + } + + @Override + public IcmpMessage parse(Packet packet, byte[] payload, DecoderContext context) { + // payload is the TCP or UDP payload, null here; the ICMP message is the IP payload + return parse(packet.getIpPayload(), packet.isIpV6Packet() ? 6 : 4); + } + + /** + * @param b the ICMP message + * @param version 4 for ICMP, 6 for ICMPv6 + * @return null if there are fewer than 4 bytes + * @throws IllegalArgumentException if the fixed part of a known type is cut short + */ + public static IcmpMessage parse(byte[] b, int version) { + if (b == null || b.length < HEADER) { + return null; + } + boolean v6 = version == 6; + IcmpMessage m = new IcmpMessage(); + m.version = version; + m.type = b[0] & 0xFF; + m.code = b[1] & 0xFF; + Type type = (v6 ? V6_TYPES : V4_TYPES).get(m.type); + if (type == null) { + return m; + } + m.typeName = type.name; + if (b.length < type.fixedLength) { + throw new IllegalArgumentException( + String.format("truncated %s: %d bytes, needs %d", type.name, b.length, type.fixedLength)); + } + String[] codes = (v6 ? V6_CODES : V4_CODES).get(m.type); + if (codes != null && m.code < codes.length) { + m.codeName = codes[m.code]; + } + if (v6) { + parseV6(b, m); + } else { + parseV4(b, m); + } + return m; + } + + private static void parseV4(byte[] b, IcmpMessage m) { + switch (m.type) { + case 0: case 8: case 13: case 14: case 15: case 16: case 17: case 18: + m.identifier = u16(b, 4); + m.sequence = u16(b, 6); + break; + case 3: + if (m.code == 4) { + m.mtu = u16(b, 6); + } + embeddedV4(b, m); + break; + case 5: + m.gateway = address(b, 4, 4); + embeddedV4(b, m); + break; + case 4: case 11: case 12: + embeddedV4(b, m); + break; + default: + break; + } + } + + private static void parseV6(byte[] b, IcmpMessage m) { + switch (m.type) { + case 128: case 129: + m.identifier = u16(b, 4); + m.sequence = u16(b, 6); + break; + case 2: + m.mtu = (int) Math.min(Integer.MAX_VALUE, ((long) u16(b, 4) << 16) | u16(b, 6)); + embeddedV6(b, m); + break; + case 1: case 3: case 4: + embeddedV6(b, m); + break; + case 135: case 136: + m.targetAddress = address(b, 8, 16); + break; + case 137: + m.targetAddress = address(b, 8, 16); + m.destinationAddress = address(b, 24, 16); + break; + default: + break; + } + } + + /** The IPv4 header and transport ports of the packet that caused an error, as far as they were captured. */ + private static void embeddedV4(byte[] b, IcmpMessage m) { + int ip = 8; + if (b.length < ip + 20 || (b[ip] & 0xF0) != 0x40) { + return; + } + int headerLength = (b[ip] & 0x0F) * 4; + if (headerLength < 20 || ip + headerLength > b.length) { + return; + } + m.originalSrcIp = address(b, ip + 12, 4); + m.originalDstIp = address(b, ip + 16, 4); + m.originalProtocol = b[ip + 9] & 0xFF; + boolean firstFragment = (u16(b, ip + 6) & 0x1FFF) == 0; + if (firstFragment) { + ports(b, ip + headerLength, m); + } + } + + /** The IPv6 header and transport ports of the packet that caused an error, as far as they were captured. */ + private static void embeddedV6(byte[] b, IcmpMessage m) { + int ip = 8; + if (b.length < ip + 40 || (b[ip] & 0xF0) != 0x60) { + return; + } + m.originalSrcIp = address(b, ip + 8, 16); + m.originalDstIp = address(b, ip + 24, 16); + int next = b[ip + 6] & 0xFF; + int pos = ip + 40; + boolean firstFragment = true; + for (int i = 0; i < MAX_EXTENSION_HEADERS; i++) { + int length; + switch (next) { + case 0: case 43: case 60: case 135: case 139: case 140: + length = pos + 2 <= b.length ? ((b[pos + 1] & 0xFF) + 1) * 8 : -1; + break; + case 44: + length = pos + 4 <= b.length ? 8 : -1; + if (length > 0 && (u16(b, pos + 2) >>> 3) != 0) { + firstFragment = false; + } + break; + case 51: + length = pos + 2 <= b.length ? ((b[pos + 1] & 0xFF) + 2) * 4 : -1; + break; + default: + // An upper-layer protocol + m.originalProtocol = next; + if (firstFragment) { + ports(b, pos, m); + } + return; + } + if (length < 0) { + // The extension header was not captured + return; + } + next = b[pos] & 0xFF; + pos += length; + } + } + + private static void ports(byte[] b, int transport, IcmpMessage m) { + int protocol = m.originalProtocol; + // TCP, UDP, SCTP, UDP-Lite: ports are the first four bytes + if ((protocol == 6 || protocol == 17 || protocol == 132 || protocol == 136) && transport + 4 <= b.length) { + m.originalSrcPort = u16(b, transport); + m.originalDstPort = u16(b, transport + 2); + } + } + + private static int u16(byte[] b, int offset) { + return ((b[offset] & 0xFF) << 8) | (b[offset + 1] & 0xFF); + } + + private static String address(byte[] b, int offset, int length) { + try { + return InetAddress.getByAddress(Arrays.copyOfRange(b, offset, offset + length)).getHostAddress(); + } catch (UnknownHostException e) { + // Not reachable: the length is always 4 or 16 + throw new IllegalStateException(e); + } + } + + @Override + public void write(IcmpMessage m, TupleWriter fields) { + fields.scalar("version").setInt(m.version); + fields.scalar("type").setInt(m.type); + fields.scalar("code").setInt(m.code); + setString(fields, "type_name", m.typeName); + setString(fields, "code_name", m.codeName); + setInt(fields, "identifier", m.identifier); + setInt(fields, "sequence", m.sequence); + setInt(fields, "mtu", m.mtu); + setString(fields, "gateway", m.gateway); + setString(fields, "target_address", m.targetAddress); + setString(fields, "destination_address", m.destinationAddress); + setString(fields, "original_src_ip", m.originalSrcIp); + setString(fields, "original_dst_ip", m.originalDstIp); + setInt(fields, "original_protocol", m.originalProtocol); + setInt(fields, "original_src_port", m.originalSrcPort); + setInt(fields, "original_dst_port", m.originalDstPort); + } + + private static void setString(TupleWriter fields, String name, String value) { + if (value != null) { + fields.scalar(name).setString(value); + } + } + + private static void setInt(TupleWriter fields, String name, Integer value) { + if (value != null) { + fields.scalar(name).setInt(value); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/icmp/IcmpMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/icmp/IcmpMessage.java new file mode 100644 index 00000000000..7cb2352187e --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/icmp/IcmpMessage.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.icmp; + +/** The fields of one ICMP or ICMPv6 message. Fields that do not apply are null. */ +public class IcmpMessage { + /** 4 for ICMP, 6 for ICMPv6. */ + public int version; + public int type; + public int code; + public String typeName; + public String codeName; + public Integer identifier; + public Integer sequence; + public Integer mtu; + public String gateway; + public String targetAddress; + public String destinationAddress; + public String originalSrcIp; + public String originalDstIp; + public Integer originalProtocol; + public Integer originalSrcPort; + public Integer originalDstPort; +} diff --git a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder index 5b788645c0f..beae056384c 100644 --- a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder +++ b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder @@ -17,3 +17,4 @@ # org.apache.drill.exec.store.pcap.protocol.dns.DnsDecoder org.apache.drill.exec.store.pcap.protocol.http.HttpPacketDecoder +org.apache.drill.exec.store.pcap.protocol.icmp.IcmpDecoder diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/icmp/TestIcmpDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/icmp/TestIcmpDecoder.java new file mode 100644 index 00000000000..7d273a14148 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/icmp/TestIcmpDecoder.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.icmp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.time.Instant; + +import org.apache.drill.exec.store.pcap.protocol.TestPackets; +import org.apache.drill.exec.store.pcapng.PacketDecoder; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestIcmpDecoder extends BaseTest { + private final IcmpDecoder decoder = new IcmpDecoder(); + + private static byte[] ip(String host) throws Exception { + return InetAddress.getByName(host).getAddress(); + } + + private static byte[] concat(byte[]... parts) { + int n = 0; + for (byte[] p : parts) { + n += p.length; + } + ByteBuffer out = ByteBuffer.allocate(n); + for (byte[] p : parts) { + out.put(p); + } + return out.array(); + } + + /** An IPv6 header with the given next header, followed by the message. */ + private static byte[] ipv6(String src, String dst, int nextHeader, byte[] payload) throws Exception { + return ByteBuffer.allocate(40 + payload.length).putInt(0x60000000).putShort((short) payload.length) + .put((byte) nextHeader).put((byte) 64).put(ip(src)).put(ip(dst)).put(payload).array(); + } + + private static PacketDecoder rawIp(byte[] packet) { + return TestPackets.decode(packet, Instant.EPOCH); + } + + @Test + public void testEchoRequestV4() { + byte[] icmp = {8, 0, 0, 0, 0x12, 0x34, 0, 7, 'h', 'i'}; + PacketDecoder packet = rawIp(TestPackets.ipv4("10.0.0.1", "10.0.0.2", 1, icmp)); + assertTrue(decoder.accepts(packet)); + IcmpMessage m = decoder.parse(packet, null, null); + assertEquals(4, m.version); + assertEquals(8, m.type); + assertEquals(0, m.code); + assertEquals("echo_request", m.typeName); + assertNull(m.codeName); + assertEquals(Integer.valueOf(0x1234), m.identifier); + assertEquals(Integer.valueOf(7), m.sequence); + assertNull(m.originalSrcIp); + } + + @Test + public void testNotAcceptedForUdp() { + assertFalse(decoder.accepts(TestPackets.udp("10.0.0.1", 1, "10.0.0.2", 2, new byte[8]))); + } + + @Test + public void testPortUnreachableWithEmbeddedUdp() throws Exception { + byte[] udp = ByteBuffer.allocate(8).putShort((short) 5000).putShort((short) 53).putShort((short) 8).array(); + byte[] original = TestPackets.ipv4("10.0.0.1", "8.8.8.8", 17, udp); + byte[] icmp = concat(new byte[] {3, 3, 0, 0, 0, 0, 0, 0}, original); + IcmpMessage m = IcmpDecoder.parse(icmp, 4); + assertEquals("destination_unreachable", m.typeName); + assertEquals("port_unreachable", m.codeName); + assertEquals("10.0.0.1", m.originalSrcIp); + assertEquals("8.8.8.8", m.originalDstIp); + assertEquals(Integer.valueOf(17), m.originalProtocol); + assertEquals(Integer.valueOf(5000), m.originalSrcPort); + assertEquals(Integer.valueOf(53), m.originalDstPort); + assertNull(m.mtu); + } + + @Test + public void testFragmentationNeededAndShortEmbeddedPacket() throws Exception { + // Embedded IPv4 header only, transport header not captured + byte[] original = TestPackets.ipv4("10.0.0.1", "10.0.0.9", 6, new byte[2]); + byte[] icmp = concat(new byte[] {3, 4, 0, 0, 0, 0, 0x05, (byte) 0xDC}, original); + IcmpMessage m = IcmpDecoder.parse(icmp, 4); + assertEquals("fragmentation_needed", m.codeName); + assertEquals(Integer.valueOf(1500), m.mtu); + assertEquals(Integer.valueOf(6), m.originalProtocol); + assertNull(m.originalSrcPort); + assertNull(m.originalDstPort); + } + + @Test + public void testEmbeddedHeaderCutShortIsIgnored() { + byte[] icmp = {11, 0, 0, 0, 0, 0, 0, 0, 0x45, 0, 0, 20}; + IcmpMessage m = IcmpDecoder.parse(icmp, 4); + assertEquals("time_exceeded", m.typeName); + assertEquals("ttl_exceeded_in_transit", m.codeName); + assertNull(m.originalSrcIp); + assertNull(m.originalProtocol); + } + + @Test + public void testRedirectGateway() throws Exception { + byte[] icmp = concat(new byte[] {5, 1, 0, 0}, ip("192.168.1.254")); + IcmpMessage m = IcmpDecoder.parse(icmp, 4); + assertEquals("redirect", m.typeName); + assertEquals("redirect_for_host", m.codeName); + assertEquals("192.168.1.254", m.gateway); + } + + @Test + public void testUnknownTypeIsStillIcmp() { + IcmpMessage m = IcmpDecoder.parse(new byte[] {(byte) 200, 1, 0, 0}, 4); + assertEquals(200, m.type); + assertEquals(1, m.code); + assertNull(m.typeName); + } + + @Test + public void testTooShortIsNotIcmp() { + assertNull(IcmpDecoder.parse(new byte[] {8, 0, 0}, 4)); + assertNull(IcmpDecoder.parse(null, 4)); + byte[] raw = TestPackets.ipv4("10.0.0.1", "10.0.0.2", 1, new byte[0]); + assertNull(decoder.parse(rawIp(raw), null, null)); + } + + @Test + public void testTruncatedEchoIsMalformed() { + try { + IcmpDecoder.parse(new byte[] {8, 0, 0, 0, 0, 1}, 4); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("truncated echo_request: 6 bytes, needs 8", e.getMessage()); + } + } + + @Test + public void testEchoReplyV6() throws Exception { + byte[] icmp = {(byte) 129, 0, 0, 0, 0, 5, 0, 9}; + PacketDecoder packet = rawIp(ipv6("2001:db8::1", "2001:db8::2", 58, icmp)); + assertTrue(decoder.accepts(packet)); + IcmpMessage m = decoder.parse(packet, null, null); + assertEquals(6, m.version); + assertEquals("echo_reply", m.typeName); + assertEquals(Integer.valueOf(5), m.identifier); + assertEquals(Integer.valueOf(9), m.sequence); + } + + @Test + public void testNeighborAdvertisement() throws Exception { + byte[] icmp = concat(new byte[] {(byte) 136, 0, 0, 0, 0x60, 0, 0, 0}, ip("fe80::1")); + IcmpMessage m = IcmpDecoder.parse(icmp, 6); + assertEquals("neighbor_advertisement", m.typeName); + assertEquals("fe80:0:0:0:0:0:0:1", m.targetAddress); + } + + @Test + public void testRedirectV6() throws Exception { + byte[] icmp = concat(new byte[] {(byte) 137, 0, 0, 0, 0, 0, 0, 0}, ip("fe80::1"), ip("2001:db8::5")); + IcmpMessage m = IcmpDecoder.parse(icmp, 6); + assertEquals("redirect", m.typeName); + assertEquals("fe80:0:0:0:0:0:0:1", m.targetAddress); + assertEquals("2001:db8:0:0:0:0:0:5", m.destinationAddress); + } + + @Test + public void testTruncatedNeighborSolicitationIsMalformed() { + try { + IcmpDecoder.parse(new byte[] {(byte) 135, 0, 0, 0, 0, 0, 0, 0, 1, 2}, 6); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("truncated neighbor_solicitation: 10 bytes, needs 24", e.getMessage()); + } + } + + @Test + public void testPacketTooBigWithEmbeddedTcpAfterExtensionHeader() throws Exception { + // Hop-by-hop options header (8 bytes) then TCP + byte[] hopByHop = {6, 0, 0, 0, 0, 0, 0, 0}; + byte[] tcp = ByteBuffer.allocate(4).putShort((short) 443).putShort((short) 50000).array(); + byte[] original = ipv6("2001:db8::1", "2001:db8::2", 0, concat(hopByHop, tcp)); + byte[] icmp = concat(new byte[] {2, 0, 0, 0, 0, 0, 0x05, 0x00}, original); + IcmpMessage m = IcmpDecoder.parse(icmp, 6); + assertEquals("packet_too_big", m.typeName); + assertEquals(Integer.valueOf(1280), m.mtu); + assertEquals("2001:db8:0:0:0:0:0:1", m.originalSrcIp); + assertEquals("2001:db8:0:0:0:0:0:2", m.originalDstIp); + assertEquals(Integer.valueOf(6), m.originalProtocol); + assertEquals(Integer.valueOf(443), m.originalSrcPort); + assertEquals(Integer.valueOf(50000), m.originalDstPort); + } + + @Test + public void testUnreachableV6CodeNames() { + IcmpMessage m = IcmpDecoder.parse(new byte[] {1, 4, 0, 0, 0, 0, 0, 0}, 6); + assertEquals("destination_unreachable", m.typeName); + assertEquals("port_unreachable", m.codeName); + assertNull(m.originalSrcIp); + } + + @Test + public void testExtensionHeaderLoopIsBounded() throws Exception { + // Every extension header claims another; the walk stops at the captured bytes + byte[] chain = new byte[64]; + for (int i = 0; i < chain.length; i += 8) { + chain[i] = 0; + } + byte[] original = ipv6("2001:db8::1", "2001:db8::2", 0, chain); + byte[] icmp = concat(new byte[] {3, 0, 0, 0, 0, 0, 0, 0}, original); + IcmpMessage m = IcmpDecoder.parse(icmp, 6); + assertEquals("2001:db8:0:0:0:0:0:1", m.originalSrcIp); + assertNull(m.originalSrcPort); + } +} From 8cef513c06ed8833da30d0d30b33f9c8a500a032 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:56:54 -0400 Subject: [PATCH 24/50] DRILL-8556: Add ARP protocol decoder --- .../store/pcap/protocol/arp/ArpDecoder.java | 163 ++++++++++++++++++ .../store/pcap/protocol/arp/ArpMessage.java | 33 ++++ ....store.pcap.protocol.PacketProtocolDecoder | 1 + .../pcap/protocol/arp/TestArpDecoder.java | 137 +++++++++++++++ 4 files changed, 334 insertions(+) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpMessage.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/arp/TestArpDecoder.java diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpDecoder.java new file mode 100644 index 00000000000..c8841d2c2d2 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpDecoder.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.arp; + +import java.util.Arrays; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** ARP and RARP (EtherType 0x0806). The message is the link-layer payload. */ +public class ArpDecoder implements PacketProtocolDecoder { + private static final int HEADER = 8; + private static final int ETHERNET = 1; + private static final int IPV4 = 0x0800; + private static final String[] OPERATIONS = {null, "request", "reply", "rarp_request", "rarp_reply", + "drarp_request", "drarp_reply", "drarp_error", "inarp_request", "inarp_reply", "arp_nak"}; + + @Override + public String protocol() { + return "arp"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("hardware_type", MinorType.INT) + .addNullable("protocol_type", MinorType.INT) + .addNullable("operation", MinorType.INT) + .addNullable("operation_name", MinorType.VARCHAR) + .addNullable("sender_mac", MinorType.VARCHAR) + .addNullable("sender_ip", MinorType.VARCHAR) + .addNullable("target_mac", MinorType.VARCHAR) + .addNullable("target_ip", MinorType.VARCHAR) + .addNullable("is_gratuitous", MinorType.BIT) + .addNullable("is_probe", MinorType.BIT); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isArpPacket(); + } + + @Override + public ArpMessage parse(Packet packet, byte[] payload, DecoderContext context) { + // payload is the TCP or UDP payload, null here; the ARP message is the link payload + return parse(packet.getLinkPayload()); + } + + /** + * @param b the ARP message, possibly followed by link-layer padding + * @return null if the address lengths in the header do not fit the message + * @throws IllegalArgumentException if an Ethernet/IPv4 message is cut short + */ + public static ArpMessage parse(byte[] b) { + if (b == null || b.length < HEADER) { + return null; + } + int hardwareType = u16(b, 0); + int protocolType = u16(b, 2); + int hardwareLength = b[4] & 0xFF; + int protocolLength = b[5] & 0xFF; + int needed = HEADER + 2 * (hardwareLength + protocolLength); + if (hardwareLength == 0 || protocolLength == 0) { + return null; + } + if (needed > b.length) { + if (hardwareType == ETHERNET && protocolType == IPV4 && hardwareLength == 6 && protocolLength == 4) { + throw new IllegalArgumentException(String.format("truncated: %d bytes, needs %d", b.length, needed)); + } + return null; + } + ArpMessage m = new ArpMessage(); + m.hardwareType = hardwareType; + m.protocolType = protocolType; + m.operation = u16(b, 6); + m.operationName = m.operation < OPERATIONS.length ? OPERATIONS[m.operation] : null; + int senderHardware = HEADER; + int senderProtocol = senderHardware + hardwareLength; + int targetHardware = senderProtocol + protocolLength; + int targetProtocol = targetHardware + hardwareLength; + boolean ipv4 = protocolType == IPV4 && protocolLength == 4; + m.senderMac = hardware(b, senderHardware, hardwareLength); + m.senderIp = protocolAddress(b, senderProtocol, protocolLength, ipv4); + m.targetMac = hardware(b, targetHardware, hardwareLength); + m.targetIp = protocolAddress(b, targetProtocol, protocolLength, ipv4); + m.isGratuitous = Arrays.equals(Arrays.copyOfRange(b, senderProtocol, senderProtocol + protocolLength), + Arrays.copyOfRange(b, targetProtocol, targetProtocol + protocolLength)); + m.isProbe = true; + for (int i = senderProtocol; i < senderProtocol + protocolLength; i++) { + if (b[i] != 0) { + m.isProbe = false; + break; + } + } + return m; + } + + /** Colon-separated upper-case hex, like the eth_src column. */ + private static String hardware(byte[] b, int offset, int length) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i > 0) { + sb.append(':'); + } + sb.append(String.format("%02X", b[offset + i] & 0xFF)); + } + return sb.toString(); + } + + /** Dotted quad for IPv4, otherwise hex. */ + private static String protocolAddress(byte[] b, int offset, int length, boolean ipv4) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (ipv4) { + if (i > 0) { + sb.append('.'); + } + sb.append(b[offset + i] & 0xFF); + } else { + sb.append(String.format("%02x", b[offset + i] & 0xFF)); + } + } + return sb.toString(); + } + + private static int u16(byte[] b, int offset) { + return ((b[offset] & 0xFF) << 8) | (b[offset + 1] & 0xFF); + } + + @Override + public void write(ArpMessage m, TupleWriter fields) { + fields.scalar("hardware_type").setInt(m.hardwareType); + fields.scalar("protocol_type").setInt(m.protocolType); + fields.scalar("operation").setInt(m.operation); + if (m.operationName != null) { + fields.scalar("operation_name").setString(m.operationName); + } + fields.scalar("sender_mac").setString(m.senderMac); + fields.scalar("sender_ip").setString(m.senderIp); + fields.scalar("target_mac").setString(m.targetMac); + fields.scalar("target_ip").setString(m.targetIp); + fields.scalar("is_gratuitous").setBoolean(m.isGratuitous); + fields.scalar("is_probe").setBoolean(m.isProbe); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpMessage.java new file mode 100644 index 00000000000..87bb2be0cf5 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpMessage.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.arp; + +/** The fields of one ARP message. */ +public class ArpMessage { + public int hardwareType; + public int protocolType; + public int operation; + /** Null for an unknown operation. */ + public String operationName; + public String senderMac; + public String senderIp; + public String targetMac; + public String targetIp; + public boolean isGratuitous; + public boolean isProbe; +} diff --git a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder index beae056384c..09ba0dff9b4 100644 --- a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder +++ b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder @@ -18,3 +18,4 @@ org.apache.drill.exec.store.pcap.protocol.dns.DnsDecoder org.apache.drill.exec.store.pcap.protocol.http.HttpPacketDecoder org.apache.drill.exec.store.pcap.protocol.icmp.IcmpDecoder +org.apache.drill.exec.store.pcap.protocol.arp.ArpDecoder diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/arp/TestArpDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/arp/TestArpDecoder.java new file mode 100644 index 00000000000..a318bb8ec5d --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/arp/TestArpDecoder.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.arp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.Arrays; + +import org.apache.drill.exec.store.pcap.protocol.TestPackets; +import org.apache.drill.exec.store.pcapng.PacketDecoder; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestArpDecoder extends BaseTest { + private static final byte[] ETHERNET = {(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, + 2, 0, 0, 0, 0, 1, 8, 6}; + private final ArpDecoder decoder = new ArpDecoder(); + + private static byte[] arp(int operation, int[] senderIp, int[] targetIp, int targetLastMacByte) { + ByteBuffer b = ByteBuffer.allocate(28).putShort((short) 1).putShort((short) 0x0800).put((byte) 6).put((byte) 4) + .putShort((short) operation).put(new byte[] {2, 0, 0, 0, 0, 1}); + for (int v : senderIp) { + b.put((byte) v); + } + b.put(new byte[] {0, 0, 0, 0, 0, (byte) targetLastMacByte}); + for (int v : targetIp) { + b.put((byte) v); + } + return b.array(); + } + + private static PacketDecoder frame(byte[] arp) { + byte[] frame = new byte[ETHERNET.length + arp.length]; + System.arraycopy(ETHERNET, 0, frame, 0, ETHERNET.length); + System.arraycopy(arp, 0, frame, ETHERNET.length, arp.length); + PacketDecoder packet = new PacketDecoder(); + packet.readPcapng(frame, 1); + return packet; + } + + @Test + public void testRequest() { + PacketDecoder packet = frame(arp(1, new int[] {10, 0, 0, 1}, new int[] {10, 0, 0, 2}, 0)); + assertTrue(decoder.accepts(packet)); + ArpMessage m = decoder.parse(packet, null, null); + assertEquals(1, m.hardwareType); + assertEquals(0x0800, m.protocolType); + assertEquals(1, m.operation); + assertEquals("request", m.operationName); + assertEquals("02:00:00:00:00:01", m.senderMac); + assertEquals("10.0.0.1", m.senderIp); + assertEquals("00:00:00:00:00:00", m.targetMac); + assertEquals("10.0.0.2", m.targetIp); + assertFalse(m.isGratuitous); + assertFalse(m.isProbe); + } + + @Test + public void testIgnoresEthernetPadding() { + byte[] padded = Arrays.copyOf(arp(2, new int[] {10, 0, 0, 2}, new int[] {10, 0, 0, 1}, 0xAB), 46); + ArpMessage m = decoder.parse(frame(padded), null, null); + assertEquals("reply", m.operationName); + assertEquals("00:00:00:00:00:AB", m.targetMac); + } + + @Test + public void testGratuitousAndProbe() { + assertTrue(ArpDecoder.parse(arp(1, new int[] {10, 0, 0, 5}, new int[] {10, 0, 0, 5}, 0)).isGratuitous); + ArpMessage probe = ArpDecoder.parse(arp(1, new int[] {0, 0, 0, 0}, new int[] {10, 0, 0, 5}, 0)); + assertTrue(probe.isProbe); + assertFalse(probe.isGratuitous); + } + + @Test + public void testOtherLengthsRenderHex() { + // Hardware length 2, protocol length 3, operation 8 (InARP request) + byte[] arp = {0, 6, 0x12, 0x34, 2, 3, 0, 8, (byte) 0xAA, (byte) 0xBB, 1, 2, 3, (byte) 0xCC, (byte) 0xDD, 4, 5, 6}; + ArpMessage m = ArpDecoder.parse(arp); + assertEquals("inarp_request", m.operationName); + assertEquals("AA:BB", m.senderMac); + assertEquals("010203", m.senderIp); + assertEquals("CC:DD", m.targetMac); + assertEquals("040506", m.targetIp); + } + + @Test + public void testLengthsThatDoNotFitAreNotArp() { + // Declares 16-byte hardware addresses in a 28-byte message + byte[] arp = arp(1, new int[] {10, 0, 0, 1}, new int[] {10, 0, 0, 2}, 0); + arp[4] = 16; + assertNull(ArpDecoder.parse(arp)); + assertNull(ArpDecoder.parse(new byte[] {0, 1, 8, 0, 6})); + assertNull(ArpDecoder.parse(null)); + byte[] zero = arp.clone(); + zero[4] = 0; + zero[5] = 0; + assertNull(ArpDecoder.parse(zero)); + } + + @Test + public void testTruncatedEthernetIpv4IsMalformed() { + byte[] arp = Arrays.copyOf(arp(1, new int[] {10, 0, 0, 1}, new int[] {10, 0, 0, 2}, 0), 20); + try { + ArpDecoder.parse(arp); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("truncated: 20 bytes, needs 28", e.getMessage()); + } + } + + @Test + public void testNotAcceptedForIp() { + assertFalse(decoder.accepts(TestPackets.decode(TestPackets.ipv4("10.0.0.1", "10.0.0.2", 1, new byte[8]), + Instant.EPOCH))); + } +} From c63ab79acc02466bc4fc88717925d71fd79ddce3 Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 15:56:54 -0400 Subject: [PATCH 25/50] DRILL-8556: Add ICMP and ARP decoder fixtures and query tests --- .../protocol/icmp/TestIcmpArpQueries.java | 126 ++++++++++++++++++ .../test/resources/decoders/arp/arp.pcapng | Bin 0 -> 496 bytes .../test/resources/decoders/icmp/icmp.pcapng | Bin 0 -> 684 bytes .../resources/decoders/icmp/icmp_arp.pcap | Bin 0 -> 276 bytes .../resources/fixtures/icmp_arp_fixtures.py | 116 ++++++++++++++++ 5 files changed, 242 insertions(+) create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/icmp/TestIcmpArpQueries.java create mode 100644 contrib/format-pcapng/src/test/resources/decoders/arp/arp.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/decoders/icmp/icmp.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/decoders/icmp/icmp_arp.pcap create mode 100644 contrib/format-pcapng/src/test/resources/fixtures/icmp_arp_fixtures.py diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/icmp/TestIcmpArpQueries.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/icmp/TestIcmpArpQueries.java new file mode 100644 index 00000000000..af9dfc1e456 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/icmp/TestIcmpArpQueries.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.icmp; + +import java.nio.file.Paths; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.physical.rowSet.RowSet; +import org.apache.drill.exec.physical.rowSet.RowSetBuilder; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.ClusterTest; +import org.apache.drill.test.rowSet.RowSetComparison; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Queries of the icmp and arp decoders through both readers. Fixtures: fixtures/icmp_arp_fixtures.py. */ +public class TestIcmpArpQueries extends ClusterTest { + + @BeforeClass + public static void setup() throws Exception { + ClusterTest.startCluster(ClusterFixture.builder(dirTestWatcher)); + dirTestWatcher.copyResourceToRoot(Paths.get("decoders/")); + } + + @Test + public void testIcmpPcapng() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.icmp.version as version, " + + "t.parsed_data.icmp.type_name as type_name, t.parsed_data.icmp.code_name as code_name, " + + "t.parsed_data.icmp.identifier as identifier, t.parsed_data.icmp.mtu as mtu, " + + "t.parsed_data.icmp.target_address as target, t.parsed_data.icmp.original_dst_ip as original_dst, " + + "t.parsed_data.icmp.original_dst_port as original_port, decode_error " + + "from dfs.`decoders/icmp/icmp.pcapng` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("version", MinorType.INT) + .addNullable("type_name", MinorType.VARCHAR) + .addNullable("code_name", MinorType.VARCHAR) + .addNullable("identifier", MinorType.INT) + .addNullable("mtu", MinorType.INT) + .addNullable("target", MinorType.VARCHAR) + .addNullable("original_dst", MinorType.VARCHAR) + .addNullable("original_port", MinorType.INT) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("icmp", 4, "echo_request", null, 0x1234, null, null, null, null, null) + .addRow("icmp", 4, "echo_reply", null, 0x1234, null, null, null, null, null) + .addRow("icmp", 4, "destination_unreachable", "port_unreachable", null, null, null, "8.8.8.8", 53, null) + .addRow("icmp", 6, "neighbor_solicitation", null, null, null, "fe80:0:0:0:0:0:0:2", null, null, null) + .addRow("icmp", 6, "packet_too_big", null, null, 1280, null, "2001:db8:0:0:0:0:0:2", 443, null) + .addRow(null, null, null, null, null, null, null, null, null, null) + .addRow("icmp", null, null, null, null, null, null, null, null, "icmp: truncated echo_request: 6 bytes, needs 8") + .addRow(null, null, null, null, null, null, null, null, null, null) + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testArpPcapng() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.arp.operation_name as op, " + + "t.parsed_data.arp.sender_mac as sender_mac, t.parsed_data.arp.sender_ip as sender_ip, " + + "t.parsed_data.arp.target_ip as target_ip, t.parsed_data.arp.is_gratuitous as gratuitous, " + + "t.parsed_data.arp.is_probe as probe, decode_error from dfs.`decoders/arp/arp.pcapng` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("op", MinorType.VARCHAR) + .addNullable("sender_mac", MinorType.VARCHAR) + .addNullable("sender_ip", MinorType.VARCHAR) + .addNullable("target_ip", MinorType.VARCHAR) + .addNullable("gratuitous", MinorType.BIT) + .addNullable("probe", MinorType.BIT) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("arp", "request", "02:00:00:00:00:01", "10.0.0.1", "10.0.0.2", false, false, null) + .addRow("arp", "reply", "02:00:00:00:00:02", "10.0.0.2", "10.0.0.1", false, false, null) + .addRow("arp", "request", "02:00:00:00:00:01", "10.0.0.1", "10.0.0.1", true, false, null) + .addRow("arp", "request", "02:00:00:00:00:01", "0.0.0.0", "10.0.0.9", false, true, null) + .addRow(null, null, null, null, null, null, null, null) + .addRow("arp", null, null, null, null, null, null, "arp: truncated: 20 bytes, needs 28") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testClassicPcap() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.icmp.type_name as icmp_type, " + + "t.parsed_data.icmp.sequence as seq, t.parsed_data.arp.operation_name as arp_op, " + + "t.parsed_data.arp.target_ip as arp_target, decode_error from dfs.`decoders/icmp/icmp_arp.pcap` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("icmp_type", MinorType.VARCHAR) + .addNullable("seq", MinorType.INT) + .addNullable("arp_op", MinorType.VARCHAR) + .addNullable("arp_target", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("arp", null, null, "request", "10.0.0.2", null) + .addRow("icmp", "echo_request", 1, null, null, null) + .addRow("icmp", null, null, null, null, "icmp: truncated echo_request: 6 bytes, needs 8") + .addRow(null, null, null, null, null, null) + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } +} diff --git a/contrib/format-pcapng/src/test/resources/decoders/arp/arp.pcapng b/contrib/format-pcapng/src/test/resources/decoders/arp/arp.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..ea407f3c3b00600cf686480eb7ea5f69e48eafc3 GIT binary patch literal 496 zcmd<$<>iuLU|{gI(UxKa(*L0VBnIM(K~i{DeWyc;G#nC4=wC01oId^f~8>N3;7C( z=iA*IWVgz|Vb1)VnVZbUNfIBJ*?Br1I{8&)We_X%$y$Wl6QKmfe->Q0;f2kN`fR&sfLF)3={+Xhq%TK=|`7Rlm*>T n97u<2=(r#nv}ZKPf|i2-X>se@@ZIcu*fzJh&gb(x)zJO{lHp3i literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/decoders/icmp/icmp_arp.pcap b/contrib/format-pcapng/src/test/resources/decoders/icmp/icmp_arp.pcap new file mode 100644 index 0000000000000000000000000000000000000000..c3b98402aa0c7c7e26b64a22babaee2dc513b3d8 GIT binary patch literal 276 zcmca|c+)~A1{MYw`2U}Qff2}2t(}|-60?Eg|6srb5@O_F17Zd?76wKLp9`V|CKn-e^9#9B`!Ah8*W-+)jFerf3IsmN&nFKO}1IQOLVPGuC%u82?>ec{?fiQ+{ ZDWD*RZU#mTs6H*AEC^%qfd+OT008=C9RvUX literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/fixtures/icmp_arp_fixtures.py b/contrib/format-pcapng/src/test/resources/fixtures/icmp_arp_fixtures.py new file mode 100644 index 00000000000..fa95d0a97f1 --- /dev/null +++ b/contrib/format-pcapng/src/test/resources/fixtures/icmp_arp_fixtures.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Generates the ICMP, ICMPv6 and ARP decoder fixtures. Run from any directory: + python3 contrib/format-pcapng/src/test/resources/fixtures/icmp_arp_fixtures.py""" +import os +import socket +import struct +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from pcap_fixtures import ETHERNET, epb, idb, ipv4, pcap_file, shb, udp, write # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) +DECODERS = os.path.join(HERE, '..', 'decoders') +ETHERNET_ARP = bytes.fromhex('ffffffffffff' '020000000001' '0806') +MAC1 = bytes.fromhex('020000000001') +MAC2 = bytes.fromhex('020000000002') + + +def ipv6(next_header, payload, src, dst): + return struct.pack('>IHBB16s16s', 0x60000000, len(payload), next_header, 64, + socket.inet_pton(socket.AF_INET6, src), socket.inet_pton(socket.AF_INET6, dst)) + payload + + +def icmp(icmp_type, code, rest, data=b''): + # Checksums are not verified by the decoder, so they are left zero + return struct.pack('>BBH', icmp_type, code, 0) + rest + data + + +def echo(icmp_type, ident, seq): + return icmp(icmp_type, 0, struct.pack('>HH', ident, seq), b'ping') + + +def port_unreachable(): + original = ipv4(17, udp(5000, 53, b'query'), '10.0.0.1', '8.8.8.8') + return icmp(3, 3, bytes(4), original) + + +def neighbor_solicitation(): + return icmp(135, 0, bytes(4), socket.inet_pton(socket.AF_INET6, 'fe80::2')) + + +def packet_too_big(): + original = ipv6(17, udp(5000, 443, b'data'), '2001:db8::1', '2001:db8::2') + return icmp(2, 0, struct.pack('>I', 1280), original) + + +def arp(operation, sender_mac, sender_ip, target_mac, target_ip): + return struct.pack('>HHBBH6s4s6s4s', 1, 0x0800, 6, 4, operation, sender_mac, socket.inet_aton(sender_ip), + target_mac, socket.inet_aton(target_ip)) + + +def icmp_pcapng(): + # Link type 101. Rows: echo request, echo reply, port unreachable, ICMPv6 neighbor solicitation, + # ICMPv6 packet too big, a 3-byte message (not ICMP), a truncated echo request, a UDP packet + f = shb() + idb(101) + f += epb(0, ipv4(1, echo(8, 0x1234, 1), '10.0.0.1', '10.0.0.2')) + f += epb(0, ipv4(1, echo(0, 0x1234, 1), '10.0.0.2', '10.0.0.1')) + f += epb(0, ipv4(1, port_unreachable(), '8.8.8.8', '10.0.0.1')) + f += epb(0, ipv6(58, neighbor_solicitation(), 'fe80::1', 'ff02::1:ff00:2')) + f += epb(0, ipv6(58, packet_too_big(), '2001:db8::fe', '2001:db8::1')) + f += epb(0, ipv4(1, b'\x08\x00\x00', '10.0.0.1', '10.0.0.2')) + f += epb(0, ipv4(1, echo(8, 1, 2)[:6], '10.0.0.1', '10.0.0.2')) + f += epb(0, ipv4(17, udp(4000, 7, b'hello'), '10.0.0.1', '10.0.0.2')) + write(os.path.join(DECODERS, 'icmp', 'icmp.pcapng'), f) + + +def arp_pcapng(): + # Link type 1. Rows: request, reply, gratuitous, probe, lengths that do not fit (not ARP), truncated + bad_lengths = bytearray(arp(1, MAC1, '10.0.0.1', bytes(6), '10.0.0.2')) + bad_lengths[4] = 16 + f = shb() + idb(1) + for message in (arp(1, MAC1, '10.0.0.1', bytes(6), '10.0.0.2'), + arp(2, MAC2, '10.0.0.2', MAC1, '10.0.0.1'), + arp(1, MAC1, '10.0.0.1', bytes(6), '10.0.0.1'), + arp(1, MAC1, '0.0.0.0', bytes(6), '10.0.0.9'), + bytes(bad_lengths), + arp(1, MAC1, '10.0.0.1', bytes(6), '10.0.0.2')[:20]): + f += epb(0, ETHERNET_ARP + message) + write(os.path.join(DECODERS, 'arp', 'arp.pcapng'), f) + + +def classic_pcap(): + # The classic reader decodes Ethernet only. Rows: ARP request padded to the Ethernet minimum, + # echo request, truncated echo request, ARP with lengths that do not fit + bad_lengths = bytearray(arp(1, MAC1, '10.0.0.1', bytes(6), '10.0.0.2')) + bad_lengths[5] = 40 + frames = [ETHERNET_ARP + arp(1, MAC1, '10.0.0.1', bytes(6), '10.0.0.2') + bytes(18), + ETHERNET + ipv4(1, echo(8, 0x1234, 1), '10.0.0.1', '10.0.0.2'), + ETHERNET + ipv4(1, echo(8, 1, 2)[:6], '10.0.0.1', '10.0.0.2'), + ETHERNET_ARP + bytes(bad_lengths)] + pcap_file(os.path.join(DECODERS, 'icmp', 'icmp_arp.pcap'), frames) + + +if __name__ == '__main__': + for name in ('icmp', 'arp'): + os.makedirs(os.path.join(DECODERS, name), exist_ok=True) + icmp_pcapng() + arp_pcapng() + classic_pcap() From 8c4f981d89257d3c7c148fa6077e6d282ea9b9ac Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 16:00:34 -0400 Subject: [PATCH 26/50] DRILL-8556: Add NTP, syslog, SSDP and SIP protocol decoders Packet decoders for NTP (UDP 123, including control and private mode request codes), syslog (UDP 514, RFC 5424 and RFC 3164), SSDP (UDP 1900) and SIP (UDP and TCP 5060, with compact header names and folded lines). Each has unit tests, a generated fixture under decoders/, and a query test. --- .../store/pcap/protocol/ntp/NtpDecoder.java | 97 +++++++ .../store/pcap/protocol/ntp/NtpMessage.java | 39 +++ .../store/pcap/protocol/ntp/NtpParser.java | 155 +++++++++++ .../store/pcap/protocol/sip/SipDecoder.java | 114 ++++++++ .../store/pcap/protocol/sip/SipMessage.java | 46 ++++ .../store/pcap/protocol/sip/SipParser.java | 246 ++++++++++++++++++ .../store/pcap/protocol/ssdp/SsdpDecoder.java | 99 +++++++ .../store/pcap/protocol/ssdp/SsdpMessage.java | 41 +++ .../store/pcap/protocol/ssdp/SsdpParser.java | 142 ++++++++++ .../pcap/protocol/syslog/SyslogDecoder.java | 85 ++++++ .../pcap/protocol/syslog/SyslogMessage.java | 35 +++ .../pcap/protocol/syslog/SyslogParser.java | 179 +++++++++++++ ....store.pcap.protocol.PacketProtocolDecoder | 4 + .../protocol/TestServiceDecoderQueries.java | 152 +++++++++++ .../pcap/protocol/ntp/TestNtpDecoder.java | 158 +++++++++++ .../pcap/protocol/sip/TestSipDecoder.java | 153 +++++++++++ .../pcap/protocol/ssdp/TestSsdpDecoder.java | 135 ++++++++++ .../protocol/syslog/TestSyslogDecoder.java | 163 ++++++++++++ .../test/resources/decoders/ntp/ntp.pcapng | Bin 0 -> 556 bytes .../test/resources/decoders/sip/sip.pcapng | Bin 0 -> 1160 bytes .../test/resources/decoders/ssdp/ssdp.pcapng | Bin 0 -> 968 bytes .../resources/decoders/syslog/syslog.pcapng | Bin 0 -> 556 bytes .../test/resources/fixtures/ntp_fixtures.py | 64 +++++ .../test/resources/fixtures/sip_fixtures.py | 68 +++++ .../test/resources/fixtures/ssdp_fixtures.py | 54 ++++ .../resources/fixtures/syslog_fixtures.py | 50 ++++ 26 files changed, 2279 insertions(+) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpMessage.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpParser.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipMessage.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipParser.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpMessage.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpParser.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogMessage.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogParser.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestServiceDecoderQueries.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/ntp/TestNtpDecoder.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/sip/TestSipDecoder.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/ssdp/TestSsdpDecoder.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/syslog/TestSyslogDecoder.java create mode 100644 contrib/format-pcapng/src/test/resources/decoders/ntp/ntp.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/decoders/sip/sip.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/decoders/ssdp/ssdp.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/decoders/syslog/syslog.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/fixtures/ntp_fixtures.py create mode 100644 contrib/format-pcapng/src/test/resources/fixtures/sip_fixtures.py create mode 100644 contrib/format-pcapng/src/test/resources/fixtures/ssdp_fixtures.py create mode 100644 contrib/format-pcapng/src/test/resources/fixtures/syslog_fixtures.py diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpDecoder.java new file mode 100644 index 00000000000..914bba69990 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpDecoder.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.ntp; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** NTP over UDP port 123. */ +public class NtpDecoder implements PacketProtocolDecoder { + private static final int PORT = 123; + + @Override + public String protocol() { + return "ntp"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("leap_indicator", MinorType.INT) + .addNullable("version", MinorType.INT) + .addNullable("mode", MinorType.VARCHAR) + .addNullable("stratum", MinorType.INT) + .addNullable("poll", MinorType.INT) + .addNullable("precision", MinorType.INT) + .addNullable("root_delay", MinorType.FLOAT8) + .addNullable("root_dispersion", MinorType.FLOAT8) + .addNullable("reference_id", MinorType.VARCHAR) + .addNullable("reference_time", MinorType.TIMESTAMP) + .addNullable("origin_time", MinorType.TIMESTAMP) + .addNullable("receive_time", MinorType.TIMESTAMP) + .addNullable("transmit_time", MinorType.TIMESTAMP) + .addNullable("request_code", MinorType.INT); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isUdpPacket() && (packet.getSrc_port() == PORT || packet.getDst_port() == PORT); + } + + @Override + public NtpMessage parse(Packet packet, byte[] payload, DecoderContext context) { + return NtpParser.parse(payload, context); + } + + @Override + public void write(NtpMessage m, TupleWriter fields) { + if (m.leapIndicator != null) { + fields.scalar("leap_indicator").setInt(m.leapIndicator); + } + fields.scalar("version").setInt(m.version); + fields.scalar("mode").setString(m.mode); + if (m.stratum != null) { + fields.scalar("stratum").setInt(m.stratum); + fields.scalar("poll").setInt(m.poll); + fields.scalar("precision").setInt(m.precision); + fields.scalar("root_delay").setDouble(m.rootDelay); + fields.scalar("root_dispersion").setDouble(m.rootDispersion); + } + if (m.referenceId != null) { + fields.scalar("reference_id").setString(m.referenceId); + } + if (m.referenceTime != null) { + fields.scalar("reference_time").setTimestamp(m.referenceTime); + } + if (m.originTime != null) { + fields.scalar("origin_time").setTimestamp(m.originTime); + } + if (m.receiveTime != null) { + fields.scalar("receive_time").setTimestamp(m.receiveTime); + } + if (m.transmitTime != null) { + fields.scalar("transmit_time").setTimestamp(m.transmitTime); + } + if (m.requestCode != null) { + fields.scalar("request_code").setInt(m.requestCode); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpMessage.java new file mode 100644 index 00000000000..9e6a6acc005 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpMessage.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.ntp; + +import java.time.Instant; + +/** One NTP message. Fields that the message's mode does not carry are null. */ +public class NtpMessage { + public Integer leapIndicator; + public int version; + public String mode; + public Integer stratum; + public Integer poll; + public Integer precision; + public Double rootDelay; + public Double rootDispersion; + public String referenceId; + public Instant referenceTime; + public Instant originTime; + public Instant receiveTime; + public Instant transmitTime; + /** Opcode of a control (mode 6) message or request code of a private (mode 7) message. */ + public Integer requestCode; +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpParser.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpParser.java new file mode 100644 index 00000000000..86a2d5ee647 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ntp/NtpParser.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.ntp; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; + +/** + * Parses NTP messages (RFC 5905 and earlier versions). Modes 1 to 5 carry the 48-byte + * header; extension fields and a MAC may follow and are ignored. Modes 6 (control, as + * used by ntpq) and 7 (private, as used by ntpdc, including monlist) have their own short + * headers, of which only the request code is returned. + */ +public final class NtpParser { + static final int HEADER_LENGTH = 48; + private static final long NTP_UNIX_OFFSET = 2208988800L; + private static final long ERA_SECONDS = 1L << 32; + private static final int MAX_STRATUM = 16; + private static final int CONTROL_HEADER = 12; + private static final int PRIVATE_HEADER = 8; + private static final String[] MODES = { + null, "symmetric_active", "symmetric_passive", "client", "server", "broadcast", "control", "private"}; + + private NtpParser() { } + + /** + * @return the message, or null if the data is not NTP + * @throws IllegalArgumentException if a control or private message is shorter than its header says + */ + public static NtpMessage parse(byte[] data, DecoderContext context) { + if (data == null || data.length < 1) { + return null; + } + int first = data[0] & 0xFF; + int version = (first >> 3) & 0x07; + int mode = first & 0x07; + if (version == 0 || version > 4 || mode == 0) { + return null; + } + NtpMessage m = new NtpMessage(); + m.version = version; + m.mode = MODES[mode]; + if (mode == 6) { + return control(data, m); + } + if (mode == 7) { + return privateMode(data, m); + } + if (data.length < HEADER_LENGTH || (data[1] & 0xFF) > MAX_STRATUM) { + return null; + } + m.leapIndicator = first >> 6; + m.stratum = data[1] & 0xFF; + m.poll = (int) data[2]; + m.precision = (int) data[3]; + m.rootDelay = u32(data, 4) / 65536.0; + m.rootDispersion = u32(data, 8) / 65536.0; + m.referenceId = referenceId(data, m.stratum); + m.referenceTime = timestamp(data, 16); + m.originTime = timestamp(data, 24); + m.receiveTime = timestamp(data, 32); + m.transmitTime = timestamp(data, 40); + return m; + } + + private static NtpMessage control(byte[] data, NtpMessage m) { + if (data.length < CONTROL_HEADER) { + return null; + } + int count = u16(data, 10); + if (CONTROL_HEADER + count > data.length) { + throw new IllegalArgumentException("truncated control message: count " + count + " exceeds " + + (data.length - CONTROL_HEADER) + " data bytes"); + } + m.requestCode = data[1] & 0x1F; + return m; + } + + private static NtpMessage privateMode(byte[] data, NtpMessage m) { + if (data.length < PRIVATE_HEADER) { + return null; + } + int items = u16(data, 4) & 0x0FFF; + int itemSize = u16(data, 6) & 0x0FFF; + if (PRIVATE_HEADER + items * itemSize > data.length) { + throw new IllegalArgumentException("truncated private message: " + items + " items of " + itemSize + + " bytes exceed " + (data.length - PRIVATE_HEADER) + " data bytes"); + } + m.requestCode = data[3] & 0xFF; + return m; + } + + /** Kiss code or reference clock name for stratum 0 and 1, otherwise an IPv4 address. Null if zero. */ + private static String referenceId(byte[] data, int stratum) { + if (u32(data, 12) == 0) { + return null; + } + if (stratum <= 1) { + int length = 0; + while (length < 4 && data[12 + length] != 0) { + int c = data[12 + length] & 0xFF; + if (c < 0x20 || c > 0x7E) { + length = -1; + break; + } + length++; + } + if (length > 0) { + return new String(data, 12, length, StandardCharsets.US_ASCII).trim(); + } + } + return (data[12] & 0xFF) + "." + (data[13] & 0xFF) + "." + (data[14] & 0xFF) + "." + (data[15] & 0xFF); + } + + /** + * NTP timestamps count seconds from 1900. Seconds with the high bit clear are taken to be in + * era 1 (from 2036), as RFC 4330 suggests. A zero timestamp means "not set" and is null. + */ + static Instant timestamp(byte[] data, int offset) { + long seconds = u32(data, offset); + long fraction = u32(data, offset + 4); + if (seconds == 0 && fraction == 0) { + return null; + } + if ((seconds & 0x80000000L) == 0) { + seconds += ERA_SECONDS; + } + return Instant.ofEpochSecond(seconds - NTP_UNIX_OFFSET, (fraction * 1_000_000_000L) >>> 32); + } + + private static int u16(byte[] data, int offset) { + return ((data[offset] & 0xFF) << 8) | (data[offset + 1] & 0xFF); + } + + private static long u32(byte[] data, int offset) { + return ((long) u16(data, offset) << 16) | u16(data, offset + 2); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipDecoder.java new file mode 100644 index 00000000000..cf8d21b4f8a --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipDecoder.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.sip; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.ScalarWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** SIP over UDP and TCP port 5060. Port 5061 carries SIP over TLS and is not decoded. */ +public class SipDecoder implements PacketProtocolDecoder { + private static final int PORT = 5060; + + @Override + public String protocol() { + return "sip"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("is_request", MinorType.BIT) + .addNullable("method", MinorType.VARCHAR) + .addNullable("request_uri", MinorType.VARCHAR) + .addNullable("status_code", MinorType.INT) + .addNullable("reason", MinorType.VARCHAR) + .addNullable("from_address", MinorType.VARCHAR) + .addNullable("to_address", MinorType.VARCHAR) + .addNullable("call_id", MinorType.VARCHAR) + .addNullable("cseq", MinorType.VARCHAR) + .addNullable("user_agent", MinorType.VARCHAR) + .addNullable("contact", MinorType.VARCHAR) + .addArray("via", MinorType.VARCHAR) + .addNullable("content_type", MinorType.VARCHAR) + .addNullable("content_length", MinorType.BIGINT) + .addNullable("username", MinorType.VARCHAR) + .addNullable("password_present", MinorType.BIT) + .addMapArray("headers") + .addNullable("name", MinorType.VARCHAR) + .addNullable("value", MinorType.VARCHAR) + .resumeSchema(); + } + + @Override + public boolean accepts(Packet packet) { + return (packet.isUdpPacket() || packet.isTcpPacket()) + && (packet.getSrc_port() == PORT || packet.getDst_port() == PORT); + } + + @Override + public SipMessage parse(Packet packet, byte[] payload, DecoderContext context) { + return SipParser.parse(payload, context); + } + + @Override + public void write(SipMessage m, TupleWriter fields) { + fields.scalar("is_request").setBoolean(m.isRequest); + setString(fields, "method", m.method); + setString(fields, "request_uri", m.requestUri); + if (m.statusCode != null) { + fields.scalar("status_code").setInt(m.statusCode); + } + setString(fields, "reason", m.reason); + setString(fields, "from_address", m.from); + setString(fields, "to_address", m.to); + setString(fields, "call_id", m.callId); + setString(fields, "cseq", m.cseq); + setString(fields, "user_agent", m.userAgent); + setString(fields, "contact", m.contact); + ScalarWriter via = fields.array("via").scalar(); + for (String v : m.via) { + via.setString(v); + } + setString(fields, "content_type", m.contentType); + if (m.contentLength != null) { + fields.scalar("content_length").setLong(m.contentLength); + } + if (m.username != null) { + fields.scalar("username").setString(m.username); + fields.scalar("password_present").setBoolean(m.passwordPresent); + } + ArrayWriter headers = fields.array("headers"); + for (String[] h : m.headers) { + TupleWriter t = headers.tuple(); + t.scalar("name").setString(h[0]); + t.scalar("value").setString(h[1]); + headers.save(); + } + } + + private static void setString(TupleWriter fields, String name, String value) { + if (value != null) { + fields.scalar(name).setString(value); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipMessage.java new file mode 100644 index 00000000000..59cbaf63405 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipMessage.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.sip; + +import java.util.ArrayList; +import java.util.List; + +/** One SIP request or response. Absent headers are null. */ +public class SipMessage { + public boolean isRequest; + public String method; + public String requestUri; + public Integer statusCode; + public String reason; + public String from; + public String to; + public String callId; + public String cseq; + public String userAgent; + public String contact; + /** Every Via value, from all Via headers, in order. */ + public final List via = new ArrayList<>(); + public String contentType; + public Long contentLength; + /** Username of a Digest Authorization or Proxy-Authorization header. */ + public String username; + /** Always false: SIP Digest never carries the password itself. */ + public boolean passwordPresent; + /** Name (as sent) and value of each header, in order, up to the cap. */ + public final List headers = new ArrayList<>(); +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipParser.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipParser.java new file mode 100644 index 00000000000..7c9457daf42 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/sip/SipParser.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.sip; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; + +/** + * Parses the start line and headers of a SIP message (RFC 3261), including compact header + * names and folded header lines. The body is not returned. + */ +public final class SipParser { + static final int MAX_HEADERS = 64; + static final int MAX_STRING = 4096; + private static final Pattern REQUEST = Pattern.compile( + "(INVITE|ACK|BYE|CANCEL|REGISTER|OPTIONS|PRACK|SUBSCRIBE|NOTIFY|PUBLISH|INFO|REFER|MESSAGE|UPDATE) (\\S+) SIP/2\\.0"); + private static final Pattern STATUS = Pattern.compile("SIP/2\\.0 ([1-6]\\d\\d)(?: (.*))?"); + private static final Pattern HEADER = Pattern.compile("([!%'*+.^_`~0-9A-Za-z-]+)[ \\t]*:[ \\t]*(.*?)[ \\t]*"); + private static final Pattern USERNAME = + Pattern.compile("(?:^|[\\s,])username[ \\t]*=[ \\t]*(?:\"((?:[^\"\\\\]|\\\\.)*)\"|([^\\s,]+))", + Pattern.CASE_INSENSITIVE); + private static final Map COMPACT = new HashMap<>(); + + static { + COMPACT.put("i", "call-id"); + COMPACT.put("m", "contact"); + COMPACT.put("e", "content-encoding"); + COMPACT.put("l", "content-length"); + COMPACT.put("c", "content-type"); + COMPACT.put("f", "from"); + COMPACT.put("s", "subject"); + COMPACT.put("k", "supported"); + COMPACT.put("t", "to"); + COMPACT.put("v", "via"); + } + + private SipParser() { } + + /** + * @return the message, or null if the data does not start with a SIP request or status line + * @throws IllegalArgumentException if a header line is malformed + */ + public static SipMessage parse(byte[] data, DecoderContext context) { + if (data == null || data.length == 0) { + return null; + } + String text = new String(data, StandardCharsets.UTF_8); + int lineEnd = text.indexOf('\n'); + String startLine = stripCr(lineEnd < 0 ? text : text.substring(0, lineEnd)); + SipMessage m = new SipMessage(); + Matcher request = REQUEST.matcher(startLine); + Matcher status = STATUS.matcher(startLine); + if (request.matches()) { + m.isRequest = true; + m.method = request.group(1); + m.requestUri = cap(request.group(2)); + } else if (status.matches()) { + m.statusCode = Integer.parseInt(status.group(1)); + m.reason = status.group(2) == null ? null : cap(status.group(2)); + } else { + return null; + } + State state = new State(m, context); + int lineNumber = 0; + int p = lineEnd + 1; + while (lineEnd >= 0 && p < text.length()) { + lineEnd = text.indexOf('\n', p); + String line = stripCr(lineEnd < 0 ? text.substring(p) : text.substring(p, lineEnd)); + p = lineEnd + 1; + lineNumber++; + if (line.isEmpty()) { + break; + } + if ((line.charAt(0) == ' ' || line.charAt(0) == '\t') && state.name != null) { + state.value.append(' ').append(line.trim()); + continue; + } + Matcher header = HEADER.matcher(line); + if (!header.matches()) { + if (lineEnd < 0) { + break; // the capture cut the last line short + } + throw new IllegalArgumentException("malformed header line " + lineNumber); + } + state.finish(); + state.name = header.group(1); + state.value.append(header.group(2)); + } + state.finish(); + return m; + } + + /** The header being read, which folded lines may still extend. */ + private static final class State { + final SipMessage m; + final DecoderContext context; + final StringBuilder value = new StringBuilder(); + String name; + boolean headersCapped; + boolean viaCapped; + + State(SipMessage m, DecoderContext context) { + this.m = m; + this.context = context; + } + + void finish() { + if (name == null) { + return; + } + String v = cap(value.toString()); + if (m.headers.size() < MAX_HEADERS) { + m.headers.add(new String[] {name, v}); + } else if (!headersCapped) { + context.warn("headers truncated to " + MAX_HEADERS); + headersCapped = true; + } + field(name.toLowerCase(), v); + name = null; + value.setLength(0); + } + + private void field(String lower, String v) { + String canonical = COMPACT.getOrDefault(lower, lower); + switch (canonical) { + case "from": + m.from = m.from == null ? v : m.from; + break; + case "to": + m.to = m.to == null ? v : m.to; + break; + case "call-id": + m.callId = m.callId == null ? v : m.callId; + break; + case "cseq": + m.cseq = m.cseq == null ? v : m.cseq; + break; + case "user-agent": + m.userAgent = m.userAgent == null ? v : m.userAgent; + break; + case "contact": + m.contact = m.contact == null ? v : m.contact; + break; + case "content-type": + m.contentType = m.contentType == null ? v : m.contentType; + break; + case "content-length": + contentLength(v); + break; + case "via": + via(v); + break; + case "authorization": + case "proxy-authorization": + username(v); + break; + default: + break; + } + } + + private void contentLength(String v) { + if (m.contentLength != null) { + return; + } + try { + long length = Long.parseLong(v); + if (length >= 0) { + m.contentLength = length; + return; + } + } catch (NumberFormatException e) { + // reported below + } + context.warn("invalid Content-Length " + (v.length() > 32 ? v.substring(0, 32) : v)); + } + + /** Splits a Via header on commas outside quotes and angle brackets. */ + private void via(String v) { + boolean quoted = false; + int depth = 0; + int start = 0; + for (int i = 0; i <= v.length(); i++) { + char c = i < v.length() ? v.charAt(i) : ','; + if (c == '"') { + quoted = !quoted; + } else if (!quoted && c == '<') { + depth++; + } else if (!quoted && c == '>' && depth > 0) { + depth--; + } else if ((!quoted && depth == 0 && c == ',') || i == v.length()) { + String part = v.substring(start, i).trim(); + start = i + 1; + if (part.isEmpty()) { + continue; + } + if (m.via.size() < MAX_HEADERS) { + m.via.add(part); + } else if (!viaCapped) { + context.warn("via truncated to " + MAX_HEADERS); + viaCapped = true; + } + } + } + } + + private void username(String v) { + if (m.username != null || !v.regionMatches(true, 0, "Digest", 0, 6)) { + return; + } + Matcher u = USERNAME.matcher(v.substring(6)); + if (u.find()) { + m.username = cap(u.group(1) != null ? u.group(1).replaceAll("\\\\(.)", "$1") : u.group(2)); + m.passwordPresent = false; + } + } + } + + private static String stripCr(String line) { + return line.endsWith("\r") ? line.substring(0, line.length() - 1) : line; + } + + static String cap(String s) { + return s.length() > MAX_STRING ? s.substring(0, MAX_STRING) : s; + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpDecoder.java new file mode 100644 index 00000000000..a2c242a7cd1 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpDecoder.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.ssdp; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** SSDP (UPnP discovery) over UDP port 1900. */ +public class SsdpDecoder implements PacketProtocolDecoder { + private static final int PORT = 1900; + + @Override + public String protocol() { + return "ssdp"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("method", MinorType.VARCHAR) + .addNullable("status_code", MinorType.INT) + .addNullable("st", MinorType.VARCHAR) + .addNullable("nt", MinorType.VARCHAR) + .addNullable("nts", MinorType.VARCHAR) + .addNullable("usn", MinorType.VARCHAR) + .addNullable("location", MinorType.VARCHAR) + .addNullable("server", MinorType.VARCHAR) + .addNullable("user_agent", MinorType.VARCHAR) + .addNullable("man", MinorType.VARCHAR) + .addNullable("mx", MinorType.INT) + .addNullable("cache_control", MinorType.VARCHAR) + .addMapArray("headers") + .addNullable("name", MinorType.VARCHAR) + .addNullable("value", MinorType.VARCHAR) + .resumeSchema(); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isUdpPacket() && (packet.getSrc_port() == PORT || packet.getDst_port() == PORT); + } + + @Override + public SsdpMessage parse(Packet packet, byte[] payload, DecoderContext context) { + return SsdpParser.parse(payload, context); + } + + @Override + public void write(SsdpMessage m, TupleWriter fields) { + setString(fields, "method", m.method); + if (m.statusCode != null) { + fields.scalar("status_code").setInt(m.statusCode); + } + setString(fields, "st", m.st); + setString(fields, "nt", m.nt); + setString(fields, "nts", m.nts); + setString(fields, "usn", m.usn); + setString(fields, "location", m.location); + setString(fields, "server", m.server); + setString(fields, "user_agent", m.userAgent); + setString(fields, "man", m.man); + if (m.mx != null) { + fields.scalar("mx").setInt(m.mx); + } + setString(fields, "cache_control", m.cacheControl); + ArrayWriter headers = fields.array("headers"); + for (String[] h : m.headers) { + TupleWriter t = headers.tuple(); + t.scalar("name").setString(h[0]); + t.scalar("value").setString(h[1]); + headers.save(); + } + } + + private static void setString(TupleWriter fields, String name, String value) { + if (value != null) { + fields.scalar(name).setString(value); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpMessage.java new file mode 100644 index 00000000000..bbc79a41123 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpMessage.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.ssdp; + +import java.util.ArrayList; +import java.util.List; + +/** One SSDP message: an M-SEARCH or NOTIFY request, or a search response. */ +public class SsdpMessage { + /** M-SEARCH or NOTIFY; null for a response. */ + public String method; + /** Status code of a response; null for a request. */ + public Integer statusCode; + public String st; + public String nt; + public String nts; + public String usn; + public String location; + public String server; + public String userAgent; + public String man; + public Integer mx; + public String cacheControl; + /** Name and value of each header, in order, up to the cap. */ + public final List headers = new ArrayList<>(); +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpParser.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpParser.java new file mode 100644 index 00000000000..c30cbf7d7b8 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ssdp/SsdpParser.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.ssdp; + +import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; + +/** + * Parses SSDP (HTTP over UDP, as used by UPnP discovery). Lines may end with CRLF or a bare LF, + * which some devices send. + */ +public final class SsdpParser { + static final int MAX_HEADERS = 64; + static final int MAX_STRING = 4096; + private static final Pattern REQUEST = Pattern.compile("(M-SEARCH|NOTIFY) \\* HTTP/1\\.[01]"); + private static final Pattern STATUS = Pattern.compile("HTTP/1\\.[01] ([1-5]\\d\\d)(?: .*)?"); + private static final Pattern HEADER = Pattern.compile("([!#$%&'*+.^_`|~0-9A-Za-z-]+):[ \\t]*(.*?)[ \\t]*"); + + private SsdpParser() { } + + /** + * @return the message, or null if the data does not start with an SSDP start line + * @throws IllegalArgumentException if a header line is malformed + */ + public static SsdpMessage parse(byte[] data, DecoderContext context) { + if (data == null || data.length == 0) { + return null; + } + String text = new String(data, StandardCharsets.UTF_8); + int lineEnd = text.indexOf('\n'); + String startLine = stripCr(lineEnd < 0 ? text : text.substring(0, lineEnd)); + SsdpMessage m = new SsdpMessage(); + Matcher request = REQUEST.matcher(startLine); + Matcher status = STATUS.matcher(startLine); + if (request.matches()) { + m.method = request.group(1); + } else if (status.matches()) { + m.statusCode = Integer.parseInt(status.group(1)); + } else { + return null; + } + boolean capped = false; + int lineNumber = 0; + int p = lineEnd + 1; + while (lineEnd >= 0 && p < text.length()) { + lineEnd = text.indexOf('\n', p); + String line = stripCr(lineEnd < 0 ? text.substring(p) : text.substring(p, lineEnd)); + p = lineEnd + 1; + lineNumber++; + if (line.isEmpty()) { + break; + } + Matcher header = HEADER.matcher(line); + if (!header.matches()) { + if (lineEnd < 0) { + break; // the capture cut the last line short + } + throw new IllegalArgumentException("malformed header line " + lineNumber); + } + String name = header.group(1); + String value = cap(header.group(2)); + if (m.headers.size() < MAX_HEADERS) { + m.headers.add(new String[] {name, value}); + } else if (!capped) { + context.warn("headers truncated to " + MAX_HEADERS); + capped = true; + } + field(m, name, value, context); + } + return m; + } + + /** Sets the named field for a header, keeping the first occurrence. */ + private static void field(SsdpMessage m, String name, String value, DecoderContext context) { + switch (name.toUpperCase()) { + case "ST": + m.st = m.st == null ? value : m.st; + break; + case "NT": + m.nt = m.nt == null ? value : m.nt; + break; + case "NTS": + m.nts = m.nts == null ? value : m.nts; + break; + case "USN": + m.usn = m.usn == null ? value : m.usn; + break; + case "LOCATION": + m.location = m.location == null ? value : m.location; + break; + case "SERVER": + m.server = m.server == null ? value : m.server; + break; + case "USER-AGENT": + m.userAgent = m.userAgent == null ? value : m.userAgent; + break; + case "MAN": + m.man = m.man == null ? value : m.man; + break; + case "CACHE-CONTROL": + m.cacheControl = m.cacheControl == null ? value : m.cacheControl; + break; + case "MX": + if (m.mx == null) { + try { + m.mx = Integer.parseInt(value); + } catch (NumberFormatException e) { + context.warn("invalid MX " + (value.length() > 32 ? value.substring(0, 32) : value)); + } + } + break; + default: + break; + } + } + + private static String stripCr(String line) { + return line.endsWith("\r") ? line.substring(0, line.length() - 1) : line; + } + + private static String cap(String s) { + return s.length() > MAX_STRING ? s.substring(0, MAX_STRING) : s; + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogDecoder.java new file mode 100644 index 00000000000..a19ed0cc9e8 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogDecoder.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.syslog; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** Syslog over UDP port 514. */ +public class SyslogDecoder implements PacketProtocolDecoder { + private static final int PORT = 514; + + @Override + public String protocol() { + return "syslog"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("facility", MinorType.INT) + .addNullable("facility_name", MinorType.VARCHAR) + .addNullable("severity", MinorType.INT) + .addNullable("severity_name", MinorType.VARCHAR) + .addNullable("version", MinorType.INT) + .addNullable("timestamp_text", MinorType.VARCHAR) + .addNullable("hostname", MinorType.VARCHAR) + .addNullable("app_name", MinorType.VARCHAR) + .addNullable("proc_id", MinorType.VARCHAR) + .addNullable("msg_id", MinorType.VARCHAR) + .addNullable("structured_data", MinorType.VARCHAR) + .addNullable("message", MinorType.VARCHAR); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isUdpPacket() && (packet.getSrc_port() == PORT || packet.getDst_port() == PORT); + } + + @Override + public SyslogMessage parse(Packet packet, byte[] payload, DecoderContext context) { + return SyslogParser.parse(payload, context); + } + + @Override + public void write(SyslogMessage m, TupleWriter fields) { + fields.scalar("facility").setInt(m.facility); + fields.scalar("facility_name").setString(m.facilityName); + fields.scalar("severity").setInt(m.severity); + fields.scalar("severity_name").setString(m.severityName); + if (m.version != null) { + fields.scalar("version").setInt(m.version); + } + setString(fields, "timestamp_text", m.timestampText); + setString(fields, "hostname", m.hostname); + setString(fields, "app_name", m.appName); + setString(fields, "proc_id", m.procId); + setString(fields, "msg_id", m.msgId); + setString(fields, "structured_data", m.structuredData); + setString(fields, "message", m.message); + } + + private static void setString(TupleWriter fields, String name, String value) { + if (value != null) { + fields.scalar(name).setString(value); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogMessage.java new file mode 100644 index 00000000000..9a4aada5d9b --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogMessage.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.syslog; + +/** One syslog message. Fields the message does not carry are null. */ +public class SyslogMessage { + public int facility; + public String facilityName; + public int severity; + public String severityName; + /** Protocol version of an RFC 5424 message; null for RFC 3164 and bare messages. */ + public Integer version; + public String timestampText; + public String hostname; + public String appName; + public String procId; + public String msgId; + public String structuredData; + public String message; +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogParser.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogParser.java new file mode 100644 index 00000000000..51d9bb7962d --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/syslog/SyslogParser.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.syslog; + +import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; + +/** + * Parses syslog messages: RFC 5424 ({@code 1 TIMESTAMP HOST APP PROCID MSGID SD MSG}), + * RFC 3164 ({@code Mmm dd hh:mm:ss HOST TAG: MSG}), and messages that have only a priority. + */ +public final class SyslogParser { + static final int MAX_STRING = 4096; + private static final int MAX_PRI = 191; + private static final String[] FACILITIES = { + "kern", "user", "mail", "daemon", "auth", "syslog", "lpr", "news", "uucp", "cron", "authpriv", "ftp", "ntp", + "audit", "alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7"}; + private static final String[] SEVERITIES = {"emerg", "alert", "crit", "err", "warning", "notice", "info", "debug"}; + private static final Pattern VERSION = Pattern.compile("([1-9]\\d?) "); + private static final Pattern BSD_TIMESTAMP = Pattern.compile("[A-Z][a-z]{2} [ \\d]\\d \\d\\d:\\d\\d:\\d\\d "); + private static final Pattern TAG = Pattern.compile("([^\\s\\[\\]:]{1,48})(?:\\[([^\\]\\s]{0,32})\\])?:(?: ?(.*))?", + Pattern.DOTALL); + private static final int BSD_TIMESTAMP_LENGTH = 15; + private static final int RFC5424_HEADER_FIELDS = 5; + + private SyslogParser() { } + + /** + * @return the message, or null if the data does not start with a syslog priority + * @throws IllegalArgumentException if an RFC 5424 header is incomplete or its structured data is invalid + */ + public static SyslogMessage parse(byte[] data, DecoderContext context) { + if (data == null || data.length < 3 || data[0] != '<') { + return null; + } + int pri = 0; + int i = 1; + while (i < data.length && i <= 3 && data[i] >= '0' && data[i] <= '9') { + pri = pri * 10 + (data[i] - '0'); + i++; + } + if (i == 1 || i >= data.length || data[i] != '>' || pri > MAX_PRI) { + return null; + } + SyslogMessage m = new SyslogMessage(); + m.facility = pri >> 3; + m.facilityName = FACILITIES[m.facility]; + m.severity = pri & 0x07; + m.severityName = SEVERITIES[m.severity]; + String rest = stripEnd(new String(data, i + 1, data.length - i - 1, StandardCharsets.UTF_8)); + Matcher version = VERSION.matcher(rest); + if (version.lookingAt()) { + m.version = Integer.parseInt(version.group(1)); + parse5424(rest.substring(version.end()), m, context); + } else if (BSD_TIMESTAMP.matcher(rest).lookingAt()) { + m.timestampText = rest.substring(0, BSD_TIMESTAMP_LENGTH); + parse3164(rest.substring(BSD_TIMESTAMP_LENGTH + 1), m, context); + } else { + m.message = message(rest, context); + } + return m; + } + + private static void parse5424(String s, SyslogMessage m, DecoderContext context) { + String[] header = new String[RFC5424_HEADER_FIELDS]; + int p = 0; + for (int f = 0; f < RFC5424_HEADER_FIELDS; f++) { + int space = s.indexOf(' ', p); + if (space < 0) { + throw new IllegalArgumentException("truncated RFC 5424 header"); + } + header[f] = nil(s.substring(p, space), context); + p = space + 1; + } + m.timestampText = header[0]; + m.hostname = header[1]; + m.appName = header[2]; + m.procId = header[3]; + m.msgId = header[4]; + if (p >= s.length()) { + throw new IllegalArgumentException("truncated RFC 5424 header: no structured data"); + } + int sdStart = p; + if (s.charAt(p) == '-') { + p++; + } else { + while (p < s.length() && s.charAt(p) == '[') { + p = elementEnd(s, p) + 1; + } + if (p == sdStart) { + throw new IllegalArgumentException("invalid structured data"); + } + m.structuredData = cap(s.substring(sdStart, p), "structured data", context); + } + if (p < s.length()) { + if (s.charAt(p) != ' ') { + throw new IllegalArgumentException("invalid structured data"); + } + m.message = message(s.substring(p + 1), context); + } + } + + /** Index of the ']' closing the SD-ELEMENT that starts at start, skipping quoted and escaped characters. */ + private static int elementEnd(String s, int start) { + boolean quoted = false; + for (int j = start + 1; j < s.length(); j++) { + char c = s.charAt(j); + if (quoted && c == '\\') { + j++; + } else if (c == '"') { + quoted = !quoted; + } else if (!quoted && c == ']') { + return j; + } + } + throw new IllegalArgumentException("unterminated structured data"); + } + + private static void parse3164(String s, SyslogMessage m, DecoderContext context) { + int space = s.indexOf(' '); + if (space > 0 && !TAG.matcher(s.substring(0, space)).matches()) { + m.hostname = cap(s.substring(0, space), "hostname", context); + s = s.substring(space + 1); + } + Matcher tag = TAG.matcher(s); + if (tag.matches()) { + m.appName = tag.group(1); + m.procId = tag.group(2) == null || tag.group(2).isEmpty() ? null : tag.group(2); + m.message = tag.group(3) == null ? null : message(tag.group(3), context); + } else { + m.message = message(s, context); + } + } + + private static String nil(String value, DecoderContext context) { + return value.equals("-") || value.isEmpty() ? null : cap(value, "header field", context); + } + + private static String message(String s, DecoderContext context) { + if (s.startsWith("\uFEFF")) { + s = s.substring(1); + } + return s.isEmpty() ? null : cap(s, "message", context); + } + + private static String stripEnd(String s) { + int end = s.length(); + while (end > 0 && (s.charAt(end - 1) == '\n' || s.charAt(end - 1) == '\r' || s.charAt(end - 1) == '\0')) { + end--; + } + return s.substring(0, end); + } + + private static String cap(String s, String what, DecoderContext context) { + if (s.length() <= MAX_STRING) { + return s; + } + context.warn(what + " truncated to " + MAX_STRING + " characters"); + return s.substring(0, MAX_STRING); + } +} diff --git a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder index 5b788645c0f..77ff99c2ffd 100644 --- a/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder +++ b/contrib/format-pcapng/src/main/resources/META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder @@ -17,3 +17,7 @@ # org.apache.drill.exec.store.pcap.protocol.dns.DnsDecoder org.apache.drill.exec.store.pcap.protocol.http.HttpPacketDecoder +org.apache.drill.exec.store.pcap.protocol.ntp.NtpDecoder +org.apache.drill.exec.store.pcap.protocol.syslog.SyslogDecoder +org.apache.drill.exec.store.pcap.protocol.ssdp.SsdpDecoder +org.apache.drill.exec.store.pcap.protocol.sip.SipDecoder diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestServiceDecoderQueries.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestServiceDecoderQueries.java new file mode 100644 index 00000000000..85270767989 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/TestServiceDecoderQueries.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.nio.file.Paths; +import java.time.Instant; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.physical.rowSet.RowSet; +import org.apache.drill.exec.physical.rowSet.RowSetBuilder; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.ClusterTest; +import org.apache.drill.test.rowSet.RowSetComparison; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Query tests of the NTP, syslog, SSDP and SIP decoders on the fixtures under decoders/. */ +public class TestServiceDecoderQueries extends ClusterTest { + + @BeforeClass + public static void setup() throws Exception { + ClusterTest.startCluster(ClusterFixture.builder(dirTestWatcher)); + dirTestWatcher.copyResourceToRoot(Paths.get("decoders/")); + } + + @Test + public void testNtp() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.ntp.mode as mode, t.parsed_data.ntp.stratum as stratum, " + + "t.parsed_data.ntp.reference_id as reference_id, t.parsed_data.ntp.transmit_time as transmit_time, " + + "t.parsed_data.ntp.request_code as request_code, decode_error from dfs.`decoders/ntp/ntp.pcapng` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("mode", MinorType.VARCHAR) + .addNullable("stratum", MinorType.INT) + .addNullable("reference_id", MinorType.VARCHAR) + .addNullable("transmit_time", MinorType.TIMESTAMP) + .addNullable("request_code", MinorType.INT) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("ntp", "client", 0, null, Instant.ofEpochMilli(1704164645500L), null, null) + .addRow("ntp", "server", 2, "192.168.1.1", Instant.ofEpochMilli(1704164646500L), null, null) + .addRow(null, null, null, null, null, null, null) + .addRow("ntp", "private", null, null, null, 42, null) + .addRow("ntp", null, null, null, null, null, "ntp: truncated control message: count 100 exceeds 0 data bytes") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testSyslog() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.syslog.facility_name as facility, " + + "t.parsed_data.syslog.severity as severity, t.parsed_data.syslog.version as version, " + + "t.parsed_data.syslog.hostname as hostname, t.parsed_data.syslog.app_name as app_name, " + + "t.parsed_data.syslog.message as message, decode_error from dfs.`decoders/syslog/syslog.pcapng` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("facility", MinorType.VARCHAR) + .addNullable("severity", MinorType.INT) + .addNullable("version", MinorType.INT) + .addNullable("hostname", MinorType.VARCHAR) + .addNullable("app_name", MinorType.VARCHAR) + .addNullable("message", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("syslog", "local4", 5, 1, "mymachine.example.com", "evntslog", "An application event", null) + .addRow("syslog", "auth", 6, null, "gateway", "sshd", "Accepted password for root from 10.0.0.9", null) + .addRow(null, null, null, null, null, null, null, null) + .addRow("syslog", null, null, null, null, null, null, "syslog: truncated RFC 5424 header") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testSsdp() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.ssdp.method as method, " + + "t.parsed_data.ssdp.status_code as status_code, t.parsed_data.ssdp.st as st, t.parsed_data.ssdp.nts as nts, " + + "t.parsed_data.ssdp.server as server, t.parsed_data.ssdp.mx as mx, " + + "t.parsed_data.ssdp.headers[0].name as first_header, decode_error from dfs.`decoders/ssdp/ssdp.pcapng` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("method", MinorType.VARCHAR) + .addNullable("status_code", MinorType.INT) + .addNullable("st", MinorType.VARCHAR) + .addNullable("nts", MinorType.VARCHAR) + .addNullable("server", MinorType.VARCHAR) + .addNullable("mx", MinorType.INT) + .addNullable("first_header", MinorType.VARCHAR) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("ssdp", "M-SEARCH", null, "ssdp:all", null, null, 2, "HOST", null) + .addRow("ssdp", null, 200, "upnp:rootdevice", null, "Linux/3.14 UPnP/1.0 IpBridge/1.26.0", null, + "CACHE-CONTROL", null) + .addRow("ssdp", "NOTIFY", null, null, "ssdp:byebye", null, null, "HOST", null) + .addRow(null, null, null, null, null, null, null, null, null) + .addRow("ssdp", null, null, null, null, null, null, null, "ssdp: malformed header line 2") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } + + @Test + public void testSip() throws Exception { + String sql = "select parsed_protocol, t.parsed_data.sip.method as method, " + + "t.parsed_data.sip.status_code as status_code, t.parsed_data.sip.call_id as call_id, " + + "t.parsed_data.sip.to_address as to_address, " + + "t.parsed_data.sip.via[0] as via, t.parsed_data.sip.username as username, " + + "t.parsed_data.sip.content_length as content_length, decode_error from dfs.`decoders/sip/sip.pcapng` t"; + RowSet results = client.queryBuilder().sql(sql).rowSet(); + TupleMetadata schema = new SchemaBuilder() + .addNullable("parsed_protocol", MinorType.VARCHAR) + .addNullable("method", MinorType.VARCHAR) + .addNullable("status_code", MinorType.INT) + .addNullable("call_id", MinorType.VARCHAR) + .addNullable("to_address", MinorType.VARCHAR) + .addNullable("via", MinorType.VARCHAR) + .addNullable("username", MinorType.VARCHAR) + .addNullable("content_length", MinorType.BIGINT) + .addNullable("decode_error", MinorType.VARCHAR) + .buildSchema(); + RowSet expected = new RowSetBuilder(client.allocator(), schema) + .addRow("sip", "INVITE", null, "a84b4c76e66710@pc33.atlanta.example.com", "Bob ", + "SIP/2.0/UDP pc33.atlanta.example.com;branch=z9hG4bK776asdhds", "alice", 4L, null) + .addRow("sip", null, 180, "a84b4c76e66710@pc33.atlanta.example.com", "Bob ;tag=a6c85cf", + "SIP/2.0/TCP pc33.atlanta.example.com;branch=z9hG4bK776asdhds", null, 0L, null) + .addRow(null, null, null, null, null, null, null, null, null) + .addRow("sip", null, null, null, null, null, null, null, "sip: malformed header line 2") + .build(); + new RowSetComparison(expected).verifyAndClearAll(results); + } +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/ntp/TestNtpDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/ntp/TestNtpDecoder.java new file mode 100644 index 00000000000..3ea1c0b5152 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/ntp/TestNtpDecoder.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.ntp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.TestPackets; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestNtpDecoder extends BaseTest { + private static final long NTP_UNIX_OFFSET = 2208988800L; + + private static final class Context implements DecoderContext { + final List warnings = new ArrayList<>(); + + @Override + public boolean exposeCredentials() { + return false; + } + + @Override + public void warn(String message) { + warnings.add(message); + } + } + + /** A 48-byte header; timestamps are Unix seconds plus half a second, 0 meaning a zero timestamp. */ + static byte[] header(int first, int stratum, int poll, int precision, int rootDelay, int rootDispersion, + byte[] refId, long ref, long origin, long receive, long transmit) { + ByteBuffer b = ByteBuffer.allocate(48).put((byte) first).put((byte) stratum).put((byte) poll).put((byte) precision) + .putInt(rootDelay).putInt(rootDispersion).put(refId); + for (long t : new long[] {ref, origin, receive, transmit}) { + if (t == 0) { + b.putLong(0); + } else { + b.putInt((int) (t + NTP_UNIX_OFFSET)).putInt(0x80000000); + } + } + return b.array(); + } + + @Test + public void testServerResponse() { + byte[] data = header(0x24, 2, 6, -20, 0x0800, 0x00010000, new byte[] {(byte) 192, (byte) 168, 1, 1}, + 1704164000L, 1704164645L, 1704164645L, 1704164646L); + NtpMessage m = NtpParser.parse(data, new Context()); + assertEquals(0, (int) m.leapIndicator); + assertEquals(4, m.version); + assertEquals("server", m.mode); + assertEquals(2, (int) m.stratum); + assertEquals(6, (int) m.poll); + assertEquals(-20, (int) m.precision); + assertEquals(0.03125, m.rootDelay, 1e-9); + assertEquals(1.0, m.rootDispersion, 1e-9); + assertEquals("192.168.1.1", m.referenceId); + assertEquals(Instant.ofEpochMilli(1704164000500L), m.referenceTime); + assertEquals(Instant.ofEpochMilli(1704164645500L), m.originTime); + assertEquals(Instant.ofEpochMilli(1704164646500L), m.transmitTime); + } + + @Test + public void testClientRequestHasNullTimestamps() { + byte[] data = header(0xE3, 0, 3, -6, 0, 0, new byte[4], 0, 0, 0, 1704164645L); + NtpMessage m = NtpParser.parse(data, new Context()); + assertEquals(3, (int) m.leapIndicator); + assertEquals("client", m.mode); + assertNull(m.referenceId); + assertNull(m.referenceTime); + assertNull(m.originTime); + assertNull(m.receiveTime); + assertEquals(Instant.ofEpochMilli(1704164645500L), m.transmitTime); + } + + @Test + public void testKissCodeAndRefId() { + assertEquals("RATE", NtpParser.parse(header(0x24, 0, 6, -20, 0, 0, "RATE".getBytes(StandardCharsets.US_ASCII), + 0, 0, 0, 1704164645L), new Context()).referenceId); + assertEquals("GPS", NtpParser.parse(header(0x24, 1, 6, -20, 0, 0, new byte[] {'G', 'P', 'S', 0}, + 0, 0, 0, 1704164645L), new Context()).referenceId); + } + + @Test + public void testExtensionsAreIgnored() { + byte[] data = ByteBuffer.allocate(68).put(header(0x23, 0, 6, -20, 0, 0, new byte[4], 0, 0, 0, 1704164645L)).array(); + assertEquals("client", NtpParser.parse(data, new Context()).mode); + } + + @Test + public void testNotNtp() { + // Version 0, version 5, too short, reserved mode 0 + assertNull(NtpParser.parse(header(0x03, 2, 6, -20, 0, 0, new byte[4], 0, 0, 0, 1L), new Context())); + assertNull(NtpParser.parse(header(0x2B, 2, 6, -20, 0, 0, new byte[4], 0, 0, 0, 1L), new Context())); + assertNull(NtpParser.parse(new byte[47], new Context())); + assertNull(NtpParser.parse(new byte[] {0x23, 0, 0}, new Context())); + assertNull(NtpParser.parse(header(0x20, 2, 6, -20, 0, 0, new byte[4], 0, 0, 0, 1L), new Context())); + assertNull(NtpParser.parse("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n123456789".getBytes(), new Context())); + } + + @Test + public void testPrivateModeMonlistRequest() { + // ntpdc monlist: response bit 0, version 2, mode 7, implementation 3, request code 42 + byte[] data = ByteBuffer.allocate(8).put((byte) 0x17).put((byte) 0).put((byte) 3).put((byte) 42) + .putShort((short) 0).putShort((short) 0).array(); + NtpMessage m = NtpParser.parse(data, new Context()); + assertEquals("private", m.mode); + assertEquals(2, m.version); + assertEquals(42, (int) m.requestCode); + assertNull(m.stratum); + } + + @Test + public void testControlModeTruncatedIsMalformed() { + // Mode 6 header claims 100 data bytes but carries none + byte[] data = ByteBuffer.allocate(12).put((byte) 0x16).put((byte) 2).putShort((short) 1).putShort((short) 0) + .putShort((short) 0).putShort((short) 0).putShort((short) 100).array(); + try { + NtpParser.parse(data, new Context()); + fail("expected malformed NTP"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("truncated")); + } + } + + @Test + public void testDecoderAccepts() { + NtpDecoder decoder = new NtpDecoder(); + byte[] data = header(0x23, 0, 6, -20, 0, 0, new byte[4], 0, 0, 0, 1L); + assertTrue(decoder.accepts(TestPackets.udp("10.0.0.1", 40000, "10.0.0.2", 123, data))); + assertTrue(!decoder.accepts(TestPackets.udp("10.0.0.1", 40000, "10.0.0.2", 124, data))); + assertTrue(!decoder.accepts(TestPackets.tcp("10.0.0.1", 40000, "10.0.0.2", 123, 1, TestPackets.ACK, data))); + } +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/sip/TestSipDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/sip/TestSipDecoder.java new file mode 100644 index 00000000000..32bef4ff8f0 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/sip/TestSipDecoder.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.sip; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.TestPackets; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestSipDecoder extends BaseTest { + + private static final class Context implements DecoderContext { + final List warnings = new ArrayList<>(); + + @Override + public boolean exposeCredentials() { + return true; + } + + @Override + public void warn(String message) { + warnings.add(message); + } + } + + private static SipMessage parse(String text, Context context) { + return SipParser.parse(text.getBytes(StandardCharsets.UTF_8), context); + } + + static final String INVITE = "INVITE sip:bob@biloxi.example.com SIP/2.0\r\n" + + "Via: SIP/2.0/UDP pc33.atlanta.example.com;branch=z9hG4bK776asdhds, SIP/2.0/UDP proxy.example.com;branch=z9hG4bK1\r\n" + + "Via: SIP/2.0/UDP edge.example.com;branch=z9hG4bK2\r\n" + + "Max-Forwards: 70\r\n" + + "To: Bob \r\n" + + "From: \"Alice, A.\" ;tag=1928301774\r\n" + + "Call-ID: a84b4c76e66710@pc33.atlanta.example.com\r\n" + + "CSeq: 314159 INVITE\r\n" + + "Contact: \r\n" + + "User-Agent: softphone/1.0\r\n" + + "Authorization: Digest username=\"alice\", realm=\"atlanta.example.com\",\r\n" + + " nonce=\"84a4cc6f3082121f32b42a2187831a9e\", response=\"7587245234b3434cc3412213e5f113a5\"\r\n" + + "Content-Type: application/sdp\r\n" + + "Content-Length: 4\r\n" + + "\r\n" + + "v=0\n"; + + @Test + public void testInvite() { + SipMessage m = parse(INVITE, new Context()); + assertTrue(m.isRequest); + assertEquals("INVITE", m.method); + assertEquals("sip:bob@biloxi.example.com", m.requestUri); + assertNull(m.statusCode); + assertEquals("Bob ", m.to); + assertEquals("\"Alice, A.\" ;tag=1928301774", m.from); + assertEquals("a84b4c76e66710@pc33.atlanta.example.com", m.callId); + assertEquals("314159 INVITE", m.cseq); + assertEquals("", m.contact); + assertEquals("softphone/1.0", m.userAgent); + assertEquals(Arrays.asList("SIP/2.0/UDP pc33.atlanta.example.com;branch=z9hG4bK776asdhds", + "SIP/2.0/UDP proxy.example.com;branch=z9hG4bK1", "SIP/2.0/UDP edge.example.com;branch=z9hG4bK2"), m.via); + assertEquals("application/sdp", m.contentType); + assertEquals(4L, (long) m.contentLength); + assertEquals("alice", m.username); + assertFalse(m.passwordPresent); + assertEquals(12, m.headers.size()); + assertTrue(m.headers.get(9)[1].endsWith("response=\"7587245234b3434cc3412213e5f113a5\"")); + } + + @Test + public void testCompactResponse() { + SipMessage m = parse("SIP/2.0 180 Ringing\r\nv: SIP/2.0/UDP host;branch=z9hG4bK1\r\nf: ;tag=1\r\n" + + "t: ;tag=2\r\ni: abc@x\r\nCSeq: 1 INVITE\r\nm: \r\nc: text/plain\r\nl: 0\r\n\r\n", + new Context()); + assertFalse(m.isRequest); + assertNull(m.method); + assertEquals(180, (int) m.statusCode); + assertEquals("Ringing", m.reason); + assertEquals(";tag=1", m.from); + assertEquals(";tag=2", m.to); + assertEquals("abc@x", m.callId); + assertEquals("", m.contact); + assertEquals("text/plain", m.contentType); + assertEquals(0L, (long) m.contentLength); + assertEquals(Arrays.asList("SIP/2.0/UDP host;branch=z9hG4bK1"), m.via); + assertNull(m.username); + } + + @Test + public void testProxyAuthorizationAndBadContentLength() { + Context context = new Context(); + SipMessage m = parse("REGISTER sips:registrar.example.com SIP/2.0\r\nProxy-Authorization: Digest username=bob, realm=x\r\n" + + "Content-Length: lots\r\n\r\n", context); + assertEquals("REGISTER", m.method); + assertEquals("bob", m.username); + assertNull(m.contentLength); + assertEquals(1, context.warnings.size()); + } + + @Test + public void testNotSip() { + assertNull(parse("GET / HTTP/1.1\r\nHost: x\r\n\r\n", new Context())); + assertNull(parse("FOO sip:x SIP/2.0\r\n\r\n", new Context())); + assertNull(parse("INVITE sip:x SIP/3.0\r\n\r\n", new Context())); + assertNull(parse("\r\n\r\n", new Context())); + assertNull(SipParser.parse(new byte[0], new Context())); + } + + @Test + public void testMalformedHeader() { + try { + parse("OPTIONS sip:x@y SIP/2.0\r\nVia: SIP/2.0/UDP h\r\ngarbage\r\n\r\n", new Context()); + fail("expected malformed SIP"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("malformed header line 2")); + } + } + + @Test + public void testDecoderAccepts() { + SipDecoder decoder = new SipDecoder(); + byte[] data = INVITE.getBytes(StandardCharsets.UTF_8); + assertTrue(decoder.accepts(TestPackets.udp("10.0.0.1", 5060, "10.0.0.2", 5060, data))); + assertTrue(decoder.accepts(TestPackets.tcp("10.0.0.1", 40000, "10.0.0.2", 5060, 1, TestPackets.ACK, data))); + assertFalse(decoder.accepts(TestPackets.udp("10.0.0.1", 40000, "10.0.0.2", 5061, data))); + } +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/ssdp/TestSsdpDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/ssdp/TestSsdpDecoder.java new file mode 100644 index 00000000000..9fe85e870d0 --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/ssdp/TestSsdpDecoder.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.ssdp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.TestPackets; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestSsdpDecoder extends BaseTest { + + private static final class Context implements DecoderContext { + final List warnings = new ArrayList<>(); + + @Override + public boolean exposeCredentials() { + return false; + } + + @Override + public void warn(String message) { + warnings.add(message); + } + } + + private static SsdpMessage parse(String text, Context context) { + return SsdpParser.parse(text.getBytes(StandardCharsets.ISO_8859_1), context); + } + + @Test + public void testSearch() { + SsdpMessage m = parse("M-SEARCH * HTTP/1.1\r\nHOST: 239.255.255.250:1900\r\nMAN: \"ssdp:discover\"\r\n" + + "MX: 2\r\nST: ssdp:all\r\nUSER-AGENT: Linux/5 UPnP/1.1 test/1.0\r\n\r\n", new Context()); + assertEquals("M-SEARCH", m.method); + assertNull(m.statusCode); + assertEquals("\"ssdp:discover\"", m.man); + assertEquals(2, (int) m.mx); + assertEquals("ssdp:all", m.st); + assertEquals("Linux/5 UPnP/1.1 test/1.0", m.userAgent); + assertEquals(5, m.headers.size()); + assertEquals("HOST", m.headers.get(0)[0]); + } + + @Test + public void testNotifyWithBareLineFeeds() { + SsdpMessage m = parse("NOTIFY * HTTP/1.1\nHost: 239.255.255.250:1900\nCache-Control: max-age=1800\n" + + "Location: http://192.168.1.10:49152/desc.xml\nNT: upnp:rootdevice\nNTS: ssdp:alive\n" + + "Server: Linux/3.14 UPnP/1.0 IpBridge/1.26.0\nUSN: uuid:2f402f80-da50-11e1-9b23-001788255acc::upnp:rootdevice\n\n", + new Context()); + assertEquals("NOTIFY", m.method); + assertEquals("max-age=1800", m.cacheControl); + assertEquals("http://192.168.1.10:49152/desc.xml", m.location); + assertEquals("upnp:rootdevice", m.nt); + assertEquals("ssdp:alive", m.nts); + assertEquals("Linux/3.14 UPnP/1.0 IpBridge/1.26.0", m.server); + assertEquals("uuid:2f402f80-da50-11e1-9b23-001788255acc::upnp:rootdevice", m.usn); + } + + @Test + public void testResponse() { + Context context = new Context(); + SsdpMessage m = parse("HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=100\r\nEXT:\r\nST: upnp:rootdevice\r\nMX: x\r\n\r\n", + context); + assertNull(m.method); + assertEquals(200, (int) m.statusCode); + assertEquals("upnp:rootdevice", m.st); + assertEquals("", m.headers.get(1)[1]); + assertNull(m.mx); + assertEquals(1, context.warnings.size()); + } + + @Test + public void testHeaderCap() { + StringBuilder text = new StringBuilder("NOTIFY * HTTP/1.1\r\n"); + for (int i = 0; i < 70; i++) { + text.append("X-").append(i).append(": v\r\n"); + } + Context context = new Context(); + SsdpMessage m = parse(text.append("NT: last\r\n\r\n").toString(), context); + assertEquals(64, m.headers.size()); + assertEquals("last", m.nt); + assertEquals(1, context.warnings.size()); + } + + @Test + public void testNotSsdp() { + assertNull(parse("GET / HTTP/1.1\r\n\r\n", new Context())); + assertNull(parse("M-SEARCH /x HTTP/1.1\r\n\r\n", new Context())); + assertNull(parse("hello", new Context())); + assertNull(SsdpParser.parse(new byte[] {1, 2, 3, 4}, new Context())); + } + + @Test + public void testMalformedHeader() { + try { + parse("NOTIFY * HTTP/1.1\r\nNT: upnp:rootdevice\r\nthis is not a header\r\n\r\n", new Context()); + fail("expected malformed SSDP"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("malformed header line 2")); + } + } + + @Test + public void testDecoderAccepts() { + SsdpDecoder decoder = new SsdpDecoder(); + byte[] data = "NOTIFY * HTTP/1.1\r\n\r\n".getBytes(StandardCharsets.ISO_8859_1); + assertTrue(decoder.accepts(TestPackets.udp("10.0.0.1", 40000, "239.255.255.250", 1900, data))); + assertTrue(!decoder.accepts(TestPackets.udp("10.0.0.1", 40000, "10.0.0.2", 1901, data))); + assertTrue(!decoder.accepts(TestPackets.tcp("10.0.0.1", 40000, "10.0.0.2", 1900, 1, TestPackets.ACK, data))); + } +} diff --git a/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/syslog/TestSyslogDecoder.java b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/syslog/TestSyslogDecoder.java new file mode 100644 index 00000000000..a0b9548726d --- /dev/null +++ b/contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/syslog/TestSyslogDecoder.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.syslog; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.TestPackets; +import org.apache.drill.test.BaseTest; +import org.junit.Test; + +public class TestSyslogDecoder extends BaseTest { + + private static final class Context implements DecoderContext { + final List warnings = new ArrayList<>(); + + @Override + public boolean exposeCredentials() { + return false; + } + + @Override + public void warn(String message) { + warnings.add(message); + } + } + + private static SyslogMessage parse(String text) { + return SyslogParser.parse(text.getBytes(StandardCharsets.UTF_8), new Context()); + } + + @Test + public void testRfc5424() { + SyslogMessage m = parse("<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 " + + "[exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"] An application event\n"); + assertEquals(20, m.facility); + assertEquals("local4", m.facilityName); + assertEquals(5, m.severity); + assertEquals("notice", m.severityName); + assertEquals(1, (int) m.version); + assertEquals("2003-10-11T22:14:15.003Z", m.timestampText); + assertEquals("mymachine.example.com", m.hostname); + assertEquals("evntslog", m.appName); + assertNull(m.procId); + assertEquals("ID47", m.msgId); + assertEquals("[exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]", m.structuredData); + assertEquals("An application event", m.message); + } + + @Test + public void testRfc5424NilValuesAndBom() { + SyslogMessage m = parse("<34>1 - - su 77 - [a@1 x=\"q\\]\"][b@2] 'su root' failed"); + assertEquals(4, m.facility); + assertEquals("auth", m.facilityName); + assertEquals(2, m.severity); + assertEquals("crit", m.severityName); + assertNull(m.timestampText); + assertNull(m.hostname); + assertEquals("su", m.appName); + assertEquals("77", m.procId); + assertNull(m.msgId); + assertEquals("[a@1 x=\"q\\]\"][b@2]", m.structuredData); + assertEquals("'su root' failed", m.message); + + m = parse("<14>1 2024-01-02T03:04:05Z host app - - -"); + assertNull(m.structuredData); + assertNull(m.message); + } + + @Test + public void testRfc3164() { + SyslogMessage m = parse("<38>Jan 2 03:04:05 gateway sshd[1234]: Accepted password for root"); + assertEquals(4, m.facility); + assertEquals(6, m.severity); + assertNull(m.version); + assertEquals("Jan 2 03:04:05", m.timestampText); + assertEquals("gateway", m.hostname); + assertEquals("sshd", m.appName); + assertEquals("1234", m.procId); + assertEquals("Accepted password for root", m.message); + + // No hostname + m = parse("<13>Oct 11 22:14:15 su: 'su root' failed for lonvick on /dev/pts/8"); + assertNull(m.hostname); + assertEquals("su", m.appName); + assertNull(m.procId); + assertEquals("'su root' failed for lonvick on /dev/pts/8", m.message); + + // No tag + m = parse("<13>Oct 11 22:14:15 host free text"); + assertEquals("host", m.hostname); + assertNull(m.appName); + assertEquals("free text", m.message); + } + + @Test + public void testPriorityOnly() { + SyslogMessage m = parse("<0>kernel panic"); + assertEquals(0, m.facility); + assertEquals("kern", m.facilityName); + assertEquals("emerg", m.severityName); + assertNull(m.timestampText); + assertEquals("kernel panic", m.message); + } + + @Test + public void testNotSyslog() { + assertNull(parse("hello")); + assertNull(parse("<192>too high")); + assertNull(parse("<1234>four digits")); + assertNull(parse("<>empty")); + assertNull(parse("<12")); + assertNull(parse("letters")); + assertNull(SyslogParser.parse(new byte[0], new Context())); + } + + @Test + public void testRfc5424MissingFieldsIsMalformed() { + try { + parse("<14>1 2024-01-02T03:04:05Z host"); + fail("expected malformed syslog"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("truncated")); + } + try { + parse("<14>1 2024-01-02T03:04:05Z host app - - [unterminated x=\"1\""); + fail("expected malformed syslog"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("structured data")); + } + } + + @Test + public void testDecoderAccepts() { + SyslogDecoder decoder = new SyslogDecoder(); + byte[] data = "<13>hi".getBytes(StandardCharsets.UTF_8); + assertTrue(decoder.accepts(TestPackets.udp("10.0.0.1", 40000, "10.0.0.2", 514, data))); + assertTrue(!decoder.accepts(TestPackets.udp("10.0.0.1", 40000, "10.0.0.2", 515, data))); + assertTrue(!decoder.accepts(TestPackets.tcp("10.0.0.1", 40000, "10.0.0.2", 514, 1, TestPackets.ACK, data))); + } +} diff --git a/contrib/format-pcapng/src/test/resources/decoders/ntp/ntp.pcapng b/contrib/format-pcapng/src/test/resources/decoders/ntp/ntp.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..9a9bc44e0fb3e7e9d3551bc2c1350e2e1be227bd GIT binary patch literal 556 zcmd<$<>iuLU|{gI(UxKa(*L0VBnIM(0C6f<9KvP;vU7kK1m5zpF>KI|IN$?hgRm=v zVsH>-VBi932V$l<4h+={77Pr^3~X;edI*7+w!fD)02P7kAk{90YM^aEyHuD!c5#3` za9{-^<4fD03LrCKW+TzdK;kgF3V>`7fceW0$Od7czd#hGzpNM-++9Ny^c6fpLIU&+ z^$dBrJo1Z6tQ1l!5_1c3QuUJab9uQOlao^mfPz~3TD)Amz(4`nL#p5AI6}fvoPk*j p{n%;PoEJ)g7S^?U%r!_OJDQ(8mq< z1NtNHlb;7)ekouh4@pBozx#p^3DSWZ&1G}hfXhY~S0iso%Q$!Z%y<1b$rf=(EciT6 zL?_7RbTS%u)s7-dS21@sR`>H@O!Gj~I=o7FTJeV|hkn7+AnM=tqW8N0!Lm$VPNQil zOXqyiKFf;lyqJ~_wct`ObLhLwr$@UxIF_ZeBAYw(B#8sDj^wDikJLUsta#XGJ=N9} zW?8x{o$@4Uj|L9qw(jeJWeU@@nBwm09x^84t3x%WGowe>0|I9TuM8^ZVf|fKK*uN7 zRg@L+EwAD%b?6`tMOo47QWPnli+-zqK&`FY_!d z!D$(`rRs*N>At3GaIWc0RhcG?8DknZTI-fYTD85KT)^}59EMi+(=Dg@{qaYUhE?QH zn3XJj>njrC2=U}0(1130VqhZ=$sR+$3qgn_#8h+n!zJX|Z}g8}tyU=SkVBcR&`)t1 z!Ve1EW5_#w3WjD+1p<7#e<7^~4>tqb2xhVr|1V||hXTZ9m`^=RoT&?pw3#Ud9eKDG zTRTdg6M|Q`5QF&@Fw!R9105OiQEcbCgj|lj(dEUYUh;)|v)SQCHae1}kQY7=1=hq` Fe*wiLGe!Ua literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/decoders/ssdp/ssdp.pcapng b/contrib/format-pcapng/src/test/resources/decoders/ssdp/ssdp.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..fae3eec9ba3c4450ecbb6beff3d5450456fffa1c GIT binary patch literal 968 zcmb_aF>jkt5Pr^(NJtEoGF2U(qE3x~_pnLLQzMxo8e4TBvPtER1YQy(Y>>f`^CR*P zvSjPnrF(|{i2jMTdl&bP7%8nYbm)oG-8Ee0_i;?$bO(ksIi*jXh`N_US9y55p6K z>s(d(!8jDuY}>kNS!-v4+l;BovEvK6SC&a3lC+HTD>>hT>IA-575uG!6=j*KoQ}Pb z<{WuGREOzou{11tdNQj)XeG-E(4eELz>GLMa(Id4z#0x)a7On?YYXd-IqYP&J@SK) zW^igUMuYdN;yP~M)7-!hN5N3gX|&X$3wg+Kes2;ART&1Z6Al8Pa#>Y{Fbr;+IzQ;@ zz@LeZ%`MYNWEtzrDG&i3AG}dRqOJ2z!-LU5@phgj7ZSLb2S^7sSQN8D%=5fT4Dd54+jU8W_G#Wc<(*X%R1tue-5ce|i>6vskrG(eG18`S6<3IWz=^aXNs z*fWN|G=e?f5#k5o;N8c6-ZHLiV+KDQuh)BaE#di3v}!)cE5YHx8eR_M=)eoZAH3@O zs}XYKh17U_b@Q?kR4$`wkx4zyr|bKLy%D3rMqmv_2st`nVA#ayJ*nN3`poi*M)Xof KiJT)WV*LhjQt&DO literal 0 HcmV?d00001 diff --git a/contrib/format-pcapng/src/test/resources/decoders/syslog/syslog.pcapng b/contrib/format-pcapng/src/test/resources/decoders/syslog/syslog.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..27be17f1e165157693c6df61e867c7fc91b4768f GIT binary patch literal 556 zcmZWmK~BOz6dhTxAYoy2WnBDpbRG3Kg6gMP zXP=kTuMgCOyB)FqF=Gw9j;Ou&Yc=-9SVvs7dIC(&EnRS3h>2-B!gfT_vAI4YB3tk#()yV@X4 zqtJ7UP$k*|1_`+Tjol<=pe(Nlswzu_nt)k|4zo-#i*?<5a1#I%xaDx$;VoFXh1|GX z$n(IT3)8X}4h%d`riJt&b@P0qG9Q*IgG{LcmKmlK+@R-%cA#DE>4R9+E*Gc?xC_zY zE+>oyGKv(SdE}S($>)iNu_~Wi)CBY)I`l#JjII', unix + NTP_UNIX_OFFSET, fraction) if unix else bytes(8) + + +def header(first, stratum, poll, precision, root_delay, root_disp, ref_id, ref, orig, recv, xmit): + return (struct.pack('>BBbbII4s', first, stratum, poll, precision, root_delay, root_disp, ref_id) + + ts(ref) + ts(orig) + ts(recv) + ts(xmit)) + + +def main(): + os.makedirs(OUT, exist_ok=True) + payloads = [ + # Client request: LI 0, version 4, mode 3 + (40000, 123, header(0x23, 0, 6, -20, 0, 0, bytes(4), 0, 0, 0, 1704164645)), + # Server response: stratum 2, reference 192.168.1.1 + (123, 40000, header(0x24, 2, 6, -20, 0x0800, 0x10000, bytes([192, 168, 1, 1]), + 1704164000, 1704164645, 1704164645, 1704164646)), + # Not NTP (version 0) on port 123 + (40000, 123, b'GET / HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n'), + # ntpdc monlist request: version 2, mode 7, implementation 3, request code 42 + (40001, 123, struct.pack('>BBBBHH', 0x17, 0, 3, 42, 0, 0) + bytes(40)), + # Control message (mode 6) whose count runs past the datagram + (40002, 123, struct.pack('>BBHHHHH', 0x16, 2, 1, 0, 0, 0, 100)), + ] + f = shb() + idb(101) + for sport, dport, data in payloads: + f += epb(0, ipv4(17, udp(sport, dport, data), '10.0.0.1', '10.0.0.2')) + write(os.path.join(OUT, 'ntp.pcapng'), f) + + +if __name__ == '__main__': + main() diff --git a/contrib/format-pcapng/src/test/resources/fixtures/sip_fixtures.py b/contrib/format-pcapng/src/test/resources/fixtures/sip_fixtures.py new file mode 100644 index 00000000000..47ee94517a6 --- /dev/null +++ b/contrib/format-pcapng/src/test/resources/fixtures/sip_fixtures.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Generates decoders/sip/sip.pcapng. Run from any directory: + python3 contrib/format-pcapng/src/test/resources/fixtures/sip_fixtures.py""" +import os +import struct +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from pcap_fixtures import shb, idb, epb, ipv4, udp, tcp, write # noqa: E402 + +OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'decoders', 'sip') + +INVITE = (b'INVITE sip:bob@biloxi.example.com SIP/2.0\r\n' + b'Via: SIP/2.0/UDP pc33.atlanta.example.com;branch=z9hG4bK776asdhds\r\n' + b'Max-Forwards: 70\r\n' + b'To: Bob \r\n' + b'From: Alice ;tag=1928301774\r\n' + b'Call-ID: a84b4c76e66710@pc33.atlanta.example.com\r\n' + b'CSeq: 314159 INVITE\r\n' + b'Contact: \r\n' + b'Authorization: Digest username="alice", realm="atlanta.example.com", nonce="84a4cc6f", ' + b'response="7587245234b3434cc3412213e5f113a5"\r\n' + b'Content-Type: application/sdp\r\n' + b'Content-Length: 4\r\n' + b'\r\n' + b'v=0\n') +RINGING = (b'SIP/2.0 180 Ringing\r\n' + b'v: SIP/2.0/TCP pc33.atlanta.example.com;branch=z9hG4bK776asdhds\r\n' + b'f: Alice ;tag=1928301774\r\n' + b't: Bob ;tag=a6c85cf\r\n' + b'i: a84b4c76e66710@pc33.atlanta.example.com\r\n' + b'CSeq: 314159 INVITE\r\n' + b'l: 0\r\n' + b'\r\n') + + +def main(): + os.makedirs(OUT, exist_ok=True) + f = shb() + idb(101) + f += epb(0, ipv4(17, udp(5060, 5060, INVITE), '10.0.0.1', '10.0.0.2')) + f += epb(0, ipv4(6, tcp(5060, 40000, 1000, 0x18, RINGING), '10.0.0.2', '10.0.0.1')) + # Not SIP on port 5060: a keep-alive + f += epb(0, ipv4(17, udp(5060, 5060, b'\r\n\r\n'), '10.0.0.1', '10.0.0.2')) + # Malformed header line + f += epb(0, ipv4(17, udp(5060, 5060, b'OPTIONS sip:x@y SIP/2.0\r\nVia: SIP/2.0/UDP h\r\ngarbage\r\n\r\n'), + '10.0.0.1', '10.0.0.2')) + write(os.path.join(OUT, 'sip.pcapng'), f) + + +if __name__ == '__main__': + main() diff --git a/contrib/format-pcapng/src/test/resources/fixtures/ssdp_fixtures.py b/contrib/format-pcapng/src/test/resources/fixtures/ssdp_fixtures.py new file mode 100644 index 00000000000..e2e451b4fe9 --- /dev/null +++ b/contrib/format-pcapng/src/test/resources/fixtures/ssdp_fixtures.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Generates decoders/ssdp/ssdp.pcapng. Run from any directory: + python3 contrib/format-pcapng/src/test/resources/fixtures/ssdp_fixtures.py""" +import os +import struct +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from pcap_fixtures import shb, idb, epb, ipv4, udp, tcp, write # noqa: E402 + +OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'decoders', 'ssdp') + +MESSAGES = [ + (40000, 1900, b'M-SEARCH * HTTP/1.1\r\nHOST: 239.255.255.250:1900\r\nMAN: "ssdp:discover"\r\nMX: 2\r\n' + b'ST: ssdp:all\r\nUSER-AGENT: Linux/5 UPnP/1.1 test/1.0\r\n\r\n'), + (1900, 40000, b'HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=100\r\nEXT:\r\n' + b'LOCATION: http://192.168.1.10:49152/desc.xml\r\nSERVER: Linux/3.14 UPnP/1.0 IpBridge/1.26.0\r\n' + b'ST: upnp:rootdevice\r\nUSN: uuid:2f402f80-da50-11e1-9b23-001788255acc::upnp:rootdevice\r\n\r\n'), + (1900, 1900, b'NOTIFY * HTTP/1.1\r\nHOST: 239.255.255.250:1900\r\nNT: upnp:rootdevice\r\nNTS: ssdp:byebye\r\n' + b'USN: uuid:2f402f80-da50-11e1-9b23-001788255acc::upnp:rootdevice\r\n\r\n'), + # Not SSDP on port 1900 + (40000, 1900, b'GET / HTTP/1.1\r\nHost: example.com\r\n\r\n'), + # Malformed header line + (40000, 1900, b'NOTIFY * HTTP/1.1\r\nNT: upnp:rootdevice\r\nnot a header\r\n\r\n'), +] + + +def main(): + os.makedirs(OUT, exist_ok=True) + f = shb() + idb(101) + for sport, dport, data in MESSAGES: + f += epb(0, ipv4(17, udp(sport, dport, data), '192.168.1.20', '239.255.255.250')) + write(os.path.join(OUT, 'ssdp.pcapng'), f) + + +if __name__ == '__main__': + main() diff --git a/contrib/format-pcapng/src/test/resources/fixtures/syslog_fixtures.py b/contrib/format-pcapng/src/test/resources/fixtures/syslog_fixtures.py new file mode 100644 index 00000000000..fc795f4f65f --- /dev/null +++ b/contrib/format-pcapng/src/test/resources/fixtures/syslog_fixtures.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Generates decoders/syslog/syslog.pcapng. Run from any directory: + python3 contrib/format-pcapng/src/test/resources/fixtures/syslog_fixtures.py""" +import os +import struct +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from pcap_fixtures import shb, idb, epb, ipv4, udp, tcp, write # noqa: E402 + +OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'decoders', 'syslog') + +MESSAGES = [ + b'<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 ' + b'[exampleSDID@32473 iut="3" eventSource="Application"] An application event', + b'<38>Jan 2 03:04:05 gateway sshd[1234]: Accepted password for root from 10.0.0.9', + # Not syslog on port 514 + b'hello world', + # RFC 5424 header cut short + b'<14>1 2024-01-02T03:04:05Z host', +] + + +def main(): + os.makedirs(OUT, exist_ok=True) + f = shb() + idb(101) + for data in MESSAGES: + f += epb(0, ipv4(17, udp(40000, 514, data), '10.0.0.1', '10.0.0.2')) + write(os.path.join(OUT, 'syslog.pcapng'), f) + + +if __name__ == '__main__': + main() From 3fec4c7b08b68d0c8b6dc5fb16db55840ee632fa Mon Sep 17 00:00:00 2001 From: cgivre Date: Fri, 2 Oct 2026 16:01:03 -0400 Subject: [PATCH 27/50] DRILL-8556: Add DHCP, DHCPv6 and TFTP protocol decoders --- .../store/pcap/protocol/dhcp/DhcpDecoder.java | 301 +++++++++++++ .../store/pcap/protocol/dhcp/DhcpMessage.java | 58 +++ .../pcap/protocol/dhcpv6/Dhcpv6Decoder.java | 402 ++++++++++++++++++ .../pcap/protocol/dhcpv6/Dhcpv6Message.java | 53 +++ .../store/pcap/protocol/tftp/TftpDecoder.java | 245 +++++++++++ .../store/pcap/protocol/tftp/TftpMessage.java | 34 ++ ....store.pcap.protocol.PacketProtocolDecoder | 3 + .../protocol/dhcp/TestAddressingDecoders.java | 131 ++++++ .../pcap/protocol/dhcp/TestDhcpDecoder.java | 220 ++++++++++ .../protocol/dhcpv6/TestDhcpv6Decoder.java | 202 +++++++++ .../pcap/protocol/tftp/TestTftpDecoder.java | 166 ++++++++ .../test/resources/decoders/dhcp/dhcp.pcapng | Bin 0 -> 1456 bytes .../resources/decoders/dhcpv6/dhcpv6.pcapng | Bin 0 -> 932 bytes .../test/resources/decoders/tftp/tftp.pcapng | Bin 0 -> 568 bytes .../test/resources/fixtures/dhcp_fixtures.py | 67 +++ .../resources/fixtures/dhcpv6_fixtures.py | 80 ++++ .../test/resources/fixtures/tftp_fixtures.py | 54 +++ 17 files changed, 2016 insertions(+) create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpMessage.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcpv6/Dhcpv6Decoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcpv6/Dhcpv6Message.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/tftp/TftpDecoder.java create mode 100644 contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/tftp/TftpMessage.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dhcp/TestAddressingDecoders.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dhcp/TestDhcpDecoder.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/dhcpv6/TestDhcpv6Decoder.java create mode 100644 contrib/format-pcapng/src/test/java/org/apache/drill/exec/store/pcap/protocol/tftp/TestTftpDecoder.java create mode 100644 contrib/format-pcapng/src/test/resources/decoders/dhcp/dhcp.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/decoders/dhcpv6/dhcpv6.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/decoders/tftp/tftp.pcapng create mode 100644 contrib/format-pcapng/src/test/resources/fixtures/dhcp_fixtures.py create mode 100644 contrib/format-pcapng/src/test/resources/fixtures/dhcpv6_fixtures.py create mode 100644 contrib/format-pcapng/src/test/resources/fixtures/tftp_fixtures.py diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpDecoder.java new file mode 100644 index 00000000000..95a71f4db79 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpDecoder.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.dhcp; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.ScalarWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** + * DHCP (RFC 2131, options from RFC 2132) over UDP ports 67 and 68. A payload is DHCP only if op is 1 or 2, + * the hardware type is not 0, the hardware address length is at most 16 and the magic cookie follows the fixed BOOTP header. Plain + * BOOTP without the cookie is left undecoded. + */ +public class DhcpDecoder implements PacketProtocolDecoder { + static final int MAX_ITEMS = 64; + private static final int HEADER = 236; + private static final int MAGIC_COOKIE = 0x63825363; + private static final String[] MESSAGE_TYPES = {null, "DISCOVER", "OFFER", "REQUEST", "DECLINE", "ACK", "NAK", + "RELEASE", "INFORM", "FORCERENEW", "LEASEQUERY", "LEASEUNASSIGNED", "LEASEUNKNOWN", "LEASEACTIVE"}; + + @Override + public String protocol() { + return "dhcp"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("op", MinorType.VARCHAR) + .addNullable("message_type", MinorType.VARCHAR) + .addNullable("transaction_id", MinorType.BIGINT) + .addNullable("broadcast", MinorType.BIT) + .addNullable("client_mac", MinorType.VARCHAR) + .addNullable("client_ip", MinorType.VARCHAR) + .addNullable("your_ip", MinorType.VARCHAR) + .addNullable("server_ip", MinorType.VARCHAR) + .addNullable("relay_ip", MinorType.VARCHAR) + .addNullable("server_name", MinorType.VARCHAR) + .addNullable("boot_file", MinorType.VARCHAR) + .addNullable("hostname", MinorType.VARCHAR) + .addNullable("requested_ip", MinorType.VARCHAR) + .addNullable("server_id", MinorType.VARCHAR) + .addNullable("lease_time", MinorType.BIGINT) + .addNullable("subnet_mask", MinorType.VARCHAR) + .addNullable("vendor_class", MinorType.VARCHAR) + .addNullable("domain_name", MinorType.VARCHAR) + .addArray("routers", MinorType.VARCHAR) + .addArray("dns_servers", MinorType.VARCHAR) + .addArray("parameter_request_list", MinorType.INT) + .addMapArray("options") + .addNullable("code", MinorType.INT) + .addNullable("value", MinorType.VARCHAR) + .resumeSchema(); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isUdpPacket() && (isDhcpPort(packet.getSrc_port()) || isDhcpPort(packet.getDst_port())); + } + + private static boolean isDhcpPort(int port) { + return port == 67 || port == 68; + } + + @Override + public DhcpMessage parse(Packet packet, byte[] payload, DecoderContext context) { + if (payload == null || payload.length < HEADER + 4) { + return null; + } + int op = payload[0] & 0xFF; + int htype = payload[1] & 0xFF; + int hlen = payload[2] & 0xFF; + if ((op != 1 && op != 2) || htype == 0 || hlen > 16 || u32(payload, HEADER) != MAGIC_COOKIE) { + return null; + } + DhcpMessage m = new DhcpMessage(); + m.op = op == 1 ? "request" : "reply"; + m.transactionId = u32(payload, 4) & 0xFFFFFFFFL; + m.broadcast = (payload[10] & 0x80) != 0; + m.clientIp = ipv4(payload, 12); + m.yourIp = ipv4(payload, 16); + m.serverIp = ipv4(payload, 20); + m.relayIp = ipv4(payload, 24); + m.clientMac = hlen == 0 ? null : mac(payload, 28, hlen); + int overload = options(payload, m, context); + // Option 52 says sname and/or file carry options instead of strings + if ((overload & 2) == 0) { + m.serverName = cString(payload, 44, 64); + } + if ((overload & 1) == 0) { + m.bootFile = cString(payload, 108, 128); + } + return m; + } + + /** Walks the options after the magic cookie; returns the value of option 52 (overload), or 0. */ + private static int options(byte[] b, DhcpMessage m, DecoderContext context) { + int overload = 0; + int pos = HEADER + 4; + int count = 0; + while (pos < b.length) { + int code = b[pos] & 0xFF; + if (code == 255) { + break; + } + if (code == 0) { + pos++; + continue; + } + if (pos + 1 >= b.length) { + throw new IllegalArgumentException("option " + code + " at offset " + pos + " has no length"); + } + int len = b[pos + 1] & 0xFF; + int at = pos + 2; + if (at + len > b.length) { + throw new IllegalArgumentException("option " + code + " at offset " + pos + " overruns the message"); + } + if (count++ < MAX_ITEMS) { + m.options.add(new DhcpMessage.Option(code, hex(b, at, len))); + } else if (count == MAX_ITEMS + 1) { + context.warn("options truncated to " + MAX_ITEMS); + } + switch (code) { + case 1: + m.subnetMask = ipv4(b, fixed(code, at, len, 4)); + break; + case 3: + addresses(b, code, at, len, m.routers, context); + break; + case 6: + addresses(b, code, at, len, m.dnsServers, context); + break; + case 12: + m.hostname = text(b, at, len); + break; + case 15: + m.domainName = text(b, at, len); + break; + case 50: + m.requestedIp = ipv4(b, fixed(code, at, len, 4)); + break; + case 51: + m.leaseTime = u32(b, fixed(code, at, len, 4)) & 0xFFFFFFFFL; + break; + case 52: + overload = b[fixed(code, at, len, 1)] & 0x03; + break; + case 53: + int type = b[fixed(code, at, len, 1)] & 0xFF; + m.messageType = type < MESSAGE_TYPES.length && MESSAGE_TYPES[type] != null + ? MESSAGE_TYPES[type] : String.valueOf(type); + break; + case 54: + m.serverId = ipv4(b, fixed(code, at, len, 4)); + break; + case 55: + for (int i = 0; i < len && m.parameterRequestList.size() < MAX_ITEMS; i++) { + m.parameterRequestList.add(b[at + i] & 0xFF); + } + break; + case 60: + m.vendorClass = text(b, at, len); + break; + default: + break; + } + pos = at + len; + } + return overload; + } + + private static int fixed(int code, int at, int len, int expected) { + if (len != expected) { + throw new IllegalArgumentException("option " + code + " has length " + len); + } + return at; + } + + private static void addresses(byte[] b, int code, int at, int len, List out, DecoderContext context) { + if (len == 0 || len % 4 != 0) { + throw new IllegalArgumentException("option " + code + " has length " + len); + } + for (int i = 0; i < len; i += 4) { + if (out.size() == MAX_ITEMS) { + context.warn("option " + code + " addresses truncated to " + MAX_ITEMS); + return; + } + out.add(ipv4(b, at + i)); + } + } + + /** Option text, without the trailing NULs some clients send; null if empty. */ + private static String text(byte[] b, int at, int len) { + int end = at + len; + while (end > at && b[end - 1] == 0) { + end--; + } + return end == at ? null : new String(b, at, end - at, StandardCharsets.UTF_8); + } + + /** A NUL-terminated string in a fixed-size field; null if empty. */ + private static String cString(byte[] b, int at, int size) { + int end = at; + while (end < at + size && b[end] != 0) { + end++; + } + return end == at ? null : new String(b, at, end - at, StandardCharsets.ISO_8859_1); + } + + private static String ipv4(byte[] b, int at) { + return (b[at] & 0xFF) + "." + (b[at + 1] & 0xFF) + "." + (b[at + 2] & 0xFF) + "." + (b[at + 3] & 0xFF); + } + + private static String mac(byte[] b, int at, int len) { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < len; i++) { + if (i > 0) { + out.append(':'); + } + out.append(String.format("%02x", b[at + i])); + } + return out.toString(); + } + + private static String hex(byte[] b, int at, int len) { + StringBuilder out = new StringBuilder(); + for (int i = at; i < at + len; i++) { + out.append(String.format("%02x", b[i])); + } + return out.toString(); + } + + private static int u32(byte[] b, int at) { + return ((b[at] & 0xFF) << 24) | ((b[at + 1] & 0xFF) << 16) | ((b[at + 2] & 0xFF) << 8) | (b[at + 3] & 0xFF); + } + + @Override + public void write(DhcpMessage m, TupleWriter fields) { + set(fields, "op", m.op); + set(fields, "message_type", m.messageType); + fields.scalar("transaction_id").setLong(m.transactionId); + fields.scalar("broadcast").setBoolean(m.broadcast); + set(fields, "client_mac", m.clientMac); + set(fields, "client_ip", m.clientIp); + set(fields, "your_ip", m.yourIp); + set(fields, "server_ip", m.serverIp); + set(fields, "relay_ip", m.relayIp); + set(fields, "server_name", m.serverName); + set(fields, "boot_file", m.bootFile); + set(fields, "hostname", m.hostname); + set(fields, "requested_ip", m.requestedIp); + set(fields, "server_id", m.serverId); + if (m.leaseTime != null) { + fields.scalar("lease_time").setLong(m.leaseTime); + } + set(fields, "subnet_mask", m.subnetMask); + set(fields, "vendor_class", m.vendorClass); + set(fields, "domain_name", m.domainName); + ScalarWriter routers = fields.array("routers").scalar(); + m.routers.forEach(routers::setString); + ScalarWriter dns = fields.array("dns_servers").scalar(); + m.dnsServers.forEach(dns::setString); + ScalarWriter prl = fields.array("parameter_request_list").scalar(); + m.parameterRequestList.forEach(prl::setInt); + ArrayWriter options = fields.array("options"); + for (DhcpMessage.Option o : m.options) { + TupleWriter t = options.tuple(); + t.scalar("code").setInt(o.code); + t.scalar("value").setString(o.value); + options.save(); + } + } + + private static void set(TupleWriter fields, String name, String value) { + if (value != null) { + fields.scalar(name).setString(value); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpMessage.java new file mode 100644 index 00000000000..81c54a8030e --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpMessage.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.dhcp; + +import java.util.ArrayList; +import java.util.List; + +/** The parsed fields of one DHCP message. */ +public class DhcpMessage { + String op; + String messageType; + long transactionId; + boolean broadcast; + String clientMac; + String clientIp; + String yourIp; + String serverIp; + String relayIp; + String serverName; + String bootFile; + String hostname; + String requestedIp; + String serverId; + Long leaseTime; + String subnetMask; + String vendorClass; + String domainName; + final List routers = new ArrayList<>(); + final List dnsServers = new ArrayList<>(); + final List parameterRequestList = new ArrayList<>(); + final List