diff --git a/.gitignore b/.gitignore index e3d906196e0..f7255ef26ee 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,7 @@ tools/venv/ venv/ .vscode/* exec/java-exec/src/main/resources/webapp/ + +# Python bytecode from the test fixture generators +__pycache__/ +*.pyc diff --git a/contrib/format-pcapng/README.md b/contrib/format-pcapng/README.md index 4b46c1c2caa..0ba4116061b 100644 --- a/contrib/format-pcapng/README.md +++ b/contrib/format-pcapng/README.md @@ -29,6 +29,107 @@ The following table lists configuration attributes: Attribute|Default Value|Description ---------|-------------|----------- stat|false|return the statistics data about the each pcapng file if true +sessionizeTCPStreams|false|return one row per TCP session instead of one row per packet, for both PCAP and PCAP-NG. A session is written when it closes (FIN or RST). Sessions still open at the end of the file, such as ones that outlived the capture, are written last with `session_closed` false +exposeCredentials|false|include cleartext passwords found by protocol decoders in `parsed_data`; usernames and a `password_present` flag are always included -## ToDo -We plan to refactor the NG parser because of the `pcapngdecoder` so bad to process huge pcapng file. (Neither efficient nor support parser packet as stream) \ No newline at end of file +## PCAP-NG columns + +PCAP-NG files are streamed one block at a time, so file size is not limited by memory. + +In addition to the packet columns above, packet queries return the following metadata. +The time of each packet is in `packet_timestamp`, the same column name as in PCAP files. (It was named +`timestamp` in PCAP-NG files before; that name is a reserved word that had to be quoted in every query.) +Timestamps honor each interface's `if_tsresol` and `if_tsoffset`. + +Column|Description +------|----------- +captured_length|Bytes of the packet stored in the file (`packet_length` is the original length on the wire) +interface_id|Index of the capturing interface within its section +interface_name|`if_name` of the capturing interface +link_type|[LINKTYPE_ value](https://www.tcpdump.org/linktypes.html) of the capturing interface +comment|Packet comments (`opt_comment`), newline separated +direction|`inbound` or `outbound` (`epb_flags`) +reception_type|`unicast`, `multicast`, `broadcast` or `promiscuous` (`epb_flags`) +fcs_length|Frame check sequence length in octets (`epb_flags`) +drop_count|Packets lost since the preceding packet (`epb_dropcount`) +packet_hash|`epb_hash` as `algorithm:hex`, such as `md5:9e107d9d...` + +IP, port and TCP columns are decoded for these link types: Ethernet (1), raw IP (101, 228, 229), +BSD loopback (0, 108), PPP (9), Linux cooked capture v1 and v2 (113, 276). On other link types they are null, +and the MAC address columns are only set for Ethernet. + +With `stat` set to true, options a block does not have are null. The `comment` column holds the section, interface, name resolution or +statistics block comment. + +## Protocol decoding + +Packets and TCP sessions whose application protocol is recognized are decoded into three columns, in both +PCAP and PCAP-NG files: + +Column|Description +------|----------- +parsed_protocol|Name of the decoder that handled the row, such as `dns` or `http`; null if none did +parsed_data|One map per decoder, named after its protocol; only the one named by `parsed_protocol` is filled +decode_error|Why the row could not be fully decoded; null when there was no problem + +```sql +-- DNS questions and answers +SELECT t.parsed_data.dns.questions[0].name AS query, t.parsed_data.dns.answers +FROM dfs.`capture.pcapng` t +WHERE parsed_protocol = 'dns'; + +-- HTTP requests, one row per connection +SELECT src_ip, t.parsed_data.http.exchanges[0].host AS host, t.parsed_data.http.exchanges[0].uri AS uri +FROM table(dfs.`capture.pcap` (type => 'pcap', sessionizeTCPStreams => true)) t +WHERE parsed_protocol = 'http'; + +-- What could not be decoded +SELECT decode_error, count(*) FROM dfs.`capture.pcapng` +WHERE decode_error IS NOT NULL GROUP BY decode_error; +``` + +Built-in decoders: + +Protocol|Rows|Matches +--------|----|------- +dns|packets|DNS, mDNS and LLMNR on ports 53, 5353 and 5355 +dns|sessions|DNS over TCP port 53, such as zone transfers +http|packets and sessions|HTTP/1.x on ports 80, 591, 3128, 8000, 8008, 8080 and 8888 +icmp|packets|ICMP and ICMPv6 +arp|packets|ARP +ntp|packets|UDP 123 +syslog|packets|UDP 514, RFC 5424 and BSD formats +ssdp|packets|UDP 1900 +sip|packets|UDP and TCP 5060 +dhcp|packets|UDP 67 and 68 +dhcpv6|packets|UDP 546 and 547 +tftp|packets|UDP 69 +netbios_ns|packets|UDP 137 +radius|packets|UDP 1812, 1813, 1645 and 1646 +snmp|packets|UDP 161 and 162, versions 1, 2c and 3 +tls|packets|ClientHello and ServerHello (SNI, ALPN, JA3, JA3S, JA4) on TLS ports such as 443, 993 and 8443 +tls|sessions|Full handshake on the same ports: negotiated version and cipher, certificates +stun|packets|UDP 3478 and 19302 +quic|packets|QUIC v1 Initial on UDP 443: ClientHello (SNI, ALPN, JA4) decrypted from public keys +kerberos|packets|Kerberos on port 88: message type, principals, encryption types, errors +ftp|sessions|FTP control channel on port 21 +ssh|sessions|SSH identification and key exchange (HASSH) on ports 22 and 2222 +smtp|sessions|SMTP on ports 25, 587 and 2525: credentials, envelope, message headers +pop3|sessions|POP3 on port 110: credentials, retrieved message headers +imap|sessions|IMAP on port 143: credentials, mailboxes, fetched message headers +smb|sessions|SMB2/3 on ports 445 and 139: dialect, signing, and NTLM/Kerberos identity (no hashes) +ldap|sessions|LDAP on port 389: bind DNs, auth result, search filters +telnet|sessions|Telnet on port 23: negotiated options and session text +rdp|sessions|RDP on port 3389: the mstshash cookie and requested security protocols +mqtt|sessions|MQTT on port 1883: client id, credentials, topics + +A decoder only handles traffic that parses as its protocol: other traffic on the same port is left undecoded. +Decoding runs only when `parsed_protocol` or `parsed_data` is queried. + +Errors never stop a file from being read. Malformed packets keep the fields that could be read and explain +the problem in `decode_error`. Damage that makes the rest of a file unreadable, or a file that is not a +capture at all, produces one row with only `decode_error` set; exclude such rows with +`WHERE decode_error IS NULL`. + +Fields of each decoder, and how to write a decoder of your own, are described in +`docs/dev/PcapProtocolDecoders.md`. diff --git a/contrib/format-pcapng/pom.xml b/contrib/format-pcapng/pom.xml index 6050fe23228..ada49f65cad 100644 --- a/contrib/format-pcapng/pom.xml +++ b/contrib/format-pcapng/pom.xml @@ -36,11 +36,6 @@ drill-java-exec ${project.version} - - fr.bmartel - pcapngdecoder - 1.2 - diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java index a09c3e3f272..e9e3f117c7c 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/PcapBatchReader.java @@ -27,9 +27,10 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.store.pcap.decoder.Packet; import org.apache.drill.exec.store.pcap.decoder.PacketDecoder; -import org.apache.drill.exec.store.pcap.decoder.TcpSession; import org.apache.drill.exec.store.pcap.schema.Schema; import org.apache.drill.exec.store.pcap.plugin.PcapFormatConfig; +import org.apache.drill.exec.store.pcap.protocol.ProtocolColumns; +import org.apache.drill.exec.store.pcap.protocol.ProtocolDecoders; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,13 +38,15 @@ import java.io.IOException; import java.io.InputStream; import java.time.Instant; -import java.util.HashMap; -import java.util.Map; +import java.util.ArrayList; +import java.util.List; import static org.apache.drill.exec.store.pcap.PcapFormatUtils.parseBytesToASCII; public class PcapBatchReader implements ManagedReader { protected static final int BUFFER_SIZE = 500_000; + // libpcap's largest snapshot length; a larger header value is not trusted for allocation + private static final int MAX_SNAPLEN = 262_144; private static final Logger logger = LoggerFactory.getLogger(PcapBatchReader.class); private final FileDescrip file; private PacketDecoder decoder; @@ -82,45 +85,61 @@ public class PcapBatchReader implements ManagedReader { private ScalarWriter dataWriter; private ScalarWriter isCorruptWriter; private final PcapFormatConfig readerConfig; - // Writers for TCP Sessions - private ScalarWriter sessionStartTimeWriter; - private ScalarWriter sessionEndTimeWriter; - private ScalarWriter sessionDurationWriter; - private ScalarWriter connectionTimeWriter; - private ScalarWriter packetCountWriter; - private ScalarWriter originPacketCounterWriter; - private ScalarWriter remotePacketCounterWriter; - private ScalarWriter originDataVolumeWriter; - private ScalarWriter remoteDataVolumeWriter; - private ScalarWriter hostDataWriter; - private ScalarWriter remoteDataWriter; - private Map sessionQueue; + private TcpSessionizer sessionizer; + private ProtocolColumns protocolColumns; + // Why the file cannot be read, reported once as an error row + private String fileError; + // Set when a damaged record makes the rest of the file unreadable + private boolean finished; + private int packetCount; + // Why the stream stopped being readable; reported after the packets already buffered + private String readError; + // Snapshot length from the file header, capped at MAX_SNAPLEN + private int maxLength; public PcapBatchReader(PcapFormatConfig readerConfig, FileSchemaNegotiator negotiator) { this.readerConfig = readerConfig; - if (readerConfig.getSessionizeTCPStreams()) { - sessionQueue = new HashMap<>(); - } file = negotiator.file(); openFile(); SchemaBuilder builder = new SchemaBuilder(); Schema pcapSchema = new Schema(readerConfig.getSessionizeTCPStreams()); - TupleMetadata schema = pcapSchema.buildSchema(builder); + pcapSchema.addColumns(builder); + ProtocolColumns.Mode mode = readerConfig.getSessionizeTCPStreams() + ? ProtocolColumns.Mode.SESSION : ProtocolColumns.Mode.PACKET; + ProtocolColumns.addColumns(builder, mode, ProtocolDecoders.get()); + TupleMetadata schema = builder.buildSchema(); negotiator.tableSchema(schema, false); ResultSetLoader loader = negotiator.build(); // Creates writers for all fields (Since schema is known) rowWriter = loader.writer(); - populateColumnWriters(rowWriter); + protocolColumns = new ProtocolColumns(rowWriter, mode, ProtocolDecoders.get(), + readerConfig.getExposeCredentials()); + if (readerConfig.getSessionizeTCPStreams()) { + sessionizer = new TcpSessionizer(rowWriter, protocolColumns); + } else { + populateColumnWriters(rowWriter); + } } @Override public boolean next() { + if (fileError != null) { + if (protocolColumns != null) { + protocolColumns.writeErrorRow(fileError); + } + fileError = null; + return false; + } + if (decoder == null || finished) { + return false; + } while (!rowWriter.isFull()) { if (!parseNextPacket(rowWriter)) { - return false; + // At end of file, more batches are needed only for open sessions that did not fit + return sessionizer != null && !sessionizer.writeOpenSessions(); } } return true; @@ -128,16 +147,13 @@ public boolean next() { @Override public void close() { - - /* This warning could occur in the event of a corrupt or incomplete PCAP file. Specifically, - * if a session is started in one file and the end of the session is not captured in the same file. - */ - if (sessionQueue != null && !sessionQueue.isEmpty()) { - logger.warn("Unclosed sessions remaining in PCAP"); + if (protocolColumns != null) { + protocolColumns.logSummary(logger, file.split().getPath().toString()); } - try { - fsStream.close(); + if (fsStream != null) { + fsStream.close(); + } } catch (IOException e) { throw UserException. dataReadError() @@ -153,74 +169,54 @@ private void openFile() { try { fsStream = file.fileSystem().openPossiblyCompressedStream(file.split().getPath()); decoder = new PacketDecoder(fsStream); - buffer = new byte[BUFFER_SIZE + decoder.getMaxLength()]; - validBytes = fsStream.read(buffer); - } catch (IOException io) { - throw UserException - .dataReadError(io) - .addContext("File name:", file.split().getPath().toString()) - .build(logger); + int snaplen = decoder.getMaxLength(); + maxLength = snaplen <= 0 || snaplen > MAX_SNAPLEN ? MAX_SNAPLEN : snaplen; + buffer = new byte[BUFFER_SIZE + maxLength]; + validBytes = 0; + fill(); + } catch (IOException e) { + decoder = null; + fileError = "file: cannot read: " + ProtocolDecoders.describe(e); + } catch (RuntimeException e) { + decoder = null; + fileError = "file: not a PCAP or PCAP-NG file: " + ProtocolDecoders.describe(e); } } private void populateColumnWriters(RowSetLoader rowWriter) { - if (readerConfig.getSessionizeTCPStreams()) { - srcMacAddressWriter = rowWriter.scalar("src_mac_address"); - dstMacAddressWriter = rowWriter.scalar("dst_mac_address"); - dstIPWriter = rowWriter.scalar("dst_ip"); - srcIPWriter = rowWriter.scalar("src_ip"); - srcPortWriter = rowWriter.scalar("src_port"); - dstPortWriter = rowWriter.scalar("dst_port"); - sessionStartTimeWriter = rowWriter.scalar("session_start_time"); - sessionEndTimeWriter = rowWriter.scalar("session_end_time"); - sessionDurationWriter = rowWriter.scalar("session_duration"); - connectionTimeWriter = rowWriter.scalar("connection_time"); - tcpSessionWriter = rowWriter.scalar("tcp_session"); - packetCountWriter = rowWriter.scalar("total_packet_count"); - hostDataWriter = rowWriter.scalar("data_from_originator"); - remoteDataWriter = rowWriter.scalar("data_from_remote"); - - originPacketCounterWriter = rowWriter.scalar("packet_count_from_origin"); - remotePacketCounterWriter = rowWriter.scalar("packet_count_from_remote"); - originDataVolumeWriter = rowWriter.scalar("data_volume_from_origin"); - remoteDataVolumeWriter = rowWriter.scalar("data_volume_from_remote"); - isCorruptWriter = rowWriter.scalar("is_corrupt"); - - } else { - typeWriter = rowWriter.scalar("type"); - timestampWriter = rowWriter.scalar("packet_timestamp"); - timestampMicroWriter = rowWriter.scalar("timestamp_micro"); - networkWriter = rowWriter.scalar("network"); - srcMacAddressWriter = rowWriter.scalar("src_mac_address"); - dstMacAddressWriter = rowWriter.scalar("dst_mac_address"); - dstIPWriter = rowWriter.scalar("dst_ip"); - srcIPWriter = rowWriter.scalar("src_ip"); - srcPortWriter = rowWriter.scalar("src_port"); - dstPortWriter = rowWriter.scalar("dst_port"); - packetLengthWriter = rowWriter.scalar("packet_length"); - - //Writers for TCP Packets - tcpSessionWriter = rowWriter.scalar("tcp_session"); - tcpSequenceWriter = rowWriter.scalar("tcp_sequence"); - tcpAckNumberWriter = rowWriter.scalar("tcp_ack"); - tcpFlagsWriter = rowWriter.scalar("tcp_flags"); - tcpParsedFlagsWriter = rowWriter.scalar("tcp_parsed_flags"); - tcpNsWriter = rowWriter.scalar("tcp_flags_ns"); - tcpCwrWriter = rowWriter.scalar("tcp_flags_cwr"); - tcpEceWriter = rowWriter.scalar("tcp_flags_ece"); - tcpFlagsEceEcnCapableWriter = rowWriter.scalar("tcp_flags_ece_ecn_capable"); - tcpFlagsCongestionWriter = rowWriter.scalar("tcp_flags_ece_congestion_experienced"); - - tcpUrgWriter = rowWriter.scalar("tcp_flags_urg"); - tcpAckWriter = rowWriter.scalar("tcp_flags_ack"); - tcpPshWriter = rowWriter.scalar("tcp_flags_psh"); - tcpRstWriter = rowWriter.scalar("tcp_flags_rst"); - tcpSynWriter = rowWriter.scalar("tcp_flags_syn"); - tcpFinWriter = rowWriter.scalar("tcp_flags_fin"); - - dataWriter = rowWriter.scalar("data"); - isCorruptWriter = rowWriter.scalar("is_corrupt"); - } + typeWriter = rowWriter.scalar("type"); + timestampWriter = rowWriter.scalar("packet_timestamp"); + timestampMicroWriter = rowWriter.scalar("timestamp_micro"); + networkWriter = rowWriter.scalar("network"); + srcMacAddressWriter = rowWriter.scalar("src_mac_address"); + dstMacAddressWriter = rowWriter.scalar("dst_mac_address"); + dstIPWriter = rowWriter.scalar("dst_ip"); + srcIPWriter = rowWriter.scalar("src_ip"); + srcPortWriter = rowWriter.scalar("src_port"); + dstPortWriter = rowWriter.scalar("dst_port"); + packetLengthWriter = rowWriter.scalar("packet_length"); + + //Writers for TCP Packets + tcpSessionWriter = rowWriter.scalar("tcp_session"); + tcpSequenceWriter = rowWriter.scalar("tcp_sequence"); + tcpAckNumberWriter = rowWriter.scalar("tcp_ack"); + tcpFlagsWriter = rowWriter.scalar("tcp_flags"); + tcpParsedFlagsWriter = rowWriter.scalar("tcp_parsed_flags"); + tcpNsWriter = rowWriter.scalar("tcp_flags_ns"); + tcpCwrWriter = rowWriter.scalar("tcp_flags_cwr"); + tcpEceWriter = rowWriter.scalar("tcp_flags_ece"); + tcpFlagsEceEcnCapableWriter = rowWriter.scalar("tcp_flags_ece_ecn_capable"); + tcpFlagsCongestionWriter = rowWriter.scalar("tcp_flags_ece_congestion_experienced"); + + tcpUrgWriter = rowWriter.scalar("tcp_flags_urg"); + tcpAckWriter = rowWriter.scalar("tcp_flags_ack"); + tcpPshWriter = rowWriter.scalar("tcp_flags_psh"); + tcpRstWriter = rowWriter.scalar("tcp_flags_rst"); + tcpSynWriter = rowWriter.scalar("tcp_flags_syn"); + tcpFinWriter = rowWriter.scalar("tcp_flags_fin"); + + dataWriter = rowWriter.scalar("data"); + isCorruptWriter = rowWriter.scalar("is_corrupt"); } private boolean parseNextPacket(RowSetLoader rowWriter) { @@ -228,151 +224,140 @@ private boolean parseNextPacket(RowSetLoader rowWriter) { Packet packet = new Packet(); if (offset >= validBytes) { + if (readError != null) { + protocolColumns.writeErrorRow("file: read failed after packet " + packetCount + ": " + readError); + readError = null; + finished = true; + } return false; } - if (validBytes - offset < decoder.getMaxLength()) { - getNextPacket(rowWriter); + if (validBytes - offset < maxLength && readError == null && !finished) { + refill(); } int old = offset; - offset = decoder.decodePacket(buffer, offset, packet, decoder.getMaxLength(), validBytes); + try { + offset = decoder.decodePacket(buffer, offset, packet, maxLength, validBytes); + } catch (RuntimeException e) { + // The record's length cannot be trusted, so neither can anything after it + finished = true; + if (protocolColumns != null) { + String error = "file: invalid packet record after packet " + packetCount + ": " + ProtocolDecoders.describe(e); + // Damage from a failed read shows up as a bad record; report both in the one row + protocolColumns.writeErrorRow(readError == null ? error : error + "; file: read failed: " + readError); + readError = null; + } + return false; + } + packetCount++; if (offset > validBytes) { // Mark that the packet is corrupt and extract whatever data can be extracted from it packet.setIsCorrupt(true); logger.debug("Invalid packet at offset {}", old); } - // If we are resessionizing the TCP Stream, add the packet to the stream - if (readerConfig.getSessionizeTCPStreams()) { - // If the session has not been seen before, add it to the queue - long sessionID = packet.getSessionHash(); - if (!sessionQueue.containsKey(sessionID)) { - logger.debug("Adding session {} to session queue.", sessionID); - sessionQueue.put(sessionID, new TcpSession(sessionID)); - } - - // When the session is closed, write it and remove it from the session queue. - sessionQueue.get(sessionID).addPacket(packet); - if (sessionQueue.get(sessionID).connectionClosed()) { - // Write out the session - addSessionDataToTable(sessionQueue.get(sessionID), rowWriter); - // Remove from the queue - sessionQueue.remove(sessionID); + if (sessionizer != null) { + if (packet.getDecodeError() != null) { + // Packets are not rows here; a packet that failed to decode cannot join a session + protocolColumns.writeErrorRowOnce("packet: " + packet.getDecodeError()); + } else { + sessionizer.addPacket(packet); } - } else { addDataToTable(packet, decoder.getNetwork(), rowWriter); } return true; } - private boolean getNextPacket(RowSetLoader rowWriter) { - Packet packet = new Packet(); - try { - if (validBytes == buffer.length) { - // shift data and read more. This is the common case. - System.arraycopy(buffer, offset, buffer, 0, validBytes - offset); - validBytes = validBytes - offset; - offset = 0; + /** + * Moves the unread bytes to the front of the buffer and fills the rest. + */ + private void refill() { + System.arraycopy(buffer, offset, buffer, 0, validBytes - offset); + validBytes -= offset; + offset = 0; + fill(); + } + /** + * Reads until the buffer is full or the file ends. Compressed streams return short + * reads, so one read is not enough. A read failure keeps the bytes read before it + * and is reported once those packets have been returned. + */ + private void fill() { + try { + while (validBytes < buffer.length) { int n = fsStream.read(buffer, validBytes, buffer.length - validBytes); - if (n > 0) { - validBytes += n; - } - logger.debug("read {} bytes, at {} offset", n, validBytes); - } else { - // near the end of the file, we will just top up the buffer without shifting - int n = fsStream.read(buffer, offset, buffer.length - offset); - if (n > 0) { - validBytes = validBytes + n; - logger.debug("Topped up buffer with {} bytes to yield {}", n, validBytes); + if (n < 0) { + break; } + validBytes += n; } - } catch (Exception e) { - // Exception denotes EOF or unreadable file - return false; + } catch (IOException e) { + readError = ProtocolDecoders.describe(e); } - return true; } - private void addSessionDataToTable(TcpSession session, RowSetLoader rowWriter) { - rowWriter.start(); - - sessionStartTimeWriter.setTimestamp(session.getSessionStartTime()); - sessionEndTimeWriter.setTimestamp(session.getSessionEndTime()); - sessionDurationWriter.setPeriod(session.getSessionDuration()); - connectionTimeWriter.setPeriod(session.getConnectionTime()); - - srcMacAddressWriter.setString(session.getSrcMac()); - dstMacAddressWriter.setString(session.getDstMac()); - srcIPWriter.setString(session.getSrcIP()); - dstIPWriter.setString(session.getDstIP()); - srcPortWriter.setInt(session.getSrcPort()); - dstPortWriter.setInt(session.getDstPort()); - tcpSessionWriter.setLong(session.getSessionID()); - packetCountWriter.setInt(session.getPacketCount()); - - originPacketCounterWriter.setInt(session.getPacketCountFromOrigin()); - remotePacketCounterWriter.setInt(session.getPacketCountFromRemote()); - originDataVolumeWriter.setInt(session.getDataFromOriginator().length); - remoteDataVolumeWriter.setInt(session.getDataFromRemote().length); - isCorruptWriter.setBoolean(session.hasCorruptedData()); - - hostDataWriter.setString(session.getDataFromOriginatorAsString()); - remoteDataWriter.setString(session.getDataFromRemoteAsString()); - rowWriter.save(); - } private void addDataToTable(Packet packet, int networkType, RowSetLoader rowWriter) { rowWriter.start(); - - typeWriter.setString(packet.getPacketType()); - timestampWriter.setTimestamp(Instant.ofEpochMilli(packet.getTimestamp())); - timestampMicroWriter.setLong(packet.getTimestampMicro()); - networkWriter.setInt(networkType); - srcMacAddressWriter.setString(packet.getEthernetSource()); - dstMacAddressWriter.setString(packet.getEthernetDestination()); - - String destinationIp = packet.getDestinationIpAddressString(); - if (destinationIp == null) { - dstIPWriter.setNull(); - } else { - dstIPWriter.setString(destinationIp); + List errors = new ArrayList<>(); + if (packet.getDecodeError() != null) { + errors.add("packet: " + packet.getDecodeError()); } + try { + typeWriter.setString(packet.getPacketType()); + timestampWriter.setTimestamp(Instant.ofEpochMilli(packet.getTimestamp())); + timestampMicroWriter.setLong(packet.getTimestampMicro()); + networkWriter.setInt(networkType); + srcMacAddressWriter.setString(packet.getEthernetSource()); + dstMacAddressWriter.setString(packet.getEthernetDestination()); + + String destinationIp = packet.getDestinationIpAddressString(); + if (destinationIp == null) { + dstIPWriter.setNull(); + } else { + dstIPWriter.setString(destinationIp); + } - String sourceIp = packet.getSourceIpAddressString(); - if (sourceIp == null) { - srcIPWriter.setNull(); - } else { - srcIPWriter.setString(sourceIp); + String sourceIp = packet.getSourceIpAddressString(); + if (sourceIp == null) { + srcIPWriter.setNull(); + } else { + srcIPWriter.setString(sourceIp); + } + srcPortWriter.setInt(packet.getSrc_port()); + dstPortWriter.setInt(packet.getDst_port()); + packetLengthWriter.setInt(packet.getPacketLength()); + + // TCP Only Packet Data + tcpSessionWriter.setLong(packet.getSessionHash()); + tcpSequenceWriter.setInt(packet.getSequenceNumber()); + tcpAckNumberWriter.setInt(packet.getAckNumber()); + tcpFlagsWriter.setInt(packet.getFlags()); + tcpParsedFlagsWriter.setString(packet.getParsedFlags()); + + // TCP Flags + tcpNsWriter.setBoolean((packet.getFlags() & 0x100) != 0); + tcpCwrWriter.setBoolean((packet.getFlags() & 0x80) != 0); + tcpEceWriter.setBoolean((packet.getFlags() & 0x40) != 0); + tcpFlagsEceEcnCapableWriter.setBoolean((packet.getFlags() & 0x42) == 0x42); + tcpFlagsCongestionWriter.setBoolean((packet.getFlags() & 0x42) == 0x40); + tcpUrgWriter.setBoolean((packet.getFlags() & 0x20) != 0); + tcpAckWriter.setBoolean((packet.getFlags() & 0x10) != 0); + tcpPshWriter.setBoolean((packet.getFlags() & 0x8) != 0); + tcpRstWriter.setBoolean((packet.getFlags() & 0x4) != 0); + tcpSynWriter.setBoolean((packet.getFlags() & 0x2) != 0); + tcpFinWriter.setBoolean((packet.getFlags() & 0x1) != 0); + + // Note: getData() MUST be called before isCorrupt + dataWriter.setString(parseBytesToASCII(packet.getData())); + isCorruptWriter.setBoolean(packet.isCorrupt()); + } catch (RuntimeException e) { + // Keep the fields written so far + errors.add("packet: " + ProtocolDecoders.describe(e)); } - srcPortWriter.setInt(packet.getSrc_port()); - dstPortWriter.setInt(packet.getDst_port()); - packetLengthWriter.setInt(packet.getPacketLength()); - - // TCP Only Packet Data - tcpSessionWriter.setLong(packet.getSessionHash()); - tcpSequenceWriter.setInt(packet.getSequenceNumber()); - tcpAckNumberWriter.setInt(packet.getAckNumber()); - tcpFlagsWriter.setInt(packet.getFlags()); - tcpParsedFlagsWriter.setString(packet.getParsedFlags()); - - // TCP Flags - tcpNsWriter.setBoolean((packet.getFlags() & 0x100) != 0); - tcpCwrWriter.setBoolean((packet.getFlags() & 0x80) != 0); - tcpEceWriter.setBoolean((packet.getFlags() & 0x40) != 0); - tcpFlagsEceEcnCapableWriter.setBoolean((packet.getFlags() & 0x42) == 0x42); - tcpFlagsCongestionWriter.setBoolean((packet.getFlags() & 0x42) == 0x40); - tcpUrgWriter.setBoolean((packet.getFlags() & 0x20) != 0); - tcpAckWriter.setBoolean((packet.getFlags() & 0x10) != 0); - tcpPshWriter.setBoolean((packet.getFlags() & 0x8) != 0); - tcpRstWriter.setBoolean((packet.getFlags() & 0x4) != 0); - tcpSynWriter.setBoolean((packet.getFlags() & 0x2) != 0); - tcpFinWriter.setBoolean((packet.getFlags() & 0x1) != 0); - - // Note: getData() MUST be called before isCorrupt - dataWriter.setString(parseBytesToASCII(packet.getData())); - isCorruptWriter.setBoolean(packet.isCorrupt()); + protocolColumns.writePacket(packet, errors); rowWriter.save(); // TODO Parse Data Packet Here: diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java new file mode 100644 index 00000000000..22ce71078f7 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/TcpSessionizer.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.drill.exec.physical.resultSet.RowSetLoader; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; +import org.apache.drill.exec.store.pcap.protocol.ProtocolColumns; +import org.apache.drill.exec.vector.accessor.ScalarWriter; + +/** + * Groups TCP packets into sessions and writes one row per session, using the + * sessionized schema of {@link org.apache.drill.exec.store.pcap.schema.Schema}, + * when the session closes. Shared by the PCAP and PCAP-NG readers. + */ +public class TcpSessionizer { + // In order of each session's first packet + private final Map openSessions = new LinkedHashMap<>(); + // Sessions already written, so the ACKs that trail a FIN do not start a new one + private final Set closedSessions = new HashSet<>(); + private final RowSetLoader rowWriter; + private final ProtocolColumns protocolColumns; + private final ScalarWriter srcMacAddressWriter; + private final ScalarWriter dstMacAddressWriter; + private final ScalarWriter dstIPWriter; + private final ScalarWriter srcIPWriter; + private final ScalarWriter srcPortWriter; + private final ScalarWriter dstPortWriter; + private final ScalarWriter sessionStartTimeWriter; + private final ScalarWriter sessionEndTimeWriter; + private final ScalarWriter sessionDurationWriter; + private final ScalarWriter connectionTimeWriter; + private final ScalarWriter tcpSessionWriter; + private final ScalarWriter packetCountWriter; + private final ScalarWriter hostDataWriter; + private final ScalarWriter remoteDataWriter; + private final ScalarWriter originPacketCounterWriter; + private final ScalarWriter remotePacketCounterWriter; + private final ScalarWriter originDataVolumeWriter; + private final ScalarWriter remoteDataVolumeWriter; + private final ScalarWriter isCorruptWriter; + private final ScalarWriter sessionClosedWriter; + + public TcpSessionizer(RowSetLoader rowWriter, ProtocolColumns protocolColumns) { + this.rowWriter = rowWriter; + this.protocolColumns = protocolColumns; + srcMacAddressWriter = rowWriter.scalar("src_mac_address"); + dstMacAddressWriter = rowWriter.scalar("dst_mac_address"); + dstIPWriter = rowWriter.scalar("dst_ip"); + srcIPWriter = rowWriter.scalar("src_ip"); + srcPortWriter = rowWriter.scalar("src_port"); + dstPortWriter = rowWriter.scalar("dst_port"); + sessionStartTimeWriter = rowWriter.scalar("session_start_time"); + sessionEndTimeWriter = rowWriter.scalar("session_end_time"); + sessionDurationWriter = rowWriter.scalar("session_duration"); + connectionTimeWriter = rowWriter.scalar("connection_time"); + tcpSessionWriter = rowWriter.scalar("tcp_session"); + packetCountWriter = rowWriter.scalar("total_packet_count"); + hostDataWriter = rowWriter.scalar("data_from_originator"); + remoteDataWriter = rowWriter.scalar("data_from_remote"); + originPacketCounterWriter = rowWriter.scalar("packet_count_from_origin"); + remotePacketCounterWriter = rowWriter.scalar("packet_count_from_remote"); + originDataVolumeWriter = rowWriter.scalar("data_volume_from_origin"); + remoteDataVolumeWriter = rowWriter.scalar("data_volume_from_remote"); + isCorruptWriter = rowWriter.scalar("is_corrupt"); + sessionClosedWriter = rowWriter.scalar("session_closed"); + } + + /** + * Adds a packet to its session, writing the session's row if the packet + * closes it. Packets other than TCP are ignored. + */ + public void addPacket(Packet packet) { + if (!packet.isTcpPacket()) { + return; + } + long sessionId = packet.getSessionHash(); + if (closedSessions.contains(sessionId)) { + if (!packet.getSynFlag()) { + return; + } + // The same addresses and ports opening a new connection + closedSessions.remove(sessionId); + } + TcpSession session = openSessions.computeIfAbsent(sessionId, TcpSession::new); + session.addPacket(packet); + if (session.connectionClosed()) { + write(session); + openSessions.remove(sessionId); + closedSessions.add(sessionId); + } + } + + /** + * Writes the sessions that never saw a FIN or RST, such as ones still open + * when the capture stopped, with session_closed false. Call at end of file, + * once per batch until it returns true. + * + * @return true when every open session has been written + */ + public boolean writeOpenSessions() { + Iterator sessions = openSessions.values().iterator(); + while (sessions.hasNext() && !rowWriter.isFull()) { + TcpSession session = sessions.next(); + // Assembles the payload, as a closing handshake would + session.closeSession(); + write(session); + sessions.remove(); + } + return openSessions.isEmpty(); + } + + private void write(TcpSession session) { + rowWriter.start(); + + sessionStartTimeWriter.setTimestamp(session.getSessionStartTime()); + sessionEndTimeWriter.setTimestamp(session.getSessionEndTime()); + sessionDurationWriter.setPeriod(session.getSessionDuration()); + if (session.getConnectionTime() != null) { + connectionTimeWriter.setPeriod(session.getConnectionTime()); + } + + srcMacAddressWriter.setString(session.getSrcMac()); + dstMacAddressWriter.setString(session.getDstMac()); + srcIPWriter.setString(session.getSrcIP()); + dstIPWriter.setString(session.getDstIP()); + srcPortWriter.setInt(session.getSrcPort()); + dstPortWriter.setInt(session.getDstPort()); + tcpSessionWriter.setLong(session.getSessionID()); + packetCountWriter.setInt(session.getPacketCount()); + + originPacketCounterWriter.setInt(session.getPacketCountFromOrigin()); + remotePacketCounterWriter.setInt(session.getPacketCountFromRemote()); + originDataVolumeWriter.setInt(session.getDataFromOriginator().length); + remoteDataVolumeWriter.setInt(session.getDataFromRemote().length); + isCorruptWriter.setBoolean(session.hasCorruptedData()); + sessionClosedWriter.setBoolean(session.connectionClosed()); + + hostDataWriter.setString(session.getDataFromOriginatorAsString()); + remoteDataWriter.setString(session.getDataFromRemoteAsString()); + protocolColumns.writeSession(session); + rowWriter.save(); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java index 7c0fc92724f..94fa725a8c8 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/Packet.java @@ -25,6 +25,7 @@ import java.io.InputStream; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.Arrays; import java.util.Formatter; import static org.apache.drill.exec.store.pcap.PcapFormatUtils.convertInt; @@ -48,15 +49,18 @@ public class Packet implements Comparable { protected byte[] raw; // index into the raw data where the current ethernet packet starts - private int etherOffset; + protected int etherOffset; // index into the raw data where the current IP packet starts. Should be just after etherOffset protected int ipOffset; + // index into the raw data where the TCP/UDP header starts, or -1 if not IPv4/IPv6 + protected int transportOffset = -1; private int packetLength; protected int etherProtocol; protected int protocol; - protected boolean isRoutingV6; protected boolean isCorrupt = false; + // Why the packet could not be fully decoded, or null + private String decodeError; private static final Logger logger = LoggerFactory.getLogger(Packet.class); @@ -83,11 +87,14 @@ public boolean readPcap(final InputStream in, final boolean byteOrder, final int @SuppressWarnings("WeakerAccess") public int decodePcap(final byte[] buffer, final int offset, final boolean byteOrder, final int maxLength) { - raw = buffer; - etherOffset = offset + PacketConstants.PCAP_HEADER_SIZE; - decodePcapHeader(raw, byteOrder, maxLength, offset); + decodePcapHeader(buffer, byteOrder, maxLength, offset); + // Copy the record: the reader reuses its buffer, and packets kept for + // sessionization must not change when it does + int end = offset + PacketConstants.PCAP_HEADER_SIZE + originalLength; + raw = Arrays.copyOfRange(buffer, offset, end); + etherOffset = PacketConstants.PCAP_HEADER_SIZE; decodeEtherPacket(); - return offset + PacketConstants.PCAP_HEADER_SIZE + originalLength; + return end; } public String getPacketType() { @@ -131,12 +138,13 @@ public boolean isUdpPacket() { @SuppressWarnings("WeakerAccess") public boolean isArpPacket() { - return protocol == PacketConstants.ARP_PROTOCOL; + return etherProtocol == PacketConstants.ARP_TYPE; } @SuppressWarnings("WeakerAccess") public boolean isIcmpPacket() { - return protocol == PacketConstants.ICMP_PROTOCOL; + return isIpV4Packet() && protocol == PacketConstants.ICMP_PROTOCOL + || isIpV6Packet() && protocol == PacketConstants.ICMPV6_PROTOCOL; } public long getSessionHash() { @@ -171,6 +179,14 @@ public long getTimestampMicro() { return timestampMicro; } + /** + * For decoders that read the timestamp from somewhere other than a PCAP record header. + */ + protected void setTimestampMicro(long timestampMicro) { + this.timestampMicro = timestampMicro; + this.timestamp = timestampMicro / 1000L; + } + public int getPacketLength() { return packetLength; } @@ -350,56 +366,107 @@ public static String formatFlags(int flags) { } public int getSrc_port() { + if (transportUnknown()) { + return 0; + } if (isPPPoV6Packet()) { return getPort(64); } - if (isIpV6Packet()) { - if (isRoutingV6) { - return getPort(136); - } - return getPort(40); - } return getPort(0); } public int getDst_port() { + if (transportUnknown()) { + return 0; + } if (isPPPoV6Packet()) { return getPort(66); } - if (isIpV6Packet()) { - if (isRoutingV6) { - return getPort(138); - } - return getPort(42); - } return getPort(2); } + /** True for an IP packet whose header could not be parsed far enough to find the transport layer. */ + private boolean transportUnknown() { + return (isIpV4Packet() || isIpV6Packet()) && transportOffset < 0; + } + public boolean isCorrupt(){ return isCorrupt; } + public String getDecodeError() { + return decodeError; + } + + protected void setDecodeError(String decodeError) { + this.decodeError = decodeError; + } + public byte[] getData() { - int payloadDataStart = getIPHeaderLength(); + int payloadStart = ipOffset + getIPHeaderLength(); if (isTcpPacket()) { - payloadDataStart += this.getTCPHeaderLength(raw); + payloadStart += getTCPHeaderLength(raw); } else if (isUdpPacket()) { - payloadDataStart += this.getUDPHeaderLength(); + payloadStart += getUDPHeaderLength(); } else { return null; } - byte[] data = null; - if (packetLength >= payloadDataStart) { - data = new byte[packetLength - payloadDataStart]; - try { - System.arraycopy(raw, ipOffset + payloadDataStart, data, 0, data.length); - } catch (Exception e) { - isCorrupt = true; - String message = "Error while parsing PCAP data: {}"; - logger.debug(message, e.getMessage()); - logger.trace(message, e); } + // The IP length excludes link-layer padding and trailers; the captured + // length bounds packets truncated by the snapshot length + int capturedEnd = Math.min(getFrameEnd(), raw.length); + if (payloadStart > capturedEnd) { + // The headers themselves were cut off + isCorrupt = true; + return null; } - return data; + int payloadEnd = Math.min(getIpPacketEnd(), capturedEnd); + return payloadStart < payloadEnd ? Arrays.copyOfRange(raw, payloadStart, payloadEnd) : null; + } + + /** + * The bytes after the link-layer header, up to the end of the captured data: + * for example the whole ARP message of an ARP frame. + * + * @return null if there are none + */ + public byte[] getLinkPayload() { + int start = etherOffset + PacketConstants.IP_OFFSET; + int end = Math.min(getFrameEnd(), raw.length); + return start >= 0 && start < end ? Arrays.copyOfRange(raw, start, end) : null; + } + + /** + * The bytes after the IP header and any IPv6 extension headers, bounded by the + * IP length: for example the whole ICMP message, or a UDP header and its data. + * + * @return null if this is not an IP packet whose header could be parsed + */ + public byte[] getIpPayload() { + if (transportOffset < 0) { + return null; + } + int end = Math.min(Math.min(getIpPacketEnd(), getFrameEnd()), raw.length); + return transportOffset < end ? Arrays.copyOfRange(raw, transportOffset, end) : null; + } + + /** + * @return index just past the captured bytes of this packet + */ + protected int getFrameEnd() { + return etherOffset + originalLength; + } + + private int getIpPacketEnd() { + if (isIpV4Packet()) { + int totalLength = convertShort(raw, ipOffset + 2); + // Zero when the capture happened before TCP segmentation offload filled it in + return totalLength == 0 ? Integer.MAX_VALUE : ipOffset + totalLength; + } else if (isIpV6Packet()) { + int payloadLength = convertShort(raw, ipOffset + 4); + // Zero for jumbograms + return payloadLength == 0 ? Integer.MAX_VALUE : ipOffset + 40 + payloadLength; + } + return Integer.MAX_VALUE; } private InetAddress getIPAddress(final boolean src) { @@ -417,16 +484,13 @@ private InetAddress getIPAddress(final boolean src) { private byte[] getIpAddressBytes(final boolean src) { int srcPos; byte[] ipBuffer; - int byteShift = 0; if (isIpV4Packet()) { ipBuffer = new byte[4]; srcPos = src ? PacketConstants.IP4_SRC_OFFSET : PacketConstants.IP4_DST_OFFSET; } else if (isIpV6Packet()) { + // Always in the fixed header, whatever extension headers follow ipBuffer = new byte[16]; - if (isRoutingV6) { - byteShift = 96; - } - srcPos = src ? PacketConstants.IP6_SRC_OFFSET + byteShift : PacketConstants.IP6_DST_OFFSET + byteShift; + srcPos = src ? PacketConstants.IP6_SRC_OFFSET : PacketConstants.IP6_DST_OFFSET; } else if (isPPPoV6Packet()) { ipBuffer = new byte[16]; srcPos = src ? PacketConstants.IP6_SRC_OFFSET + PacketConstants.PPPoV6_IP_OFFSET : PacketConstants.IP6_DST_OFFSET + PacketConstants.PPPoV6_IP_OFFSET; @@ -437,7 +501,13 @@ private byte[] getIpAddressBytes(final boolean src) { return ipBuffer; } + /** + * @return length of the IPv4 header, or of the IPv6 header plus extension headers + */ private int getIPHeaderLength() { + if (transportOffset >= 0) { + return transportOffset - ipOffset; + } return (raw[etherOffset + PacketConstants.VER_IHL_OFFSET] & 0xF) * 4; } @@ -445,8 +515,9 @@ private int getTCPHeaderLength(final byte[] packet) { final int inTCPHeaderDataOffset = 12; // tcp packet header can have options - int dataOffset = etherOffset + getIPHeaderLength() + inTCPHeaderDataOffset; - return 20 + ((packet[dataOffset] >> 4) & 0xF) * 4; + int dataOffset = ipOffset + getIPHeaderLength() + inTCPHeaderDataOffset; + // Data offset is the whole header length in 32-bit words + return ((packet[dataOffset] >> 4) & 0xF) * 4; } private int getUDPHeaderLength() { @@ -478,18 +549,26 @@ private long getTimestampMicro(final byte[] header, final boolean byteOrder, fin private void decodeEtherPacket() { etherProtocol = getShort(raw, etherOffset + PacketConstants.PACKET_PROTOCOL_OFFSET); ipOffset = etherOffset + PacketConstants.IP_OFFSET; - if (isIpV4Packet()) { - protocol = processIpV4Packet(); - } else if (isIpV6Packet()) { - protocol = processIpV6Packet(); - } else if (isPPPoV6Packet()) { - protocol = getByte(raw, etherOffset + 48); + try { + if (isIpV4Packet()) { + protocol = processIpV4Packet(); + } else if (isIpV6Packet()) { + protocol = processIpV6Packet(); + } else if (isPPPoV6Packet()) { + protocol = getByte(raw, etherOffset + 48); + } + } catch (RuntimeException e) { + // Keep what was read (link layer, IP addresses); the transport layer is unknown + isCorrupt = true; + decodeError = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + transportOffset = -1; } // everything is decoded lazily } protected int processIpV4Packet() { validateIpV4Packet(); + transportOffset = ipOffset + ipV4HeaderLength(); return getByte(raw, ipOffset + 9); } @@ -497,40 +576,40 @@ protected int processIpV6Packet() { Preconditions.checkState(ipVersion() == 6, "Should have seen IP version 6, got %d", ipVersion()); int headerLength = 40; int nextHeader = raw[ipOffset + 6] & 0xff; - while (nextHeader != PacketConstants.TCP_PROTOCOL && nextHeader != PacketConstants.UDP_PROTOCOL && nextHeader != PacketConstants.NO_NEXT_HEADER) { + while (true) { + int extension = ipOffset + headerLength; switch (nextHeader) { case PacketConstants.FRAGMENT_V6: - nextHeader = getByte(raw, ipOffset + headerLength); + nextHeader = getByte(raw, extension); headerLength += 8; break; case PacketConstants.ROUTING_V6: - isRoutingV6 = true; - nextHeader = getByte(raw, ipOffset + headerLength + 15); - headerLength += (getByte(raw, ipOffset + headerLength) + 1) * 8; - break; case PacketConstants.HOP_BY_HOP_EXTENSION_V6: case PacketConstants.DESTINATION_OPTIONS_V6: - case PacketConstants.AUTHENTICATION_V6: - case PacketConstants.ENCAPSULATING_SECURITY_V6: case PacketConstants.MOBILITY_EXTENSION_V6: case PacketConstants.HOST_IDENTITY_PROTOCOL: case PacketConstants.SHIM6_PROTOCOL: - nextHeader = getByte(raw, ipOffset + headerLength); - headerLength += (getByte(raw, ipOffset + headerLength) + 1) * 8; + // Next header, then length in 8-octet units not counting the first 8 + nextHeader = getByte(raw, extension); + headerLength += (getByte(raw, extension + 1) + 1) * 8; + break; + case PacketConstants.AUTHENTICATION_V6: + // Length in 4-octet units not counting the first 8 + nextHeader = getByte(raw, extension); + headerLength += (getByte(raw, extension + 1) + 2) * 4; break; default: - //noinspection ConstantConditions - logger.warn("Unknown V6 extension or protocol: {}", nextHeader); - return getByte(raw, ipOffset + headerLength); + // Upper-layer protocol, or ESP whose contents are encrypted + transportOffset = extension; + return nextHeader; } } - return nextHeader; } private void validateIpV4Packet() { Preconditions.checkState(ipVersion() == 4, "Should have seen IP version 4, got %d", ipVersion()); int n = ipV4HeaderLength(); - Preconditions.checkState(n >= 20 && n < 200, "Invalid header length: ", n); + Preconditions.checkState(n >= 20 && n < 200, "Invalid IPv4 header length %s", n); } private String getEthernetAddress(int offset) { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketConstants.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketConstants.java index 9525d3ebd3b..cbe37927ac4 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketConstants.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketConstants.java @@ -29,8 +29,8 @@ public final class PacketConstants { public static final int PACKET_PROTOCOL_OFFSET = 12; - public static final byte ARP_PROTOCOL = 0; public static final byte ICMP_PROTOCOL = 1; + public static final byte ICMPV6_PROTOCOL = 58; public static final byte TCP_PROTOCOL = 6; public static final byte UDP_PROTOCOL = 17; @@ -52,6 +52,7 @@ public final class PacketConstants { public static final int ETHER_TYPE_OFFSET = 12; public static final int IPv4_TYPE = 0x800; + public static final int ARP_TYPE = 0x806; public static final int IPv6_TYPE = 0x86dd; public static final int PPPoV6_TYPE = 0x8864; diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketDecoder.java index 1794eea4c98..c78604982d9 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketDecoder.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/PacketDecoder.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.store.pcap.decoder; import com.google.common.base.Preconditions; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,7 +75,8 @@ public class PacketDecoder { public PacketDecoder(final InputStream input) throws IOException { this.input = input; byte[] globalHeader = new byte[GLOBAL_HEADER_SIZE]; - int n = input.read(globalHeader); + // Read fully: a compressed stream may return fewer bytes per read + int n = IOUtils.read(input, globalHeader); if (n != globalHeader.length) { throw new IOException("Can't read PCAP file header"); } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java index f67b85f5bb0..df46c9787ab 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/decoder/TcpSession.java @@ -168,11 +168,11 @@ public void addPacket(Packet p) { } // Add the start and ending time stamp. The packets are not necessarily received in order, so we have to check the timestamps this way - if (p.getTimestampMicro() < startTime) { + if (p.getTimestamp() < startTime) { startTime = p.getTimestamp(); } - if (p.getTimestampMicro() > endTime) { + if (p.getTimestamp() > endTime) { endTime = p.getTimestamp(); } @@ -190,6 +190,14 @@ public boolean connectionEstablished() { return handshake.isConnected(); } + public List getPacketsFromSender() { + return Collections.unmodifiableList(packetsFromSender); + } + + public List getPacketsFromReceiver() { + return Collections.unmodifiableList(packetsFromReceiver); + } + public boolean connectionClosed() { return handshake.isClosed(); } @@ -247,8 +255,12 @@ public Period getSessionDuration() { return new Period(endTime - startTime); } + /** + * @return time from SYN to the final handshake ACK, or null if the capture + * did not see both, such as when it started mid-connection + */ public Period getConnectionTime() { - return new Period(connectTime); + return synTime == 0 || ackTime == 0 ? null : new Period(connectTime); } public Instant getSessionEndTime() { diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java index fe2490756ca..77b459623a9 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/BasePcapFormatPlugin.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.pcap.plugin; -import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; @@ -44,7 +43,6 @@ public abstract class BasePcapFormatPlugin extends EasyFormatPlugin { static final Logger logger = LoggerFactory.getLogger(ManagedScanFramework.class); - private static PacketDecoder.FileFormat fileFormat = PacketDecoder.FileFormat.UNKNOWN; public BasePcapFormatPlugin(String name, DrillbitContext context, @@ -86,18 +84,14 @@ public PcapReaderFactory(PcapFormatConfig config, EasySubScan scan) { */ @Override public ManagedReader newReader(FileSchemaNegotiator negotiator) { - if (negotiator.file().fileSystem() != null) { // todo: can be simplified with java9 - // ifPresentOrElse - Path path = scan.getWorkUnits().stream() - .findFirst() - .orElseThrow(() -> UserException. - dataReadError() - .addContext("There are no files for scanning") - .build(logger)) - .getPath(); - fileFormat = getFileFormat(negotiator.file().fileSystem(), path); + // Local, and detected from this reader's own file: readers for + // different files can be created concurrently + PacketDecoder.FileFormat fileFormat = PacketDecoder.FileFormat.UNKNOWN; + if (negotiator.file().fileSystem() != null) { + fileFormat = getFileFormat(negotiator.file().fileSystem(), negotiator.file().split().getPath()); + String formatName = fileFormat.name().toLowerCase(); if (config.getExtensions().stream() - .noneMatch(f -> f.equals(fileFormat.name().toLowerCase()))) { + .noneMatch(f -> f.equals(formatName))) { logger.error("File format {} is not within plugin extensions: {}. Trying to use default PCAP format plugin to " + "read the file", fileFormat, config.getExtensions()); } @@ -105,12 +99,12 @@ public ManagedReader newReader(FileSchemaNegotiator negotiator) { logger.error("It is not possible to detect file format, because the File Framework is not initialized. " + "Trying to use default PCAP format plugin to read the file"); } - return createReader(scan, config, negotiator); + return createReader(fileFormat, scan, config, negotiator); } } - private static ManagedReader createReader(EasySubScan scan, PcapFormatConfig config, - FileSchemaNegotiator negotiator) { + private static ManagedReader createReader(PacketDecoder.FileFormat fileFormat, EasySubScan scan, + PcapFormatConfig config, FileSchemaNegotiator negotiator) { switch(fileFormat) { case PCAPNG: return new PcapngBatchReader(config, scan, negotiator); case PCAP: @@ -134,11 +128,10 @@ private static PacketDecoder.FileFormat getFileFormat(DrillFileSystem dfs, Path try (InputStream inputStream = dfs.openPossiblyCompressedStream(path)) { PacketDecoder decoder = new PacketDecoder(inputStream); return decoder.getFileFormat(); - } catch (IOException io) { - throw UserException - .dataReadError(io) - .addContext("File name:", path.toString()) - .build(logger); + } catch (IOException | RuntimeException e) { + // Not readable or not a capture: the PCAP reader reports it as an error row + logger.debug("Cannot detect the format of {}", path, e); + return PacketDecoder.FileFormat.UNKNOWN; } } } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/PcapFormatConfig.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/PcapFormatConfig.java index 37226b11807..75c3691d86e 100644 --- a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/PcapFormatConfig.java +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/plugin/PcapFormatConfig.java @@ -38,14 +38,21 @@ public class PcapFormatConfig implements FormatPluginConfig { private final List extensions; private final boolean stat; private final boolean sessionizeTCPStreams; + private final boolean exposeCredentials; + + public PcapFormatConfig(List extensions, boolean stat, Boolean sessionizeTCPStreams) { + this(extensions, stat, sessionizeTCPStreams, null); + } @JsonCreator public PcapFormatConfig(@JsonProperty("extensions") List extensions, @JsonProperty("stat") boolean stat, - @JsonProperty("sessionizeTCPStreams") Boolean sessionizeTCPStreams) { + @JsonProperty("sessionizeTCPStreams") Boolean sessionizeTCPStreams, + @JsonProperty("exposeCredentials") Boolean exposeCredentials) { this.extensions = extensions == null ? DEFAULT_EXTNS : ImmutableList.copyOf(extensions); this.stat = stat; this.sessionizeTCPStreams = sessionizeTCPStreams != null && sessionizeTCPStreams; + this.exposeCredentials = exposeCredentials != null && exposeCredentials; } @JsonProperty("extensions") @@ -63,6 +70,12 @@ public boolean getSessionizeTCPStreams() { return sessionizeTCPStreams; } + /** True if protocol decoders may return cleartext passwords. */ + @JsonProperty("exposeCredentials") + public boolean getExposeCredentials() { + return exposeCredentials; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -73,12 +86,13 @@ public boolean equals(Object o) { } PcapFormatConfig that = (PcapFormatConfig) o; return Objects.equals(extensions, that.extensions) && Objects.equals(stat, that.getStat()) && - Objects.equals(sessionizeTCPStreams, that.sessionizeTCPStreams); + Objects.equals(sessionizeTCPStreams, that.sessionizeTCPStreams) && + Objects.equals(exposeCredentials, that.exposeCredentials); } @Override public int hashCode() { - return Objects.hash(extensions, stat, sessionizeTCPStreams); + return Objects.hash(extensions, stat, sessionizeTCPStreams, exposeCredentials); } @Override @@ -87,6 +101,7 @@ public String toString() { .field("extensions", extensions) .field("stat", stat) .field("sessionizeTCPStreams", sessionizeTCPStreams) + .field("exposeCredentials", exposeCredentials) .toString(); } } diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java new file mode 100644 index 00000000000..677c03883aa --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecodeResult.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** The decoder that handled a row, and its parsed value or the reason it failed. */ +public final class DecodeResult { + private final ProtocolDecoder decoder; + private final Object parsed; + private final String error; + + private DecodeResult(ProtocolDecoder decoder, Object parsed, String error) { + this.decoder = decoder; + this.parsed = parsed; + this.error = error; + } + + static DecodeResult success(ProtocolDecoder decoder, Object parsed) { + return new DecodeResult(decoder, parsed, null); + } + + static DecodeResult failure(ProtocolDecoder decoder, String error) { + return new DecodeResult(decoder, null, error); + } + + public String protocol() { + return decoder.protocol(); + } + + /** Null if parsing failed. */ + public Object parsed() { + return parsed; + } + + /** Null if parsing succeeded. */ + public String error() { + return error; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + public void write(TupleWriter fields) { + if (decoder instanceof SessionProtocolDecoder) { + ((SessionProtocolDecoder) decoder).write(parsed, fields); + } else { + ((PacketProtocolDecoder) decoder).write(parsed, fields); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecoderContext.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecoderContext.java new file mode 100644 index 00000000000..a08bbaaa6d6 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/DecoderContext.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +/** What a decoder may use while parsing one packet or session. */ +public interface DecoderContext { + + /** True if decoders may store cleartext passwords (format option exposeCredentials). */ + boolean exposeCredentials(); + + /** + * Notes a problem that did not prevent decoding, such as a truncated list. + * The message appears in decode_error, prefixed with the protocol name. + */ + void warn(String message); +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/PacketProtocolDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/PacketProtocolDecoder.java new file mode 100644 index 00000000000..4e4c81d7c84 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/PacketProtocolDecoder.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** + * Decodes the payload of single packets. Implementations are listed in + * META-INF/services/org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder. + * + * @param the parsed form of one packet + */ +public interface PacketProtocolDecoder extends ProtocolDecoder { + + /** Declares the fields of parsed_data.<protocol>. All fields should be nullable or repeated. */ + void defineSchema(SchemaBuilder fields); + + /** Cheap pre-check, typically transport and port. */ + boolean accepts(Packet packet); + + /** + * Fully validates and parses the payload. + * + * @param payload the TCP or UDP payload; null if the packet has none + * @return null if the payload is not this protocol + * @throws RuntimeException if the payload is this protocol but malformed + */ + T parse(Packet packet, byte[] payload, DecoderContext context); + + /** Writes a value returned by {@link #parse}. Skip null fields. */ + void write(T parsed, TupleWriter fields); +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java new file mode 100644 index 00000000000..b4265786907 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolColumns.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.physical.resultSet.RowSetLoader; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; +import org.apache.drill.exec.vector.accessor.ScalarWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; +import org.slf4j.Logger; + +/** + * The parsed_protocol, parsed_data and decode_error columns, shared by the PCAP and PCAP-NG readers. + */ +public class ProtocolColumns { + public static final String PARSED_PROTOCOL = "parsed_protocol"; + public static final String PARSED_DATA = "parsed_data"; + public static final String DECODE_ERROR = "decode_error"; + + /** Packet rows, session rows, or stat rows (which only get decode_error). */ + public enum Mode { PACKET, SESSION, STAT } + + private final RowSetLoader loader; + private final ProtocolDecoders decoders; + private final RowDecoderContext context; + private final ScalarWriter protocolWriter; + private final TupleWriter dataWriter; + private final ScalarWriter errorWriter; + private final boolean decoding; + // Errors written so far, by prefix (dns, packet, file, ...), for the end-of-file warning + private final Map errorCounts = new TreeMap<>(); + // Errors already written by writeErrorRowOnce + private final Set reportedErrors = new HashSet<>(); + + public static void addColumns(SchemaBuilder schema, Mode mode, ProtocolDecoders decoders) { + if (mode != Mode.STAT) { + schema.addNullable(PARSED_PROTOCOL, MinorType.VARCHAR); + schema.add(MetadataUtils.newMap(PARSED_DATA, dataSchema(mode, decoders))); + } + schema.addNullable(DECODE_ERROR, MinorType.VARCHAR); + } + + /** A copy per reader: map metadata binds its member schema to the parent map. */ + private static TupleMetadata dataSchema(Mode mode, ProtocolDecoders decoders) { + return (mode == Mode.SESSION ? decoders.sessionDataSchema() : decoders.packetDataSchema()).copy(); + } + + public ProtocolColumns(RowSetLoader loader, Mode mode, ProtocolDecoders decoders, boolean exposeCredentials) { + this.loader = loader; + this.decoders = decoders; + this.context = new RowDecoderContext(exposeCredentials); + this.errorWriter = loader.scalar(DECODE_ERROR); + if (mode == Mode.STAT) { + protocolWriter = null; + dataWriter = null; + decoding = false; + } else { + protocolWriter = loader.scalar(PARSED_PROTOCOL); + dataWriter = loader.tuple(PARSED_DATA); + decoding = loader.isProjected(PARSED_PROTOCOL) || loader.isProjected(PARSED_DATA); + } + } + + /** True if parsed_protocol or parsed_data is projected, so decoding is worth doing. */ + public boolean isDecoding() { + return decoding; + } + + /** + * Decodes a packet into the current row and writes decode_error. + * + * @param packet the decoded packet; null if it could not be decoded at all + * @param errors problems the reader already found for this row; may be null + */ + public void writePacket(Packet packet, List errors) { + List all = errors == null ? new ArrayList<>() : new ArrayList<>(errors); + if (decoding && packet != null) { + context.reset(); + byte[] payload = null; + try { + payload = packet.getData(); + } catch (RuntimeException e) { + all.add("packet: " + ProtocolDecoders.describe(e)); + } + write(decoders.decodePacket(packet, payload, context), all); + } + writeErrors(all); + } + + /** Decodes a session into the current row and writes decode_error. */ + public void writeSession(TcpSession session) { + List all = new ArrayList<>(); + if (decoding) { + context.reset(); + write(decoders.decodeSession(session, context), all); + } + writeErrors(all); + } + + void write(DecodeResult result, List all) { + if (result != null) { + protocolWriter.setString(result.protocol()); + if (result.parsed() != null) { + try { + result.write(dataWriter.tuple(result.protocol())); + } catch (RuntimeException e) { + all.add(result.protocol() + ": write failed: " + ProtocolDecoders.describe(e)); + } + } + if (result.error() != null) { + all.add(result.error()); + } + } + all.addAll(context.warnings()); + } + + /** Sets decode_error on the current row; does nothing for an empty list. */ + public void writeErrors(List errors) { + if (errors != null && !errors.isEmpty()) { + errorWriter.setString(String.join("; ", errors)); + errors.forEach(this::count); + } + } + + /** Writes a row with only decode_error set. */ + public void writeErrorRow(String error) { + loader.start(); + errorWriter.setString(error); + loader.save(); + count(error); + } + + /** + * Writes an error row unless the same error was already written. Used where + * packets are not rows (session mode), so each distinct problem appears once. + * + * @return true if a row was written + */ + public boolean writeErrorRowOnce(String error) { + if (!reportedErrors.add(error)) { + count(error); + return false; + } + writeErrorRow(error); + return true; + } + + private void count(String error) { + int colon = error.indexOf(':'); + errorCounts.merge(colon > 0 ? error.substring(0, colon) : error, 1, Integer::sum); + } + + /** Logs one warning for the file if any row had an error, such as "{dns=3, packet=1}". */ + public void logSummary(Logger logger, String file) { + if (!errorCounts.isEmpty()) { + logger.warn("{}: rows with decode_error by kind: {}", file, errorCounts); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoder.java new file mode 100644 index 00000000000..ae0dc15b1ca --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoder.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +/** + * Common part of packet and session protocol decoders. See docs/dev/PcapProtocolDecoders.md. + */ +public interface ProtocolDecoder { + + /** Value of parsed_protocol and the name of this decoder's sub-map in parsed_data. Lower case, unique. */ + String protocol(); + + /** When several decoders accept the same data, higher priority is tried first. */ + default int priority() { + return 0; + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java new file mode 100644 index 00000000000..ed5a83a7169 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/ProtocolDecoders.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; + +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The protocol decoders found with {@link ServiceLoader}, and the matching rules of + * docs/dev/PcapProtocolDecoders.md. + */ +public class ProtocolDecoders { + private static final Logger logger = LoggerFactory.getLogger(ProtocolDecoders.class); + private static volatile ProtocolDecoders instance; + /** Initial width of VARCHAR fields in parsed_data, which most rows leave empty. */ + public static final int SPARSE_WIDTH = 8; + + private final List> packetDecoders; + private final TupleMetadata packetDataSchema; + private final List> sessionDecoders; + private final TupleMetadata sessionDataSchema; + + /** The decoders on the classpath, loaded once. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public static ProtocolDecoders get() { + if (instance == null) { + synchronized (ProtocolDecoders.class) { + if (instance == null) { + ClassLoader loader = ProtocolDecoders.class.getClassLoader(); + instance = new ProtocolDecoders((Iterable) ServiceLoader.load(PacketProtocolDecoder.class, loader), + (Iterable) ServiceLoader.load(SessionProtocolDecoder.class, loader)); + } + } + } + return instance; + } + + public ProtocolDecoders(Iterable> packetDecoders) { + this(packetDecoders, Collections.emptyList()); + } + + public ProtocolDecoders(Iterable> packetDecoders, + Iterable> sessionDecoders) { + SchemaBuilder packetData = new SchemaBuilder(); + this.packetDecoders = usable(packetDecoders, packetData); + this.packetDataSchema = packetData.buildSchema(); + SchemaBuilder sessionData = new SchemaBuilder(); + this.sessionDecoders = usable(sessionDecoders, sessionData); + this.sessionDataSchema = sessionData.buildSchema(); + } + + public List> packetDecoders() { + return packetDecoders; + } + + /** Members of parsed_data in packet mode: one map per decoder. */ + public TupleMetadata packetDataSchema() { + return packetDataSchema; + } + + public List> sessionDecoders() { + return sessionDecoders; + } + + /** Members of parsed_data in session mode: one map per decoder. */ + public TupleMetadata sessionDataSchema() { + return sessionDataSchema; + } + + /** + * Finds the decoder that handles a TCP session. The client and server streams are + * reassembled only if some decoder accepts the session. + * + * @return null if no decoder handles it + */ + public DecodeResult decodeSession(TcpSession session, RowDecoderContext context) { + TcpStream[] streams = null; + for (SessionProtocolDecoder decoder : sessionDecoders) { + try { + if (!decoder.accepts(session)) { + continue; + } + } catch (RuntimeException e) { + continue; + } + context.begin(decoder.protocol()); + try { + if (streams == null) { + streams = TcpStream.clientServer(session); + } + Object parsed = decoder.parse(streams[0], streams[1], context); + if (parsed != null) { + return DecodeResult.success(decoder, parsed); + } + context.discard(); + } catch (RuntimeException e) { + context.discard(); + return DecodeResult.failure(decoder, decoder.protocol() + ": " + describe(e)); + } + } + return null; + } + + /** + * Finds the decoder that handles a packet. + * + * @return null if no decoder handles it + */ + public DecodeResult decodePacket(Packet packet, byte[] payload, RowDecoderContext context) { + for (PacketProtocolDecoder decoder : packetDecoders) { + try { + if (!decoder.accepts(packet)) { + continue; + } + } catch (RuntimeException e) { + // A pre-check that cannot evaluate the packet does not claim it + continue; + } + context.begin(decoder.protocol()); + try { + Object parsed = decoder.parse(packet, payload, context); + if (parsed != null) { + return DecodeResult.success(decoder, parsed); + } + context.discard(); + } catch (RuntimeException e) { + context.discard(); + return DecodeResult.failure(decoder, decoder.protocol() + ": " + describe(e)); + } + } + return null; + } + + public static String describe(Throwable e) { + return e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + } + + /** + * Orders decoders by priority, drops duplicate protocol names and decoders whose schema + * cannot be built, and adds each decoder's map to the parsed_data schema. + */ + private static List usable(Iterable decoders, SchemaBuilder data) { + List sorted = new ArrayList<>(); + decoders.forEach(sorted::add); + sorted.sort(Comparator.comparingInt(ProtocolDecoder::priority).reversed()); + Map byName = new HashMap<>(); + List result = new ArrayList<>(); + for (D decoder : sorted) { + D existing = byName.get(decoder.protocol()); + if (existing != null) { + logger.warn("Ignoring protocol decoder {}: protocol {} is already handled by {}", + decoder.getClass().getName(), decoder.protocol(), existing.getClass().getName()); + continue; + } + try { + SchemaBuilder fields = new SchemaBuilder(); + defineSchema(decoder, fields); + TupleMetadata schema = fields.buildSchema(); + sizeSparse(schema); + data.add(MetadataUtils.newMap(decoder.protocol(), schema)); + } catch (RuntimeException e) { + logger.warn("Ignoring protocol decoder {}: its schema could not be built", decoder.getClass().getName(), e); + continue; + } + byName.put(decoder.protocol(), decoder); + result.add(decoder); + } + return Collections.unmodifiableList(result); + } + + /** + * Starts every vector small. Drill otherwise reserves room for a full batch of + * values per column, and ten elements per array row, which for the many mostly + * empty decoder fields would use up the batch memory budget before any row is + * written. Vectors still grow when a row needs more. + */ + private static void sizeSparse(TupleMetadata schema) { + for (ColumnMetadata column : schema) { + if (column.isArray()) { + column.setExpectedElementCount(1); + } + if (column.isMap()) { + sizeSparse(column.tupleSchema()); + } else if (column.isVariableWidth()) { + column.setExpectedWidth(SPARSE_WIDTH); + } + } + } + + private static void defineSchema(ProtocolDecoder decoder, SchemaBuilder fields) { + if (decoder instanceof SessionProtocolDecoder) { + ((SessionProtocolDecoder) decoder).defineSchema(fields); + } else { + ((PacketProtocolDecoder) decoder).defineSchema(fields); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/RowDecoderContext.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/RowDecoderContext.java new file mode 100644 index 00000000000..669a5b65437 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/RowDecoderContext.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.util.ArrayList; +import java.util.List; + +/** Decoder context for one row: collects warnings, prefixed with the decoder's protocol. */ +public class RowDecoderContext implements DecoderContext { + private final boolean exposeCredentials; + private final List warnings = new ArrayList<>(); + private String protocol; + private int mark; + + public RowDecoderContext(boolean exposeCredentials) { + this.exposeCredentials = exposeCredentials; + } + + @Override + public boolean exposeCredentials() { + return exposeCredentials; + } + + @Override + public void warn(String message) { + warnings.add(protocol + ": " + message); + } + + /** Starts a new row. */ + public void reset() { + warnings.clear(); + } + + public List warnings() { + return warnings; + } + + void begin(String protocol) { + this.protocol = protocol; + mark = warnings.size(); + } + + /** Drops warnings from a decoder that turned out not to handle the row. */ + void discard() { + while (warnings.size() > mark) { + warnings.remove(warnings.size() - 1); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/SessionProtocolDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/SessionProtocolDecoder.java new file mode 100644 index 00000000000..30d1550b356 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/SessionProtocolDecoder.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** + * Decodes reassembled TCP sessions. Implementations are listed in + * META-INF/services/org.apache.drill.exec.store.pcap.protocol.SessionProtocolDecoder. + * + * @param the parsed form of one session + */ +public interface SessionProtocolDecoder extends ProtocolDecoder { + + /** Declares the fields of parsed_data.<protocol> for session rows. */ + void defineSchema(SchemaBuilder fields); + + /** Cheap pre-check, typically ports. */ + boolean accepts(TcpSession session); + + /** + * @return null if the session is not this protocol + * @throws RuntimeException if the session is this protocol but malformed + */ + T parse(TcpStream fromClient, TcpStream fromServer, DecoderContext context); + + void write(T parsed, TupleWriter fields); +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/TcpStream.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/TcpStream.java new file mode 100644 index 00000000000..b97f189254a --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/TcpStream.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.decoder.TcpSession; + +/** + * The reassembled bytes of one direction of a TCP session: ordered by sequence number, + * retransmissions removed, and cut at the first missing range. + */ +public final class TcpStream { + private final byte[] data; + private final long firstGap; + + private TcpStream(byte[] data, long firstGap) { + this.data = data; + this.firstGap = firstGap; + } + + /** Contiguous bytes from the start of the stream up to the first gap. */ + public byte[] data() { + return data; + } + + public boolean hasGaps() { + return firstGap >= 0; + } + + /** Offset of the first missing byte, or -1. */ + public long firstGap() { + return firstGap; + } + + /** The client stream and the server stream of a session, in that order. */ + public static TcpStream[] clientServer(TcpSession session) { + List sender = session.getPacketsFromSender(); + List receiver = session.getPacketsFromReceiver(); + boolean senderIsClient; + if (hasInitialSyn(receiver)) { + senderIsClient = false; + } else if (hasInitialSyn(sender)) { + senderIsClient = true; + } else { + // No handshake captured: the endpoint with the lower port is taken as the server + senderIsClient = session.getSrcPort() > session.getDstPort(); + } + TcpStream fromSender = of(sender); + TcpStream fromReceiver = of(receiver); + return senderIsClient ? new TcpStream[] {fromSender, fromReceiver} : new TcpStream[] {fromReceiver, fromSender}; + } + + private static boolean hasInitialSyn(List packets) { + for (Packet p : packets) { + if (p.getSynFlag() && !p.getAckFlag()) { + return true; + } + } + return false; + } + + public static TcpStream of(List packets) { + if (packets.isEmpty()) { + return new TcpStream(new byte[0], -1); + } + // Sequence numbers relative to the first packet, as signed 32-bit differences, handle wraparound + long reference = packets.get(0).getSequenceNumber() & 0xFFFFFFFFL; + Integer base = null; + for (Packet p : packets) { + if (p.getSynFlag()) { + base = relative(p, reference) + 1; + break; + } + } + List ranges = new ArrayList<>(); + List payloads = new ArrayList<>(); + for (Packet p : packets) { + byte[] payload = p.getData(); + if (payload == null || payload.length == 0) { + continue; + } + ranges.add(new long[] {relative(p, reference), payloads.size()}); + payloads.add(payload); + } + if (payloads.isEmpty()) { + return new TcpStream(new byte[0], -1); + } + ranges.sort((a, b) -> Long.compare(a[0], b[0])); + long start = base != null ? base : ranges.get(0)[0]; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + long next = start; + for (long[] range : ranges) { + byte[] payload = payloads.get((int) range[1]); + long segmentStart = range[0]; + long segmentEnd = segmentStart + payload.length; + if (segmentEnd <= next) { + continue; // retransmission of bytes already taken + } + if (segmentStart > next) { + return new TcpStream(out.toByteArray(), next - start); + } + int skip = (int) (next - segmentStart); + out.write(payload, skip, payload.length - skip); + next = segmentEnd; + } + return new TcpStream(out.toByteArray(), -1); + } + + private static int relative(Packet p, long reference) { + return (int) ((p.getSequenceNumber() & 0xFFFFFFFFL) - reference); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpDecoder.java new file mode 100644 index 00000000000..c8841d2c2d2 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpDecoder.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.arp; + +import java.util.Arrays; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** ARP and RARP (EtherType 0x0806). The message is the link-layer payload. */ +public class ArpDecoder implements PacketProtocolDecoder { + private static final int HEADER = 8; + private static final int ETHERNET = 1; + private static final int IPV4 = 0x0800; + private static final String[] OPERATIONS = {null, "request", "reply", "rarp_request", "rarp_reply", + "drarp_request", "drarp_reply", "drarp_error", "inarp_request", "inarp_reply", "arp_nak"}; + + @Override + public String protocol() { + return "arp"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("hardware_type", MinorType.INT) + .addNullable("protocol_type", MinorType.INT) + .addNullable("operation", MinorType.INT) + .addNullable("operation_name", MinorType.VARCHAR) + .addNullable("sender_mac", MinorType.VARCHAR) + .addNullable("sender_ip", MinorType.VARCHAR) + .addNullable("target_mac", MinorType.VARCHAR) + .addNullable("target_ip", MinorType.VARCHAR) + .addNullable("is_gratuitous", MinorType.BIT) + .addNullable("is_probe", MinorType.BIT); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isArpPacket(); + } + + @Override + public ArpMessage parse(Packet packet, byte[] payload, DecoderContext context) { + // payload is the TCP or UDP payload, null here; the ARP message is the link payload + return parse(packet.getLinkPayload()); + } + + /** + * @param b the ARP message, possibly followed by link-layer padding + * @return null if the address lengths in the header do not fit the message + * @throws IllegalArgumentException if an Ethernet/IPv4 message is cut short + */ + public static ArpMessage parse(byte[] b) { + if (b == null || b.length < HEADER) { + return null; + } + int hardwareType = u16(b, 0); + int protocolType = u16(b, 2); + int hardwareLength = b[4] & 0xFF; + int protocolLength = b[5] & 0xFF; + int needed = HEADER + 2 * (hardwareLength + protocolLength); + if (hardwareLength == 0 || protocolLength == 0) { + return null; + } + if (needed > b.length) { + if (hardwareType == ETHERNET && protocolType == IPV4 && hardwareLength == 6 && protocolLength == 4) { + throw new IllegalArgumentException(String.format("truncated: %d bytes, needs %d", b.length, needed)); + } + return null; + } + ArpMessage m = new ArpMessage(); + m.hardwareType = hardwareType; + m.protocolType = protocolType; + m.operation = u16(b, 6); + m.operationName = m.operation < OPERATIONS.length ? OPERATIONS[m.operation] : null; + int senderHardware = HEADER; + int senderProtocol = senderHardware + hardwareLength; + int targetHardware = senderProtocol + protocolLength; + int targetProtocol = targetHardware + hardwareLength; + boolean ipv4 = protocolType == IPV4 && protocolLength == 4; + m.senderMac = hardware(b, senderHardware, hardwareLength); + m.senderIp = protocolAddress(b, senderProtocol, protocolLength, ipv4); + m.targetMac = hardware(b, targetHardware, hardwareLength); + m.targetIp = protocolAddress(b, targetProtocol, protocolLength, ipv4); + m.isGratuitous = Arrays.equals(Arrays.copyOfRange(b, senderProtocol, senderProtocol + protocolLength), + Arrays.copyOfRange(b, targetProtocol, targetProtocol + protocolLength)); + m.isProbe = true; + for (int i = senderProtocol; i < senderProtocol + protocolLength; i++) { + if (b[i] != 0) { + m.isProbe = false; + break; + } + } + return m; + } + + /** Colon-separated upper-case hex, like the eth_src column. */ + private static String hardware(byte[] b, int offset, int length) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (i > 0) { + sb.append(':'); + } + sb.append(String.format("%02X", b[offset + i] & 0xFF)); + } + return sb.toString(); + } + + /** Dotted quad for IPv4, otherwise hex. */ + private static String protocolAddress(byte[] b, int offset, int length, boolean ipv4) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (ipv4) { + if (i > 0) { + sb.append('.'); + } + sb.append(b[offset + i] & 0xFF); + } else { + sb.append(String.format("%02x", b[offset + i] & 0xFF)); + } + } + return sb.toString(); + } + + private static int u16(byte[] b, int offset) { + return ((b[offset] & 0xFF) << 8) | (b[offset + 1] & 0xFF); + } + + @Override + public void write(ArpMessage m, TupleWriter fields) { + fields.scalar("hardware_type").setInt(m.hardwareType); + fields.scalar("protocol_type").setInt(m.protocolType); + fields.scalar("operation").setInt(m.operation); + if (m.operationName != null) { + fields.scalar("operation_name").setString(m.operationName); + } + fields.scalar("sender_mac").setString(m.senderMac); + fields.scalar("sender_ip").setString(m.senderIp); + fields.scalar("target_mac").setString(m.targetMac); + fields.scalar("target_ip").setString(m.targetIp); + fields.scalar("is_gratuitous").setBoolean(m.isGratuitous); + fields.scalar("is_probe").setBoolean(m.isProbe); + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpMessage.java new file mode 100644 index 00000000000..87bb2be0cf5 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/arp/ArpMessage.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.arp; + +/** The fields of one ARP message. */ +public class ArpMessage { + public int hardwareType; + public int protocolType; + public int operation; + /** Null for an unknown operation. */ + public String operationName; + public String senderMac; + public String senderIp; + public String targetMac; + public String targetIp; + public boolean isGratuitous; + public boolean isProbe; +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/asn1/Asn1Reader.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/asn1/Asn1Reader.java new file mode 100644 index 00000000000..c857642860e --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/asn1/Asn1Reader.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.asn1; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Arrays; + +/** + * A bounded reader of ASN.1 BER/DER elements shared by the Kerberos and LDAP decoders. It reads + * single-byte tags (universal, application and context classes) and definite lengths only, checks every + * length against the enclosing element, and limits nesting. BER lengths come from untrusted capture data, + * so every read is bounds checked. + * + *

Modelled on the SNMP decoder's package-private {@code BerReader}, promoted to a shared, public helper + * with value accessors (integers, strings, OIDs, GeneralizedTime) that both ASN.1 decoders need. + */ +public final class Asn1Reader { + public static final int MAX_DEPTH = 32; + public static final int MAX_STRING = 4096; + + // Universal tags. + public static final int BOOLEAN = 0x01; + public static final int INTEGER = 0x02; + public static final int BIT_STRING = 0x03; + public static final int OCTET_STRING = 0x04; + public static final int NULL = 0x05; + public static final int OID = 0x06; + public static final int ENUMERATED = 0x0A; + public static final int UTF8_STRING = 0x0C; + public static final int SEQUENCE = 0x30; + public static final int SET = 0x31; + public static final int GENERAL_STRING = 0x1B; + public static final int GENERALIZED_TIME = 0x18; + + /** One element: its tag, and the range of its content. */ + public static final class Element { + public final int tag; + public final int start; + public final int length; + + Element(int tag, int start, int length) { + this.tag = tag; + this.start = start; + this.length = length; + } + + public int end() { + return start + length; + } + + /** The low five bits of the tag: the universal type, or the application/context number. */ + public int number() { + return tag & 0x1F; + } + + public boolean isConstructed() { + return (tag & 0x20) != 0; + } + } + + private final byte[] b; + private final int limit; + private final int depth; + private int pos; + + public Asn1Reader(byte[] b, int start, int limit, int depth) { + if (depth > MAX_DEPTH) { + throw new IllegalArgumentException("nesting deeper than " + MAX_DEPTH); + } + this.b = b; + this.pos = start; + this.limit = limit; + this.depth = depth; + } + + public byte[] bytes() { + return b; + } + + public boolean hasMore() { + return pos < limit; + } + + /** Reads the next element and moves past it. */ + public Element next() { + Element e = header(b, pos, limit); + pos = e.end(); + return e; + } + + /** Reads the next element, which must have the given tag. */ + public Element expect(int tag, String what) { + if (!hasMore()) { + throw new IllegalArgumentException("missing " + what); + } + Element e = next(); + if (e.tag != tag) { + throw new IllegalArgumentException( + String.format("expected %s (tag 0x%02x) but found tag 0x%02x", what, tag, e.tag)); + } + return e; + } + + /** A reader over the content of a constructed element. */ + public Asn1Reader enter(Element e) { + return new Asn1Reader(b, e.start, e.end(), depth + 1); + } + + /** + * Reads a tag and length at {@code at}, checking that the content fits before {@code limit}. + * + * @throws IllegalArgumentException if the header is invalid or the content overruns the limit + */ + public static Element header(byte[] b, int at, int limit) { + if (at + 2 > limit) { + throw new IllegalArgumentException("truncated element at offset " + at); + } + int tag = b[at] & 0xFF; + if ((tag & 0x1F) == 0x1F) { + throw new IllegalArgumentException("unsupported multi-byte tag at offset " + at); + } + int first = b[at + 1] & 0xFF; + int p = at + 2; + long length; + if (first < 0x80) { + length = first; + } else { + int count = first & 0x7F; + if (count == 0) { + throw new IllegalArgumentException("indefinite length at offset " + at); + } + if (count > 4 || p + count > limit) { + throw new IllegalArgumentException("bad length encoding at offset " + at); + } + length = 0; + for (int i = 0; i < count; i++) { + length = (length << 8) | (b[p + i] & 0xFF); + } + p += count; + } + if (p + length > limit) { + throw new IllegalArgumentException("length " + length + " at offset " + at + " overruns its container"); + } + return new Element(tag, p, (int) length); + } + + /** Reads a signed integer, which must be no longer than eight bytes. */ + public long integer(Element e, String what) { + if (e.length < 1 || e.length > 8) { + throw new IllegalArgumentException(what + " has bad integer length " + e.length); + } + return new BigInteger(Arrays.copyOfRange(b, e.start, e.end())).longValue(); + } + + /** Reads an integer that must fit in an int. */ + public int intValue(Element e, String what) { + long v = integer(e, what); + if (v < Integer.MIN_VALUE || v > Integer.MAX_VALUE) { + throw new IllegalArgumentException(what + " out of range: " + v); + } + return (int) v; + } + + public boolean booleanValue(Element e) { + for (int i = e.start; i < e.end(); i++) { + if (b[i] != 0) { + return true; + } + } + return false; + } + + /** The element's content as a UTF-8 string, capped at {@link #MAX_STRING} characters. */ + public String string(Element e) { + return cap(new String(b, e.start, Math.min(e.length, MAX_STRING), StandardCharsets.UTF_8)); + } + + /** The element's content as lower-case hex, capped. */ + public String hex(Element e) { + StringBuilder out = new StringBuilder(); + for (int i = e.start; i < e.end() && out.length() < MAX_STRING; i++) { + out.append(String.format("%02x", b[i])); + } + return out.toString(); + } + + /** + * Parses a GeneralizedTime (such as a Kerberos {@code YYYYMMDDHHMMSSZ}) into an Instant, or null if it + * is not in the expected form. Never throws: a malformed time is simply absent. + */ + public Instant generalizedTime(Element e) { + String s = new String(b, e.start, Math.min(e.length, 32), StandardCharsets.US_ASCII); + if (s.length() < 14) { + return null; + } + try { + int year = Integer.parseInt(s.substring(0, 4)); + int month = Integer.parseInt(s.substring(4, 6)); + int day = Integer.parseInt(s.substring(6, 8)); + int hour = Integer.parseInt(s.substring(8, 10)); + int minute = Integer.parseInt(s.substring(10, 12)); + int second = Integer.parseInt(s.substring(12, 14)); + return LocalDateTime.of(year, month, day, hour, minute, second).toInstant(ZoneOffset.UTC); + } catch (RuntimeException ex) { + return null; + } + } + + private static String cap(String s) { + return s.length() > MAX_STRING ? s.substring(0, MAX_STRING) : s; + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpDecoder.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpDecoder.java new file mode 100644 index 00000000000..95a71f4db79 --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpDecoder.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.dhcp; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.SchemaBuilder; +import org.apache.drill.exec.store.pcap.decoder.Packet; +import org.apache.drill.exec.store.pcap.protocol.DecoderContext; +import org.apache.drill.exec.store.pcap.protocol.PacketProtocolDecoder; +import org.apache.drill.exec.vector.accessor.ArrayWriter; +import org.apache.drill.exec.vector.accessor.ScalarWriter; +import org.apache.drill.exec.vector.accessor.TupleWriter; + +/** + * DHCP (RFC 2131, options from RFC 2132) over UDP ports 67 and 68. A payload is DHCP only if op is 1 or 2, + * the hardware type is not 0, the hardware address length is at most 16 and the magic cookie follows the fixed BOOTP header. Plain + * BOOTP without the cookie is left undecoded. + */ +public class DhcpDecoder implements PacketProtocolDecoder { + static final int MAX_ITEMS = 64; + private static final int HEADER = 236; + private static final int MAGIC_COOKIE = 0x63825363; + private static final String[] MESSAGE_TYPES = {null, "DISCOVER", "OFFER", "REQUEST", "DECLINE", "ACK", "NAK", + "RELEASE", "INFORM", "FORCERENEW", "LEASEQUERY", "LEASEUNASSIGNED", "LEASEUNKNOWN", "LEASEACTIVE"}; + + @Override + public String protocol() { + return "dhcp"; + } + + @Override + public void defineSchema(SchemaBuilder fields) { + fields.addNullable("op", MinorType.VARCHAR) + .addNullable("message_type", MinorType.VARCHAR) + .addNullable("transaction_id", MinorType.BIGINT) + .addNullable("broadcast", MinorType.BIT) + .addNullable("client_mac", MinorType.VARCHAR) + .addNullable("client_ip", MinorType.VARCHAR) + .addNullable("your_ip", MinorType.VARCHAR) + .addNullable("server_ip", MinorType.VARCHAR) + .addNullable("relay_ip", MinorType.VARCHAR) + .addNullable("server_name", MinorType.VARCHAR) + .addNullable("boot_file", MinorType.VARCHAR) + .addNullable("hostname", MinorType.VARCHAR) + .addNullable("requested_ip", MinorType.VARCHAR) + .addNullable("server_id", MinorType.VARCHAR) + .addNullable("lease_time", MinorType.BIGINT) + .addNullable("subnet_mask", MinorType.VARCHAR) + .addNullable("vendor_class", MinorType.VARCHAR) + .addNullable("domain_name", MinorType.VARCHAR) + .addArray("routers", MinorType.VARCHAR) + .addArray("dns_servers", MinorType.VARCHAR) + .addArray("parameter_request_list", MinorType.INT) + .addMapArray("options") + .addNullable("code", MinorType.INT) + .addNullable("value", MinorType.VARCHAR) + .resumeSchema(); + } + + @Override + public boolean accepts(Packet packet) { + return packet.isUdpPacket() && (isDhcpPort(packet.getSrc_port()) || isDhcpPort(packet.getDst_port())); + } + + private static boolean isDhcpPort(int port) { + return port == 67 || port == 68; + } + + @Override + public DhcpMessage parse(Packet packet, byte[] payload, DecoderContext context) { + if (payload == null || payload.length < HEADER + 4) { + return null; + } + int op = payload[0] & 0xFF; + int htype = payload[1] & 0xFF; + int hlen = payload[2] & 0xFF; + if ((op != 1 && op != 2) || htype == 0 || hlen > 16 || u32(payload, HEADER) != MAGIC_COOKIE) { + return null; + } + DhcpMessage m = new DhcpMessage(); + m.op = op == 1 ? "request" : "reply"; + m.transactionId = u32(payload, 4) & 0xFFFFFFFFL; + m.broadcast = (payload[10] & 0x80) != 0; + m.clientIp = ipv4(payload, 12); + m.yourIp = ipv4(payload, 16); + m.serverIp = ipv4(payload, 20); + m.relayIp = ipv4(payload, 24); + m.clientMac = hlen == 0 ? null : mac(payload, 28, hlen); + int overload = options(payload, m, context); + // Option 52 says sname and/or file carry options instead of strings + if ((overload & 2) == 0) { + m.serverName = cString(payload, 44, 64); + } + if ((overload & 1) == 0) { + m.bootFile = cString(payload, 108, 128); + } + return m; + } + + /** Walks the options after the magic cookie; returns the value of option 52 (overload), or 0. */ + private static int options(byte[] b, DhcpMessage m, DecoderContext context) { + int overload = 0; + int pos = HEADER + 4; + int count = 0; + while (pos < b.length) { + int code = b[pos] & 0xFF; + if (code == 255) { + break; + } + if (code == 0) { + pos++; + continue; + } + if (pos + 1 >= b.length) { + throw new IllegalArgumentException("option " + code + " at offset " + pos + " has no length"); + } + int len = b[pos + 1] & 0xFF; + int at = pos + 2; + if (at + len > b.length) { + throw new IllegalArgumentException("option " + code + " at offset " + pos + " overruns the message"); + } + if (count++ < MAX_ITEMS) { + m.options.add(new DhcpMessage.Option(code, hex(b, at, len))); + } else if (count == MAX_ITEMS + 1) { + context.warn("options truncated to " + MAX_ITEMS); + } + switch (code) { + case 1: + m.subnetMask = ipv4(b, fixed(code, at, len, 4)); + break; + case 3: + addresses(b, code, at, len, m.routers, context); + break; + case 6: + addresses(b, code, at, len, m.dnsServers, context); + break; + case 12: + m.hostname = text(b, at, len); + break; + case 15: + m.domainName = text(b, at, len); + break; + case 50: + m.requestedIp = ipv4(b, fixed(code, at, len, 4)); + break; + case 51: + m.leaseTime = u32(b, fixed(code, at, len, 4)) & 0xFFFFFFFFL; + break; + case 52: + overload = b[fixed(code, at, len, 1)] & 0x03; + break; + case 53: + int type = b[fixed(code, at, len, 1)] & 0xFF; + m.messageType = type < MESSAGE_TYPES.length && MESSAGE_TYPES[type] != null + ? MESSAGE_TYPES[type] : String.valueOf(type); + break; + case 54: + m.serverId = ipv4(b, fixed(code, at, len, 4)); + break; + case 55: + for (int i = 0; i < len && m.parameterRequestList.size() < MAX_ITEMS; i++) { + m.parameterRequestList.add(b[at + i] & 0xFF); + } + break; + case 60: + m.vendorClass = text(b, at, len); + break; + default: + break; + } + pos = at + len; + } + return overload; + } + + private static int fixed(int code, int at, int len, int expected) { + if (len != expected) { + throw new IllegalArgumentException("option " + code + " has length " + len); + } + return at; + } + + private static void addresses(byte[] b, int code, int at, int len, List out, DecoderContext context) { + if (len == 0 || len % 4 != 0) { + throw new IllegalArgumentException("option " + code + " has length " + len); + } + for (int i = 0; i < len; i += 4) { + if (out.size() == MAX_ITEMS) { + context.warn("option " + code + " addresses truncated to " + MAX_ITEMS); + return; + } + out.add(ipv4(b, at + i)); + } + } + + /** Option text, without the trailing NULs some clients send; null if empty. */ + private static String text(byte[] b, int at, int len) { + int end = at + len; + while (end > at && b[end - 1] == 0) { + end--; + } + return end == at ? null : new String(b, at, end - at, StandardCharsets.UTF_8); + } + + /** A NUL-terminated string in a fixed-size field; null if empty. */ + private static String cString(byte[] b, int at, int size) { + int end = at; + while (end < at + size && b[end] != 0) { + end++; + } + return end == at ? null : new String(b, at, end - at, StandardCharsets.ISO_8859_1); + } + + private static String ipv4(byte[] b, int at) { + return (b[at] & 0xFF) + "." + (b[at + 1] & 0xFF) + "." + (b[at + 2] & 0xFF) + "." + (b[at + 3] & 0xFF); + } + + private static String mac(byte[] b, int at, int len) { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < len; i++) { + if (i > 0) { + out.append(':'); + } + out.append(String.format("%02x", b[at + i])); + } + return out.toString(); + } + + private static String hex(byte[] b, int at, int len) { + StringBuilder out = new StringBuilder(); + for (int i = at; i < at + len; i++) { + out.append(String.format("%02x", b[i])); + } + return out.toString(); + } + + private static int u32(byte[] b, int at) { + return ((b[at] & 0xFF) << 24) | ((b[at + 1] & 0xFF) << 16) | ((b[at + 2] & 0xFF) << 8) | (b[at + 3] & 0xFF); + } + + @Override + public void write(DhcpMessage m, TupleWriter fields) { + set(fields, "op", m.op); + set(fields, "message_type", m.messageType); + fields.scalar("transaction_id").setLong(m.transactionId); + fields.scalar("broadcast").setBoolean(m.broadcast); + set(fields, "client_mac", m.clientMac); + set(fields, "client_ip", m.clientIp); + set(fields, "your_ip", m.yourIp); + set(fields, "server_ip", m.serverIp); + set(fields, "relay_ip", m.relayIp); + set(fields, "server_name", m.serverName); + set(fields, "boot_file", m.bootFile); + set(fields, "hostname", m.hostname); + set(fields, "requested_ip", m.requestedIp); + set(fields, "server_id", m.serverId); + if (m.leaseTime != null) { + fields.scalar("lease_time").setLong(m.leaseTime); + } + set(fields, "subnet_mask", m.subnetMask); + set(fields, "vendor_class", m.vendorClass); + set(fields, "domain_name", m.domainName); + ScalarWriter routers = fields.array("routers").scalar(); + m.routers.forEach(routers::setString); + ScalarWriter dns = fields.array("dns_servers").scalar(); + m.dnsServers.forEach(dns::setString); + ScalarWriter prl = fields.array("parameter_request_list").scalar(); + m.parameterRequestList.forEach(prl::setInt); + ArrayWriter options = fields.array("options"); + for (DhcpMessage.Option o : m.options) { + TupleWriter t = options.tuple(); + t.scalar("code").setInt(o.code); + t.scalar("value").setString(o.value); + options.save(); + } + } + + private static void set(TupleWriter fields, String name, String value) { + if (value != null) { + fields.scalar(name).setString(value); + } + } +} diff --git a/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpMessage.java b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpMessage.java new file mode 100644 index 00000000000..81c54a8030e --- /dev/null +++ b/contrib/format-pcapng/src/main/java/org/apache/drill/exec/store/pcap/protocol/dhcp/DhcpMessage.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.pcap.protocol.dhcp; + +import java.util.ArrayList; +import java.util.List; + +/** The parsed fields of one DHCP message. */ +public class DhcpMessage { + String op; + String messageType; + long transactionId; + boolean broadcast; + String clientMac; + String clientIp; + String yourIp; + String serverIp; + String relayIp; + String serverName; + String bootFile; + String hostname; + String requestedIp; + String serverId; + Long leaseTime; + String subnetMask; + String vendorClass; + String domainName; + final List routers = new ArrayList<>(); + final List dnsServers = new ArrayList<>(); + final List parameterRequestList = new ArrayList<>(); + final List