From c5440118ad0dbd6e7a65e09c7c7dde862f965873 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 14 Jul 2026 10:48:42 +0200 Subject: [PATCH 1/2] [WARC] Format WARC-IP-Address header values following the WARC community recommendations --- .../stormcrawler/warc/WARCRecordFormat.java | 46 ++++++++++++++- .../warc/WARCRequestRecordFormat.java | 8 +-- .../warc/WARCRecordFormatTest.java | 58 ++++++++++++++++++- 3 files changed, 104 insertions(+), 8 deletions(-) diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index 4555fac3c..cee46c8fb 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -21,6 +21,7 @@ import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_HEADERS_KEY; import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY; +import com.google.common.net.InetAddresses; import java.net.URI; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -32,6 +33,7 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.UUID; import java.util.regex.Pattern; import org.apache.commons.codec.binary.Base32; @@ -317,6 +319,44 @@ protected String getCaptureTime(Metadata metadata) { return WARC_DF.format(capturedAt); } + /** + * Get the canonicalized IP address, if stored as protocol metadata. + * + * + * + * See also + * + * + * + * @param metadata + * @param protocolMDprefix + * @return The canonical IP address string, or empty if the IP address was not recorded in + * metadata. + */ + protected static Optional getIPAddress(Metadata metadata, String protocolMDprefix) { + String ip = metadata.getFirstValue(RESPONSE_IP_KEY, protocolMDprefix); + if (StringUtils.isBlank(ip)) { + return Optional.empty(); + } + try { + return Optional.of(InetAddresses.toAddrString(InetAddresses.forString(ip))); + } catch (IllegalArgumentException e) { + return Optional.empty(); + } + } + @Override public byte[] format(Tuple tuple) { @@ -383,9 +423,9 @@ public byte[] format(Tuple tuple) { buffer.append("WARC-Type").append(": ").append(WARCTypeValue).append(CRLF); // "WARC-IP-Address" if present - String IP = metadata.getFirstValue(RESPONSE_IP_KEY, this.protocolMDprefix); - if (StringUtils.isNotBlank(IP)) { - buffer.append("WARC-IP-Address").append(": ").append(IP).append(CRLF); + Optional ip = getIPAddress(metadata, this.protocolMDprefix); + if (ip.isPresent()) { + buffer.append("WARC-IP-Address").append(": ").append(ip.get()).append(CRLF); } // must be a valid URI diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java index 1b4833fba..da35d1af0 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java @@ -18,10 +18,10 @@ package org.apache.stormcrawler.warc; import static org.apache.stormcrawler.protocol.ProtocolResponse.REQUEST_HEADERS_KEY; -import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.util.Optional; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; import org.apache.storm.tuple.Tuple; @@ -65,9 +65,9 @@ public byte[] format(Tuple tuple) { buffer.append("WARC-Type: ").append(WARC_TYPE_REQUEST).append(CRLF); // "WARC-IP-Address" if present - String IP = metadata.getFirstValue(RESPONSE_IP_KEY, this.protocolMDprefix); - if (StringUtils.isNotBlank(IP)) { - buffer.append("WARC-IP-Address: ").append(IP).append(CRLF); + Optional ip = getIPAddress(metadata, this.protocolMDprefix); + if (ip.isPresent()) { + buffer.append("WARC-IP-Address").append(": ").append(ip.get()).append(CRLF); } addRecordID(buffer); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java index 6c66204d7..045579a89 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Optional; import org.apache.http.HttpHeaders; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Metadata; @@ -38,7 +39,42 @@ class WARCRecordFormatTest { - private String protocolMDprefix = "http."; + private static String protocolMDprefix = "http."; + + private static String[][] ipAddressTestData = { + // { expected, input } + // IPv4 + {"127.0.0.1", "127.0.0.1"}, + // IPv6 + { + // IPv4-mapped IPv6 addresses + "192.0.2.128", "::ffff:192.0.2.128" + }, + { + // IPv4-mapped IPv6 addresses (hex) + "192.0.2.128", "::ffff:C000:0280" + }, + // Tests below are from jwarc's InetAddressesTest + {"2001:db8::1", "2001:db8:0:0:0:0:0:1"}, + {"2001:db8::1", "2001:0DB8:0:0:0:0:0:1"}, + {"::", "0:0:0:0:0:0:0:0"}, + {"::1", "0:0:0:0:0:0:0:1"}, + {"2001:db8:1:1:1:1:1:1", "2001:db8:1:1:1:1:1:1"}, + {"2001:0:0:1::1", "2001:0:0:1:0:0:0:1"}, + {"2001:db8:f::1", "2001:db8:000f:0:0:0:0:1"}, + {"2001:db8::1:0:0:1", "2001:0db8:0000:0000:0001:0000:0000:0001"}, + {"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, + {"2001:200f::1", "2001:200f:0:0:0:0:0:1"}, + { + // https://datatracker.ietf.org/doc/html/rfc5952#section-4.2.2 + // "The symbol "::" MUST NOT be used to shorten just one 16-bit 0 field." + "2001:0:3:4:5:6:7:8", "2001:0:3:4:5:6:7:8" + }, + { + // shorten first of same-length consecutive 0 fields, also in initial position + "::4:0:0:0:ffff", "0:0:0:4:0:0:0:ffff" + }, + }; @Test void testGetDigestSha1() { @@ -72,6 +108,21 @@ void testGetDigestSha1RobotsTxt() { assertEquals(sha1str, WARCRecordFormat.getDigestSha1(content), "Wrong sha1 digest"); } + @Test + void testCanonicalizeIpAddress() { + Metadata metadata = new Metadata(); + for (String[] p : ipAddressTestData) { + metadata.setValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, p[1]); + Optional ip = WARCRecordFormat.getIPAddress(metadata, protocolMDprefix); + if (p[0] == null) { + assertTrue(ip.isEmpty()); + } else { + assertTrue(ip.isPresent(), "No canonical form for " + p[1]); + assertEquals(p[0], ip.get()); + } + } + } + @Test void testWarcRecord() { // test validity of WARC record @@ -82,6 +133,8 @@ void testWarcRecord() { metadata.addValue( protocolMDprefix + ProtocolResponse.RESPONSE_HEADERS_KEY, "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n"); + metadata.addValue( + protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "2001:0DB8:0:0:0:0:0:1"); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); when(tuple.getStringByField("url")).thenReturn("https://www.example.org/"); @@ -111,6 +164,9 @@ void testWarcRecord() { assertTrue( warcString.contains("\r\nWARC-Payload-Digest: " + sha1str + "\r\n"), "WARC record: no or incorrect payload digest"); + assertTrue( + warcString.contains("\r\nWARC-IP-Address: 2001:db8::1\r\n"), + "WARC record: WARC-IP-Address not found (canonical form expected)"); } @Test From 0c7a0aa9e04f481066eb5674c606397d9929ae6a Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 14 Jul 2026 17:38:33 +0200 Subject: [PATCH 2/2] [WARC] Format WARC-IP-Address header values following the WARC community recommendations WARC module: add Guava dependency. --- core/pom.xml | 1 - external/warc/pom.xml | 6 ++++++ pom.xml | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/pom.xml b/core/pom.xml index 6d12c4277..9a4f873f7 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -50,7 +50,6 @@ under the License. 5.4.0 3.2.4 - 33.6.0-jre true 0.72 0.54 diff --git a/external/warc/pom.xml b/external/warc/pom.xml index 694770298..32bc2c35f 100644 --- a/external/warc/pom.xml +++ b/external/warc/pom.xml @@ -89,6 +89,12 @@ under the License. ${jwarc.version} + + com.google.guava + guava + ${guava.version} + + org.apache.stormcrawler stormcrawler-core diff --git a/pom.xml b/pom.xml index bbaade1d0..98dea042a 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,7 @@ under the License. 2.22.0 1.28.0 1.22.0 + 33.6.0-jre 5.4 1.21.4 4.3.0