[WARC] Format WARC-IP-Address header values following the WARC community recommendations

core/pom.xml

@@ -50,7 +50,6 @@ under the License.
         <okhttp.version>5.4.0</okhttp.version>
         <caffeine.version>3.2.4</caffeine.version>
 
-        <guava.version>33.6.0-jre</guava.version>
         <jacoco.haltOnFailure>true</jacoco.haltOnFailure>
         <jacoco.classRatio>0.72</jacoco.classRatio>
         <jacoco.instructionRatio>0.54</jacoco.instructionRatio>

external/warc/pom.xml

@@ -89,6 +89,12 @@ under the License.
             <version>${jwarc.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.stormcrawler</groupId>
             <artifactId>stormcrawler-core</artifactId>

external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java

@@ -21,6 +21,7 @@
 import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_HEADERS_KEY;
 import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY;
 
+import com.google.common.net.InetAddresses;
 import java.net.URI;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
@@ -32,6 +33,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.UUID;
 import java.util.regex.Pattern;
 import org.apache.commons.codec.binary.Base32;
@@ -317,6 +319,44 @@ protected String getCaptureTime(Metadata metadata) {
         return WARC_DF.format(capturedAt);
     }
 
+    /**
+     * Get the canonicalized IP address, if stored as protocol metadata.
+     *
+     * <ul>
+     *   <li>IPv4: dot-decimal notation (dotted quad)
+     *   <li>IPv6: the canonical format (short representation) defined by <a
+     *       href="https://datatracker.ietf.org/doc/html/rfc5952">RFC 5952</a>
+     * </ul>
+     *
+     * See also
+     *
+     * <ul>
+     *   <li><a
+     *       href="https://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1-annotated/#warc-ip-address">WARC-IP-Address,
+     *       WARC specification 1.1</a>
+     *   <li><a href="https://en.wikipedia.org/wiki/Dot-decimal_notation">IPv4 address, dot-decimal
+     *       notation</a>
+     *   <li><a href="https://en.wikipedia.org/wiki/IPv6_address#Representation">IPv6 address
+     *       representation, Wikipedia</a>
+     * </ul>
+     *
+     * @param metadata
+     * @param protocolMDprefix
+     * @return The canonical IP address string, or empty if the IP address was not recorded in
+     *     metadata.
+     */
+    protected static Optional<String> getIPAddress(Metadata metadata, String protocolMDprefix) {
+        String ip = metadata.getFirstValue(RESPONSE_IP_KEY, protocolMDprefix);
+        if (StringUtils.isBlank(ip)) {
+            return Optional.empty();
+        }
+        try {
+            return Optional.of(InetAddresses.toAddrString(InetAddresses.forString(ip)));
+        } catch (IllegalArgumentException e) {
+            return Optional.empty();
+        }
+    }
+
     @Override
     public byte[] format(Tuple tuple) {
 
@@ -383,9 +423,9 @@ public byte[] format(Tuple tuple) {
         buffer.append("WARC-Type").append(": ").append(WARCTypeValue).append(CRLF);
 
         // "WARC-IP-Address" if present
-        String IP = metadata.getFirstValue(RESPONSE_IP_KEY, this.protocolMDprefix);
-        if (StringUtils.isNotBlank(IP)) {
-            buffer.append("WARC-IP-Address").append(": ").append(IP).append(CRLF);
+        Optional<String> ip = getIPAddress(metadata, this.protocolMDprefix);
+        if (ip.isPresent()) {
+            buffer.append("WARC-IP-Address").append(": ").append(ip.get()).append(CRLF);
         }
 
         // must be a valid URI

external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java

@@ -18,10 +18,10 @@
 package org.apache.stormcrawler.warc;
 
 import static org.apache.stormcrawler.protocol.ProtocolResponse.REQUEST_HEADERS_KEY;
-import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY;
 
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.util.Optional;
 import java.util.regex.Pattern;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.storm.tuple.Tuple;
@@ -65,9 +65,9 @@ public byte[] format(Tuple tuple) {
         buffer.append("WARC-Type: ").append(WARC_TYPE_REQUEST).append(CRLF);
 
         // "WARC-IP-Address" if present
-        String IP = metadata.getFirstValue(RESPONSE_IP_KEY, this.protocolMDprefix);
-        if (StringUtils.isNotBlank(IP)) {
-            buffer.append("WARC-IP-Address: ").append(IP).append(CRLF);
+        Optional<String> ip = getIPAddress(metadata, this.protocolMDprefix);
+        if (ip.isPresent()) {
+            buffer.append("WARC-IP-Address").append(": ").append(ip.get()).append(CRLF);
         }
 
         addRecordID(buffer);

external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java

@@ -27,6 +27,7 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
+import java.util.Optional;
 import org.apache.http.HttpHeaders;
 import org.apache.storm.tuple.Tuple;
 import org.apache.stormcrawler.Metadata;
@@ -38,7 +39,42 @@
 
 class WARCRecordFormatTest {
 
-    private String protocolMDprefix = "http.";
+    private static String protocolMDprefix = "http.";
+
+    private static String[][] ipAddressTestData = {
+        // { expected, input }
+        // IPv4
+        {"127.0.0.1", "127.0.0.1"},
+        // IPv6
+        {
+            // IPv4-mapped IPv6 addresses
+            "192.0.2.128", "::ffff:192.0.2.128"
+        },
+        {
+            // IPv4-mapped IPv6 addresses (hex)
+            "192.0.2.128", "::ffff:C000:0280"
+        },
+        // Tests below are from jwarc's InetAddressesTest
+        {"2001:db8::1", "2001:db8:0:0:0:0:0:1"},
+        {"2001:db8::1", "2001:0DB8:0:0:0:0:0:1"},
+        {"::", "0:0:0:0:0:0:0:0"},
+        {"::1", "0:0:0:0:0:0:0:1"},
+        {"2001:db8:1:1:1:1:1:1", "2001:db8:1:1:1:1:1:1"},
+        {"2001:0:0:1::1", "2001:0:0:1:0:0:0:1"},
+        {"2001:db8:f::1", "2001:db8:000f:0:0:0:0:1"},
+        {"2001:db8::1:0:0:1", "2001:0db8:0000:0000:0001:0000:0000:0001"},
+        {"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"},
+        {"2001:200f::1", "2001:200f:0:0:0:0:0:1"},
+        {
+            // https://datatracker.ietf.org/doc/html/rfc5952#section-4.2.2
+            // "The symbol "::" MUST NOT be used to shorten just one 16-bit 0 field."
+            "2001:0:3:4:5:6:7:8", "2001:0:3:4:5:6:7:8"
+        },
+        {
+            // shorten first of same-length consecutive 0 fields, also in initial position
+            "::4:0:0:0:ffff", "0:0:0:4:0:0:0:ffff"
+        },
+    };
 
     @Test
     void testGetDigestSha1() {
@@ -72,6 +108,21 @@ void testGetDigestSha1RobotsTxt() {
         assertEquals(sha1str, WARCRecordFormat.getDigestSha1(content), "Wrong sha1 digest");
     }
 
+    @Test
+    void testCanonicalizeIpAddress() {
+        Metadata metadata = new Metadata();
+        for (String[] p : ipAddressTestData) {
+            metadata.setValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, p[1]);
+            Optional<String> ip = WARCRecordFormat.getIPAddress(metadata, protocolMDprefix);
+            if (p[0] == null) {
+                assertTrue(ip.isEmpty());
+            } else {
+                assertTrue(ip.isPresent(), "No canonical form for " + p[1]);
+                assertEquals(p[0], ip.get());
+            }
+        }
+    }
+
     @Test
     void testWarcRecord() {
         // test validity of WARC record
@@ -82,6 +133,8 @@ void testWarcRecord() {
         metadata.addValue(
                 protocolMDprefix + ProtocolResponse.RESPONSE_HEADERS_KEY,
                 "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n");
+        metadata.addValue(
+                protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "2001:0DB8:0:0:0:0:0:1");
         Tuple tuple = mock(Tuple.class);
         when(tuple.getBinaryByField("content")).thenReturn(content);
         when(tuple.getStringByField("url")).thenReturn("https://www.example.org/");
@@ -111,6 +164,9 @@ void testWarcRecord() {
         assertTrue(
                 warcString.contains("\r\nWARC-Payload-Digest: " + sha1str + "\r\n"),
                 "WARC record: no or incorrect payload digest");
+        assertTrue(
+                warcString.contains("\r\nWARC-IP-Address: 2001:db8::1\r\n"),
+                "WARC record: WARC-IP-Address not found (canonical form expected)");
     }
 
     @Test

pom.xml

@@ -76,6 +76,7 @@ under the License.
         <commons.io.version>2.22.0</commons.io.version>
         <commons.compress.version>1.28.0</commons.compress.version>
         <commons.codec.version>1.22.0</commons.codec.version>
+        <guava.version>33.6.0-jre</guava.version>
         <git-code-format-maven-plugin.version>5.4</git-code-format-maven-plugin.version>
         <testcontainers.version>1.21.4</testcontainers.version>
         <awaitility.version>4.3.0</awaitility.version>