diff --git a/core/pom.xml b/core/pom.xml
index 6d12c4277..9a4f873f7 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -50,7 +50,6 @@ under the License.
5.4.0
3.2.4
- 33.6.0-jre
true
0.72
0.54
diff --git a/external/warc/pom.xml b/external/warc/pom.xml
index 694770298..32bc2c35f 100644
--- a/external/warc/pom.xml
+++ b/external/warc/pom.xml
@@ -89,6 +89,12 @@ under the License.
${jwarc.version}
+
+ com.google.guava
+ guava
+ ${guava.version}
+
+
org.apache.stormcrawler
stormcrawler-core
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
index 4555fac3c..cee46c8fb 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
@@ -21,6 +21,7 @@
import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_HEADERS_KEY;
import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY;
+import com.google.common.net.InetAddresses;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -32,6 +33,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Optional;
import java.util.UUID;
import java.util.regex.Pattern;
import org.apache.commons.codec.binary.Base32;
@@ -317,6 +319,44 @@ protected String getCaptureTime(Metadata metadata) {
return WARC_DF.format(capturedAt);
}
+ /**
+ * Get the canonicalized IP address, if stored as protocol metadata.
+ *
+ *
+ * - IPv4: dot-decimal notation (dotted quad)
+ *
- IPv6: the canonical format (short representation) defined by RFC 5952
+ *
+ *
+ * See also
+ *
+ *
+ *
+ * @param metadata
+ * @param protocolMDprefix
+ * @return The canonical IP address string, or empty if the IP address was not recorded in
+ * metadata.
+ */
+ protected static Optional getIPAddress(Metadata metadata, String protocolMDprefix) {
+ String ip = metadata.getFirstValue(RESPONSE_IP_KEY, protocolMDprefix);
+ if (StringUtils.isBlank(ip)) {
+ return Optional.empty();
+ }
+ try {
+ return Optional.of(InetAddresses.toAddrString(InetAddresses.forString(ip)));
+ } catch (IllegalArgumentException e) {
+ return Optional.empty();
+ }
+ }
+
@Override
public byte[] format(Tuple tuple) {
@@ -383,9 +423,9 @@ public byte[] format(Tuple tuple) {
buffer.append("WARC-Type").append(": ").append(WARCTypeValue).append(CRLF);
// "WARC-IP-Address" if present
- String IP = metadata.getFirstValue(RESPONSE_IP_KEY, this.protocolMDprefix);
- if (StringUtils.isNotBlank(IP)) {
- buffer.append("WARC-IP-Address").append(": ").append(IP).append(CRLF);
+ Optional ip = getIPAddress(metadata, this.protocolMDprefix);
+ if (ip.isPresent()) {
+ buffer.append("WARC-IP-Address").append(": ").append(ip.get()).append(CRLF);
}
// must be a valid URI
diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java
index 1b4833fba..da35d1af0 100644
--- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java
+++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java
@@ -18,10 +18,10 @@
package org.apache.stormcrawler.warc;
import static org.apache.stormcrawler.protocol.ProtocolResponse.REQUEST_HEADERS_KEY;
-import static org.apache.stormcrawler.protocol.ProtocolResponse.RESPONSE_IP_KEY;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.util.Optional;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.storm.tuple.Tuple;
@@ -65,9 +65,9 @@ public byte[] format(Tuple tuple) {
buffer.append("WARC-Type: ").append(WARC_TYPE_REQUEST).append(CRLF);
// "WARC-IP-Address" if present
- String IP = metadata.getFirstValue(RESPONSE_IP_KEY, this.protocolMDprefix);
- if (StringUtils.isNotBlank(IP)) {
- buffer.append("WARC-IP-Address: ").append(IP).append(CRLF);
+ Optional ip = getIPAddress(metadata, this.protocolMDprefix);
+ if (ip.isPresent()) {
+ buffer.append("WARC-IP-Address").append(": ").append(ip.get()).append(CRLF);
}
addRecordID(buffer);
diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java
index 6c66204d7..045579a89 100644
--- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java
+++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java
@@ -27,6 +27,7 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
+import java.util.Optional;
import org.apache.http.HttpHeaders;
import org.apache.storm.tuple.Tuple;
import org.apache.stormcrawler.Metadata;
@@ -38,7 +39,42 @@
class WARCRecordFormatTest {
- private String protocolMDprefix = "http.";
+ private static String protocolMDprefix = "http.";
+
+ private static String[][] ipAddressTestData = {
+ // { expected, input }
+ // IPv4
+ {"127.0.0.1", "127.0.0.1"},
+ // IPv6
+ {
+ // IPv4-mapped IPv6 addresses
+ "192.0.2.128", "::ffff:192.0.2.128"
+ },
+ {
+ // IPv4-mapped IPv6 addresses (hex)
+ "192.0.2.128", "::ffff:C000:0280"
+ },
+ // Tests below are from jwarc's InetAddressesTest
+ {"2001:db8::1", "2001:db8:0:0:0:0:0:1"},
+ {"2001:db8::1", "2001:0DB8:0:0:0:0:0:1"},
+ {"::", "0:0:0:0:0:0:0:0"},
+ {"::1", "0:0:0:0:0:0:0:1"},
+ {"2001:db8:1:1:1:1:1:1", "2001:db8:1:1:1:1:1:1"},
+ {"2001:0:0:1::1", "2001:0:0:1:0:0:0:1"},
+ {"2001:db8:f::1", "2001:db8:000f:0:0:0:0:1"},
+ {"2001:db8::1:0:0:1", "2001:0db8:0000:0000:0001:0000:0000:0001"},
+ {"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"},
+ {"2001:200f::1", "2001:200f:0:0:0:0:0:1"},
+ {
+ // https://datatracker.ietf.org/doc/html/rfc5952#section-4.2.2
+ // "The symbol "::" MUST NOT be used to shorten just one 16-bit 0 field."
+ "2001:0:3:4:5:6:7:8", "2001:0:3:4:5:6:7:8"
+ },
+ {
+ // shorten first of same-length consecutive 0 fields, also in initial position
+ "::4:0:0:0:ffff", "0:0:0:4:0:0:0:ffff"
+ },
+ };
@Test
void testGetDigestSha1() {
@@ -72,6 +108,21 @@ void testGetDigestSha1RobotsTxt() {
assertEquals(sha1str, WARCRecordFormat.getDigestSha1(content), "Wrong sha1 digest");
}
+ @Test
+ void testCanonicalizeIpAddress() {
+ Metadata metadata = new Metadata();
+ for (String[] p : ipAddressTestData) {
+ metadata.setValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, p[1]);
+ Optional ip = WARCRecordFormat.getIPAddress(metadata, protocolMDprefix);
+ if (p[0] == null) {
+ assertTrue(ip.isEmpty());
+ } else {
+ assertTrue(ip.isPresent(), "No canonical form for " + p[1]);
+ assertEquals(p[0], ip.get());
+ }
+ }
+ }
+
@Test
void testWarcRecord() {
// test validity of WARC record
@@ -82,6 +133,8 @@ void testWarcRecord() {
metadata.addValue(
protocolMDprefix + ProtocolResponse.RESPONSE_HEADERS_KEY,
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n");
+ metadata.addValue(
+ protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "2001:0DB8:0:0:0:0:0:1");
Tuple tuple = mock(Tuple.class);
when(tuple.getBinaryByField("content")).thenReturn(content);
when(tuple.getStringByField("url")).thenReturn("https://www.example.org/");
@@ -111,6 +164,9 @@ void testWarcRecord() {
assertTrue(
warcString.contains("\r\nWARC-Payload-Digest: " + sha1str + "\r\n"),
"WARC record: no or incorrect payload digest");
+ assertTrue(
+ warcString.contains("\r\nWARC-IP-Address: 2001:db8::1\r\n"),
+ "WARC record: WARC-IP-Address not found (canonical form expected)");
}
@Test
diff --git a/pom.xml b/pom.xml
index bbaade1d0..98dea042a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -76,6 +76,7 @@ under the License.
2.22.0
1.28.0
1.22.0
+ 33.6.0-jre
5.4
1.21.4
4.3.0