Give the caller the possibility to access the actual effective URL used for fetching a resource

core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java

@@ -17,6 +17,7 @@
 
 package org.apache.stormcrawler.protocol;
 
+import java.net.URL;
 import org.apache.stormcrawler.Metadata;
 
 public class ProtocolResponse {
@@ -81,11 +82,17 @@ public enum TrimmedContentReason {
     private final byte[] content;
     private final int statusCode;
     private final Metadata metadata;
+    private final URL url;
 
     public ProtocolResponse(byte[] c, int s, Metadata md) {
+        this(c, s, md, null);
+    }
+
+    public ProtocolResponse(byte[] c, int s, Metadata md, URL url) {
         content = c;
         statusCode = s;
         metadata = md == null ? new Metadata() : md;
+        this.url = url;
     }
 
     public byte[] getContent() {
@@ -99,4 +106,8 @@ public int getStatusCode() {
     public Metadata getMetadata() {
         return metadata;
     }
+
+    public URL getUrl() {
+        return url;
+    }
 }

core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java

@@ -44,14 +44,15 @@ public class FileResponse {
     private byte[] content;
     private int statusCode;
     private final Metadata metadata;
+    private final URL url;
 
     public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOException {
 
         metadata = new Metadata();
         content = new byte[0];
         statusCode = HttpStatus.SC_INTERNAL_SERVER_ERROR;
 
-        URL url = URLUtil.toURL(u);
+        this.url = URLUtil.toURL(u);
 
         if (!url.getPath().equals(url.getFile())) {
             LOG.warn("url.getPath() != url.getFile(): {}.", url);
@@ -89,7 +90,7 @@ public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOE
     }
 
     public ProtocolResponse toProtocolResponse() {
-        return new ProtocolResponse(content, statusCode, metadata);
+        return new ProtocolResponse(content, statusCode, metadata, this.url);
     }
 
     private void getFileAsHttpResponse(File file) {

core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java

@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.MalformedURLException;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Base64;
@@ -228,7 +229,7 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except
         }
 
         HttpRequestBase request = new HttpGet(url);
-        ResponseHandler<ProtocolResponse> responseHandler = this;
+        int maxContent = globalMaxContent;
 
         if (md != null) {
 
@@ -262,8 +263,7 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except
             String pageMaxContentStr = md.getFirstValue("http.content.limit");
             if (StringUtils.isNotBlank(pageMaxContentStr)) {
                 try {
-                    int pageMaxContent = Integer.parseInt(pageMaxContentStr);
-                    responseHandler = getResponseHandlerWithContentLimit(pageMaxContent);
+                    maxContent = Integer.parseInt(pageMaxContentStr);
                 } catch (NumberFormatException e) {
                     LOG.warn("Invalid http.content.limit in metadata: {}", pageMaxContentStr);
                 }
@@ -275,6 +275,11 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except
         }
 
         request.setConfig(reqConfig);
+        // redirects are disabled, so the URL on the wire == what HttpClient parsed from the request
+        final URL fetchedUrl = request.getURI().toURL();
+        final int contentLimit = maxContent;
+        ResponseHandler<ProtocolResponse> responseHandler =
+                response -> handleResponseWithContentLimit(response, contentLimit, fetchedUrl);
 
         return httpClient.execute(request, responseHandler);
     }
@@ -305,11 +310,11 @@ protected void addHeadersToRequest(HttpRequestBase request, Metadata md) {
 
     @Override
     public ProtocolResponse handleResponse(HttpResponse response) throws IOException {
-        return handleResponseWithContentLimit(response, globalMaxContent);
+        return handleResponseWithContentLimit(response, globalMaxContent, null);
     }
 
-    public ProtocolResponse handleResponseWithContentLimit(HttpResponse response, int maxContent)
-            throws IOException {
+    public ProtocolResponse handleResponseWithContentLimit(
+            HttpResponse response, int maxContent, URL fetcherURL) throws IOException {
         StatusLine statusLine = response.getStatusLine();
         int status = statusLine.getStatusCode();
 
@@ -345,16 +350,7 @@ public ProtocolResponse handleResponseWithContentLimit(HttpResponse response, in
             metadata.setValue(ProtocolResponse.RESPONSE_HEADERS_KEY, verbatim.toString());
         }
 
-        return new ProtocolResponse(bytes, status, metadata);
-    }
-
-    private ResponseHandler<ProtocolResponse> getResponseHandlerWithContentLimit(
-            int pageMaxContent) {
-        return new ResponseHandler<>() {
-            public ProtocolResponse handleResponse(final HttpResponse response) throws IOException {
-                return handleResponseWithContentLimit(response, pageMaxContent);
-            }
-        };
+        return new ProtocolResponse(bytes, status, metadata, fetcherURL);
     }
 
     private static byte[] toByteArray(

core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java

@@ -464,7 +464,8 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata)
                 responsemetadata.setValue("metrics.dns.resolution.msec", dnsResolution.toString());
             }
 
-            return new ProtocolResponse(bytes, response.code(), responsemetadata);
+            return new ProtocolResponse(
+                    bytes, response.code(), responsemetadata, request.url().url());
         }
     }
 

external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java

@@ -46,6 +46,7 @@
 import org.apache.stormcrawler.protocol.Protocol;
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
 import org.slf4j.LoggerFactory;
 
 public class HttpProtocol extends AbstractHttpProtocol {
@@ -303,7 +304,8 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except
 
                 responseMetaData.addValue(MD_KEY_END, Instant.now().toString());
 
-                return new ProtocolResponse(content, status.get(), responseMetaData);
+                return new ProtocolResponse(
+                        content, status.get(), responseMetaData, URLUtil.toURL(url));
 
             } finally {
                 if (isTracing && responseMetaData.getFirstValue(MD_TRACE) == null) {

external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java

@@ -25,6 +25,7 @@
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
 import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.URLUtil;
 import org.openqa.selenium.remote.RemoteWebDriver;
 import org.slf4j.LoggerFactory;
 
@@ -73,14 +74,14 @@ public ProtocolResponse getProtocolOutput(String url, Metadata metadata) throws
                 byte[] content = new byte[] {};
                 Metadata m = new Metadata();
                 m.addValue(HttpHeaders.LOCATION, u);
-                return new ProtocolResponse(content, 307, m);
+                return new ProtocolResponse(content, 307, m, URLUtil.toURL(u));
             }
 
             outputMeta.addValue(MD_KEY_END, Instant.now().toString());
 
             // if no filters got triggered
             byte[] content = driver.getPageSource().getBytes(StandardCharsets.UTF_8);
-            return new ProtocolResponse(content, 200, outputMeta);
+            return new ProtocolResponse(content, 200, outputMeta, URLUtil.toURL(u));
 
         } finally {
             // finished with this driver - return it to the queue