GH-50194: [C++] Move S3 and AWS-SDK to its own libarrow_s3.so

.github/workflows/cpp_extra.yml

@@ -341,7 +341,11 @@ jobs:
           PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data
         run: |
           # MinIO is required
+          # TODO: This doesn't seem like a good solution
+          # TODO: Waiting for PR 50215 to be merged
           exclude_tests="arrow-s3fs-test"
+          exclude_tests="${exclude_tests}|arrow-s3fs-module-test"
+          exclude_tests="${exclude_tests}|arrow-filesystem-test"
           # unstable
           exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test"
           exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test"

ci/docker/conda-cpp.dockerfile

@@ -73,7 +73,6 @@ ENV ARROW_ACERO=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
     ARROW_S3=ON \
-    ARROW_S3_MODULE=ON \
     ARROW_SUBSTRAIT=ON \
     ARROW_USE_CCACHE=ON \
     ARROW_USE_MOLD=ON \

ci/scripts/cpp_build.sh

@@ -70,7 +70,6 @@ if [ "${ARROW_ENABLE_THREADING:-ON}" = "OFF" ]; then
   ARROW_JEMALLOC=OFF
   ARROW_MIMALLOC=OFF
   ARROW_S3=OFF
-  ARROW_S3_MODULE=OFF
   ARROW_WITH_OPENTELEMETRY=OFF
 fi
 
@@ -230,7 +229,6 @@ else
     -DARROW_PARQUET=${ARROW_PARQUET:-OFF} \
     -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \
     -DARROW_S3=${ARROW_S3:-OFF} \
-    -DARROW_S3_MODULE=${ARROW_S3_MODULE:-OFF} \
     -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL:-DEFAULT} \
     -DARROW_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF} \
     -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \

ci/scripts/cpp_test.sh

@@ -51,7 +51,11 @@ if ! type storage-testbench >/dev/null 2>&1; then
   exclude_tests+=("arrow-gcsfs-test")
 fi
 if ! type minio >/dev/null 2>&1; then
+  # TODO: This doesn't seem like a good solution
+  # TODO: Waiting for PR 50215 to be merged
   exclude_tests+=("arrow-s3fs-test")
+  exclude_tests+=("arrow-s3fs-module-test")
+  exclude_tests+=("arrow-filesystem-test")
 fi
 case "$(uname)" in
   Linux)
@@ -60,7 +64,10 @@ case "$(uname)" in
   Darwin)
     n_jobs=$(sysctl -n hw.ncpu)
     # TODO: https://github.com/apache/arrow/issues/40410
+    # TODO: Waiting for PR 50215 to be merged
     exclude_tests+=("arrow-s3fs-test")
+    exclude_tests+=("arrow-s3fs-module-test")
+    exclude_tests+=("arrow-filesystem-test")
     ;;
   MINGW*)
     n_jobs=${NUMBER_OF_PROCESSORS:-1}

cpp/cmake_modules/DefineOptions.cmake

@@ -390,17 +390,11 @@ takes precedence over ccache if a storage backend is configured" ON)
                 ARROW_JSON)
 
   define_option(ARROW_S3
-                "Build Arrow with S3 support (requires the AWS SDK for C++)"
+                "Build Arrow S3 Module (requires the AWS SDK for C++)"
                 OFF
                 DEPENDS
                 ARROW_FILESYSTEM)
 
-  define_option(ARROW_S3_MODULE
-                "Build the Arrow S3 filesystem as a dynamic module"
-                OFF
-                DEPENDS
-                ARROW_S3)
-
   define_option(ARROW_SUBSTRAIT
                 "Build the Arrow Substrait Consumer Module"
                 OFF

cpp/src/arrow/ArrowS3Config.cmake.in

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This config sets the following variables in your project::
+#
+#   ArrowS3_FOUND - true if Arrow S3 found on the system
+#
+# This config sets the following targets in your project::
+#
+#   ArrowS3::arrow_s3_shared - for linked as shared library if shared library is built
+#   ArrowS3::arrow_s3_static - for linked as static library if static library is built
+
+@PACKAGE_INIT@
+
+include(CMakeFindDependencyMacro)
+find_dependency(Arrow CONFIG)
+
+include("${CMAKE_CURRENT_LIST_DIR}/ArrowS3Targets.cmake")
+
+arrow_keep_backward_compatibility(ArrowS3 arrow_s3)
+
+check_required_components(ArrowS3)
+
+arrow_show_details(ArrowS3 ARROW_S3)

cpp/src/arrow/CMakeLists.txt

@@ -960,11 +960,6 @@ if(ARROW_FILESYSTEM)
   if(ARROW_HDFS)
     list(APPEND ARROW_FILESYSTEM_SRCS filesystem/hdfs.cc)
   endif()
-  if(ARROW_S3)
-    list(APPEND ARROW_FILESYSTEM_SRCS filesystem/s3fs.cc)
-    set_source_files_properties(filesystem/s3fs.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION
-                                                              ON)
-  endif()
 
   arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS})
   if(ARROW_AZURE)
@@ -984,21 +979,38 @@ if(ARROW_FILESYSTEM)
     endforeach()
   endif()
   if(ARROW_S3)
-    foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
-      target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE ${AWSSDK_LINK_LIBRARIES})
+    if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
+      string(APPEND ARROW_S3_PC_CFLAGS "${ARROW_S3_PC_CFLAGS_PRIVATE}")
+      set(ARROW_S3_PC_CFLAGS_PRIVATE "")
+    endif()
+    list(APPEND ARROW_S3_LIB_SRCS filesystem/s3fs_module.cc filesystem/s3fs.cc)
+    add_arrow_lib(arrow_s3
+                  CMAKE_PACKAGE_NAME
+                  ArrowS3
+                  PKG_CONFIG_NAME
+                  arrow-s3
+                  SOURCES
+                  ${ARROW_S3_LIB_SRCS}
+                  SHARED_LINK_LIBS
+                  arrow_shared
+                  SHARED_PRIVATE_LINK_LIBS
+                  ${AWSSDK_LINK_LIBRARIES}
+                  SHARED_INSTALL_INTERFACE_LIBS
+                  ${ARROW_S3_SHARED_INSTALL_INTERFACE_LIBS}
+                  STATIC_LINK_LIBS
+                  arrow_static
+                  ${AWSSDK_LINK_LIBRARIES}
+                  OUTPUTS
+                  ARROW_S3_LIBRARIES)
+    foreach(LIB_TARGET ${ARROW_S3_LIBRARIES})
+      target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_S3_EXPORTING)
     endforeach()
-
-    if(ARROW_S3_MODULE)
-      if(NOT ARROW_BUILD_SHARED)
-        message(FATAL_ERROR "ARROW_S3_MODULE without shared libarrow (-DARROW_BUILD_SHARED=ON) is not supported"
-        )
-      endif()
-
-      add_library(arrow_s3fs MODULE filesystem/s3fs_module.cc filesystem/s3fs.cc)
-      target_link_libraries(arrow_s3fs PRIVATE ${AWSSDK_LINK_LIBRARIES} arrow_shared)
-      set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
-                                  PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
+    if(ARROW_BUILD_STATIC AND WIN32)
+      target_compile_definitions(arrow_s3_static PUBLIC ARROW_S3_STATIC)
     endif()
+
+    set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
+                                PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
   endif()
 
   list(APPEND ARROW_TESTING_SHARED_LINK_LIBS ${ARROW_GTEST_GMOCK})

cpp/src/arrow/arrow-s3.pc.in

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@CMAKE_INSTALL_PREFIX@
+includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@
+libdir=@ARROW_PKG_CONFIG_LIBDIR@
+
+Name: Apache Arrow S3
+Description: Apache Arrow's S3 filesystem implementation.
+Version: @ARROW_VERSION@
+Requires: arrow
+Libs: -L${libdir} -larrow_s3
+Cflags:@ARROW_S3_PC_CFLAGS@
+Cflags.private:@ARROW_S3_PC_CFLAGS_PRIVATE@

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -73,9 +73,9 @@ if(ARROW_S3)
   # static variables storage of AWS SDK for C++ in libaws*.a may be
   # mixed with one in libarrow.
   if(ARROW_TEST_LINKAGE STREQUAL "shared")
-    list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_shared)
+    list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_s3_shared)
   else()
-    list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_static)
+    list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_s3_static)
   endif()
   list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS ${AWSSDK_LINK_LIBRARIES})
   add_arrow_test(s3fs_test
@@ -107,7 +107,7 @@ if(ARROW_S3)
   if(ARROW_BUILD_TESTS)
     add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc)
     target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS}
-                          ${GFLAGS_LIBRARIES})
+                          ${ARROW_S3_TEST_EXTRA_LINK_LIBS} ${GFLAGS_LIBRARIES})
     add_dependencies(arrow-tests arrow-s3fs-narrative-test)
   endif()
 
@@ -119,6 +119,7 @@ if(ARROW_S3)
                         s3fs_benchmark.cc
                         s3_test_util.cc
                         STATIC_LINK_LIBS
+                        ${ARROW_S3_TEST_EXTRA_LINK_LIBS}
                         ${AWSSDK_LINK_LIBRARIES}
                         ${ARROW_BENCHMARK_LINK_LIBS})
     if(ARROW_TEST_LINKAGE STREQUAL "static")
@@ -128,21 +129,17 @@ if(ARROW_S3)
     endif()
   endif()
 
-  if(ARROW_S3_MODULE AND ARROW_BUILD_TESTS)
+  if(ARROW_BUILD_TESTS AND ARROW_BUILD_SHARED)
     add_arrow_test(s3fs_module_test
                    SOURCES
                    s3fs_module_test.cc
                    s3_test_util.cc
                    EXTRA_LABELS
                    filesystem
                    DEFINITIONS
-                   ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3fs>"
+                   ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3_shared>"
                    EXTRA_LINK_LIBS
                    Boost::filesystem)
-    target_compile_definitions(arrow-filesystem-test
-                               PUBLIC ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3fs>")
-    target_sources(arrow-filesystem-test PUBLIC s3fs_module_test.cc s3_test_util.cc)
-    target_link_libraries(arrow-filesystem-test PUBLIC Boost::filesystem)
   endif()
 endif()
 

cpp/src/arrow/filesystem/filesystem.h

@@ -560,6 +560,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {
 /// will fail with an error.
 ///
 /// The user is responsible for synchronization of calls to this function.
+ARROW_EXPORT
 void EnsureFinalized();
 
 /// \defgroup filesystem-factories Functions for creating FileSystem instances

cpp/src/arrow/filesystem/s3_visibility.h

@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#  if defined(_MSC_VER)
+#    pragma warning(push)
+#    pragma warning(disable : 4251)
+#  else
+#    pragma GCC diagnostic ignored "-Wattributes"
+#  endif
+
+#  ifdef ARROW_S3_STATIC
+#    define ARROW_S3_EXPORT
+#  elif defined(ARROW_S3_EXPORTING)
+#    define ARROW_S3_EXPORT __declspec(dllexport)
+#  else
+#    define ARROW_S3_EXPORT __declspec(dllimport)
+#  endif
+
+#  define ARROW_S3_NO_EXPORT
+
+#  if defined(_MSC_VER)
+#    pragma warning(pop)
+#  endif
+
+#else  // Not Windows
+#  ifndef ARROW_S3_EXPORT
+#    define ARROW_S3_EXPORT __attribute__((visibility("default")))
+#  endif
+#  ifndef ARROW_S3_NO_EXPORT
+#    define ARROW_S3_NO_EXPORT __attribute__((visibility("hidden")))
+#  endif
+#endif