Kokkos support for Lagrange and Monomial elements

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "contrib/metaphysicl"]
 	path = contrib/metaphysicl
-	url = ../../libMesh/MetaPhysicL
+	url = ../../rochi00/MetaPhysicL
 [submodule "contrib/timpi"]
 	path = contrib/timpi
 	url = ../../libMesh/TIMPI

Makefile.am

@@ -257,6 +257,12 @@ SUBDIRS += contrib
 
 include src/libmesh_SOURCES
 
+# When Kokkos support is enabled, add the Kokkos FE type-conversion sources.
+# These are kept in src/kokkos/ and not in src/libmesh_SOURCES (which is
+# auto-generated) so that they don't affect non-Kokkos builds.
+if LIBMESH_ENABLE_KOKKOS
+endif
+
 # A convenience library to hold proper libMesh
 # objects.  This will get appended with any contributed
 # sources to create the final library.

configure

@@ -672,6 +672,8 @@ libmesh_contrib_LDFLAGS
 libmesh_contrib_INCLUDES
 libmesh_optional_LIBS
 libmesh_optional_INCLUDES
+LIBMESH_ENABLE_KOKKOS_FALSE
+LIBMESH_ENABLE_KOKKOS_TRUE
 LIBMESH_ENABLE_METAPHYSICL_FALSE
 LIBMESH_ENABLE_METAPHYSICL_TRUE
 METAPHYSICL_INCLUDE
@@ -1347,6 +1349,7 @@ enable_metaphysicl
 with_metaphysicl
 with_metaphysicl_include
 enable_metaphysicl_required
+with_kokkos
 '
       ac_precious_vars='build_alias
 host_alias
@@ -2273,6 +2276,7 @@ Optional Packages:
                           internal: build from contrib
   --with-metaphysicl-include=</some/includedir>
 
+  --with-kokkos=DIR       Enable Kokkos support using the installation at DIR
 
 Some influential environment variables:
   PETSC_DIR   path to PETSc installation
@@ -63889,6 +63893,81 @@ fi
 
 
 
+# -------------------------------------------------------------
+# Kokkos -- optional, enables the native Kokkos FE math path
+# -------------------------------------------------------------
+
+# Check whether --with-kokkos was given.
+if test ${with_kokkos+y}
+then :
+  withval=$with_kokkos; KOKKOS_DIR="$withval"
+else case e in #(
+  e) KOKKOS_DIR="no" ;;
+esac
+fi
+
+
+if test "x$KOKKOS_DIR" != "xno"
+then :
+
+    as_ac_File=`printf "%s\n" "ac_cv_file_$KOKKOS_DIR/include/Kokkos_Core.hpp" | sed "$as_sed_sh"`
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $KOKKOS_DIR/include/Kokkos_Core.hpp" >&5
+printf %s "checking for $KOKKOS_DIR/include/Kokkos_Core.hpp... " >&6; }
+if eval test \${$as_ac_File+y}
+then :
+  printf %s "(cached) " >&6
+else case e in #(
+  e) test "$cross_compiling" = yes &&
+  as_fn_error $? "cannot check for file existence when cross compiling" "$LINENO" 5
+if test -r "$KOKKOS_DIR/include/Kokkos_Core.hpp"; then
+  eval "$as_ac_File=yes"
+else
+  eval "$as_ac_File=no"
+fi ;;
+esac
+fi
+eval ac_res=\$$as_ac_File
+	       { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+printf "%s\n" "$ac_res" >&6; }
+if eval test \"x\$"$as_ac_File"\" = x"yes"
+then :
+
+        enablekokkos=yes
+        libmesh_optional_INCLUDES="$libmesh_optional_INCLUDES -I$KOKKOS_DIR/include"
+        libmesh_optional_LIBS="$libmesh_optional_LIBS -L$KOKKOS_DIR/lib -lkokkoscore"
+
+printf "%s\n" "#define HAVE_KOKKOS 1" >>confdefs.h
+
+        { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: <<< Configuring library with Kokkos support >>>" >&5
+printf "%s\n" "<<< Configuring library with Kokkos support >>>" >&6; }
+
+else case e in #(
+  e)
+        { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: Kokkos not found at $KOKKOS_DIR -- disabling Kokkos FE support" >&5
+printf "%s\n" "$as_me: WARNING: Kokkos not found at $KOKKOS_DIR -- disabling Kokkos FE support" >&2;}
+        enablekokkos=no
+       ;;
+esac
+fi
+
+
+else case e in #(
+  e) enablekokkos=no ;;
+esac
+fi
+
+ if test x$enablekokkos = xyes; then
+  LIBMESH_ENABLE_KOKKOS_TRUE=
+  LIBMESH_ENABLE_KOKKOS_FALSE='#'
+else
+  LIBMESH_ENABLE_KOKKOS_TRUE='#'
+  LIBMESH_ENABLE_KOKKOS_FALSE=
+fi
+
+# -------------------------------------------------------------
+
+
+
 if test "$enableoptional" != no
 then :
 
@@ -65158,6 +65237,10 @@ if test -z "${LIBMESH_ENABLE_METAPHYSICL_TRUE}" && test -z "${LIBMESH_ENABLE_MET
   as_fn_error $? "conditional \"LIBMESH_ENABLE_METAPHYSICL\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
 fi
+if test -z "${LIBMESH_ENABLE_KOKKOS_TRUE}" && test -z "${LIBMESH_ENABLE_KOKKOS_FALSE}"; then
+  as_fn_error $? "conditional \"LIBMESH_ENABLE_KOKKOS\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
 if test -z "${GIT_CHECKOUT_TRUE}" && test -z "${GIT_CHECKOUT_FALSE}"; then
   as_fn_error $? "conditional \"GIT_CHECKOUT\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5

include/Makefile.am

@@ -1,5 +1,23 @@
 SUBDIRS = libmesh
 
+# GPU (Kokkos) FE math headers — installed preserving the gpu/ subdirectory so
+# downstream code can use #include "libmesh/gpu/kokkos_fe_types.h" etc.
+# nobase_ is used instead of the standard flat install to keep the namespace.
+if LIBMESH_ENABLE_KOKKOS
+nobase_include_HEADERS = \
+  gpu/kokkos_scalar_types.h \
+  gpu/kokkos_fe_types.h \
+  gpu/kokkos_fe_base.h \
+  gpu/kokkos_fe_evaluator.h \
+  gpu/kokkos_fe_lagrange_1d.h \
+  gpu/kokkos_fe_lagrange_2d.h \
+  gpu/kokkos_fe_lagrange_3d.h \
+  gpu/kokkos_fe_monomial.h \
+  gpu/kokkos_fe_face_map.h \
+  gpu/kokkos_fe_map.h \
+  gpu/kokkos_quadrature.h
+endif
+
 # special handholding for prefix_config.m4 generated files
 # so that 'make clean ; make' works as does 'make distcheck'
 # libmesh_config.h is made by ./configure, so it should get

include/Makefile.in

@@ -157,7 +157,8 @@ am__aclocal_m4_deps =  \
 am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
 	$(ACLOCAL_M4)
 DIST_COMMON = $(srcdir)/Makefile.am $(include_HEADERS) \
-	$(noinst_HEADERS) $(am__DIST_COMMON)
+	$(am__nobase_include_HEADERS_DIST) $(noinst_HEADERS) \
+	$(am__DIST_COMMON)
 mkinstalldirs = $(install_sh) -d
 CONFIG_HEADER = libmesh_config.h.tmp
 CONFIG_CLEAN_FILES =
@@ -215,8 +216,14 @@ am__uninstall_files_from_dir = { \
   || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
        $(am__cd) "$$dir" && echo $$files | $(am__xargs_n) 40 $(am__rm_f); }; \
   }
-am__installdirs = "$(DESTDIR)$(includedir)"
-HEADERS = $(include_HEADERS) $(noinst_HEADERS)
+am__installdirs = "$(DESTDIR)$(includedir)" "$(DESTDIR)$(includedir)"
+am__nobase_include_HEADERS_DIST = gpu/kokkos_scalar_types.h \
+	gpu/kokkos_fe_types.h gpu/kokkos_fe_base.h \
+	gpu/kokkos_fe_evaluator.h gpu/kokkos_fe_lagrange_1d.h \
+	gpu/kokkos_fe_lagrange_2d.h gpu/kokkos_fe_lagrange_3d.h \
+	gpu/kokkos_fe_monomial.h gpu/kokkos_fe_face_map.h
+HEADERS = $(include_HEADERS) $(nobase_include_HEADERS) \
+	$(noinst_HEADERS)
 RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive	\
   distclean-recursive maintainer-clean-recursive
 am__recursive_targets = \
@@ -598,6 +605,21 @@ vtkmajor = @vtkmajor@
 vtkversion = @vtkversion@
 SUBDIRS = libmesh
 
+# GPU (Kokkos) FE math headers — installed preserving the gpu/ subdirectory so
+# downstream code can use #include "libmesh/gpu/kokkos_fe_types.h" etc.
+# nobase_ is used instead of the standard flat install to keep the namespace.
+@LIBMESH_ENABLE_KOKKOS_TRUE@nobase_include_HEADERS = \
+@LIBMESH_ENABLE_KOKKOS_TRUE@  gpu/kokkos_scalar_types.h \
+@LIBMESH_ENABLE_KOKKOS_TRUE@  gpu/kokkos_fe_types.h \
+@LIBMESH_ENABLE_KOKKOS_TRUE@  gpu/kokkos_fe_base.h \
+@LIBMESH_ENABLE_KOKKOS_TRUE@  gpu/kokkos_fe_evaluator.h \
+@LIBMESH_ENABLE_KOKKOS_TRUE@  gpu/kokkos_fe_lagrange_1d.h \
+@LIBMESH_ENABLE_KOKKOS_TRUE@  gpu/kokkos_fe_lagrange_2d.h \
+@LIBMESH_ENABLE_KOKKOS_TRUE@  gpu/kokkos_fe_lagrange_3d.h \
+@LIBMESH_ENABLE_KOKKOS_TRUE@  gpu/kokkos_fe_monomial.h \
+@LIBMESH_ENABLE_KOKKOS_TRUE@  gpu/kokkos_fe_face_map.h
+
+
 # special handholding for prefix_config.m4 generated files
 # so that 'make clean ; make' works as does 'make distcheck'
 # libmesh_config.h is made by ./configure, so it should get
@@ -635,6 +657,7 @@ include_HEADERS = \
         base/libmesh_abort.h \
         base/libmesh_base.h \
         base/libmesh_common.h \
+        base/libmesh_device.h \
         base/libmesh_documentation.h \
         base/libmesh_exceptions.h \
         base/libmesh_logging.h \
@@ -656,6 +679,7 @@ include_HEADERS = \
         enums/enum_elem_quality.h \
         enums/enum_elem_type.h \
         enums/enum_error_estimator_type.h \
+        enums/enum_fe_elem_class.h \
         enums/enum_fe_family.h \
         enums/enum_inf_map_type.h \
         enums/enum_io_package.h \
@@ -1214,6 +1238,30 @@ uninstall-includeHEADERS:
 	@list='$(include_HEADERS)'; test -n "$(includedir)" || list=; \
 	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
 	dir='$(DESTDIR)$(includedir)'; $(am__uninstall_files_from_dir)
+install-nobase_includeHEADERS: $(nobase_include_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(nobase_include_HEADERS)'; test -n "$(includedir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(includedir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(includedir)" || exit 1; \
+	fi; \
+	$(am__nobase_list) | while read dir files; do \
+	  xfiles=; for file in $$files; do \
+	    if test -f "$$file"; then xfiles="$$xfiles $$file"; \
+	    else xfiles="$$xfiles $(srcdir)/$$file"; fi; done; \
+	  test -z "$$xfiles" || { \
+	    test "x$$dir" = x. || { \
+	      echo " $(MKDIR_P) '$(DESTDIR)$(includedir)/$$dir'"; \
+	      $(MKDIR_P) "$(DESTDIR)$(includedir)/$$dir"; }; \
+	    echo " $(INSTALL_HEADER) $$xfiles '$(DESTDIR)$(includedir)/$$dir'"; \
+	    $(INSTALL_HEADER) $$xfiles "$(DESTDIR)$(includedir)/$$dir" || exit $$?; }; \
+	done
+
+uninstall-nobase_includeHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(nobase_include_HEADERS)'; test -n "$(includedir)" || list=; \
+	$(am__nobase_strip_setup); files=`$(am__nobase_strip)`; \
+	dir='$(DESTDIR)$(includedir)'; $(am__uninstall_files_from_dir)
 
 # This directory's subdirectories are mostly independent; you can cd
 # into them and run 'make' without going through this Makefile.
@@ -1380,7 +1428,7 @@ check: check-recursive
 all-am: Makefile $(HEADERS) libmesh_config.h.tmp
 installdirs: installdirs-recursive
 installdirs-am:
-	for dir in "$(DESTDIR)$(includedir)"; do \
+	for dir in "$(DESTDIR)$(includedir)" "$(DESTDIR)$(includedir)"; do \
 	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
 	done
 install: install-recursive
@@ -1434,7 +1482,7 @@ info: info-recursive
 
 info-am:
 
-install-data-am: install-includeHEADERS
+install-data-am: install-includeHEADERS install-nobase_includeHEADERS
 
 install-dvi: install-dvi-recursive
 
@@ -1478,7 +1526,7 @@ ps: ps-recursive
 
 ps-am:
 
-uninstall-am: uninstall-includeHEADERS
+uninstall-am: uninstall-includeHEADERS uninstall-nobase_includeHEADERS
 
 .MAKE: $(am__recursive_targets) all install-am install-strip
 
@@ -1490,12 +1538,13 @@ uninstall-am: uninstall-includeHEADERS
 	install-data-am install-dvi install-dvi-am install-exec \
 	install-exec-am install-html install-html-am \
 	install-includeHEADERS install-info install-info-am \
-	install-man install-pdf install-pdf-am install-ps \
-	install-ps-am install-strip installcheck installcheck-am \
-	installcheck-local installdirs installdirs-am maintainer-clean \
-	maintainer-clean-generic mostlyclean mostlyclean-generic \
-	mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
-	uninstall-am uninstall-includeHEADERS
+	install-man install-nobase_includeHEADERS install-pdf \
+	install-pdf-am install-ps install-ps-am install-strip \
+	installcheck installcheck-am installcheck-local installdirs \
+	installdirs-am maintainer-clean maintainer-clean-generic \
+	mostlyclean mostlyclean-generic mostlyclean-libtool pdf pdf-am \
+	ps ps-am tags tags-am uninstall uninstall-am \
+	uninstall-includeHEADERS uninstall-nobase_includeHEADERS
 
 .PRECIOUS: Makefile
 

include/base/libmesh_common.h

@@ -30,6 +30,10 @@
 // The library configuration options
 #include "libmesh/libmesh_config.h"
 
+// Device compilation support — must be included before assert macros
+// so that LIBMESH_DEVICE_ASSERT is available for the Kokkos path.
+#include "libmesh/libmesh_device.h"
+
 // Use actual timestamps or constant dummies (to aid ccache)
 #ifdef LIBMESH_ENABLE_TIMESTAMPS
 #  define  LIBMESH_TIME __TIME__
@@ -287,7 +291,13 @@ extern bool warned_about_auto_ptr;
 #endif
 
 // The libmesh_assert() macro acts like C's assert(), but throws a
-// libmesh_error() (including stack trace, etc) instead of just exiting
+// libmesh_error() (including stack trace, etc) instead of just exiting.
+//
+// In .K translation units (LIBMESH_KOKKOS_COMPILATION defined),
+// LIBMESH_DEVICE_ASSERT is provided by libmesh_device.h using
+// printf + Kokkos::abort() — device-safe across CUDA/HIP/SYCL.
+// The assert macros delegate to it so that both host and device
+// code in the same file get assertion checking.
 #ifdef NDEBUG
 
 #define libmesh_assert_msg(asserted, msg)  ((void) 0)
@@ -299,6 +309,18 @@ extern bool warned_about_auto_ptr;
 #define libmesh_assert_less_equal_msg(expr1,expr2, msg)  ((void) 0)
 #define libmesh_assert_greater_equal_msg(expr1,expr2, msg)  ((void) 0)
 
+#elif defined(LIBMESH_DEVICE_ASSERT)
+
+// Kokkos compilation: use the device-safe assert from libmesh_device.h.
+#define libmesh_assert_msg(asserted, msg)  LIBMESH_DEVICE_ASSERT(asserted)
+#define libmesh_exceptionless_assert_msg(asserted, msg) LIBMESH_DEVICE_ASSERT(asserted)
+#define libmesh_assert_equal_to_msg(expr1,expr2, msg)  LIBMESH_DEVICE_ASSERT((expr1) == (expr2))
+#define libmesh_assert_not_equal_to_msg(expr1,expr2, msg)  LIBMESH_DEVICE_ASSERT((expr1) != (expr2))
+#define libmesh_assert_less_msg(expr1,expr2, msg)  LIBMESH_DEVICE_ASSERT((expr1) < (expr2))
+#define libmesh_assert_greater_msg(expr1,expr2, msg)  LIBMESH_DEVICE_ASSERT((expr1) > (expr2))
+#define libmesh_assert_less_equal_msg(expr1,expr2, msg)  LIBMESH_DEVICE_ASSERT((expr1) <= (expr2))
+#define libmesh_assert_greater_equal_msg(expr1,expr2, msg)  LIBMESH_DEVICE_ASSERT((expr1) >= (expr2))
+
 #else
 
 #define libmesh_assertion_types(expr1,expr2)                            \