From c3377ebff35f4706c07794c003affa938d1b78d7 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Tue, 4 Nov 2025 04:42:00 +0000 Subject: [PATCH 01/15] osc/sm: Add notification support for put/get operations This commit adds notification support to the OSC SM component by implementing the put_with_notify, get_with_notify, rput_with_notify, and rget_with_notify functions. These functions perform the same operations as their non-notify counterparts but also increment notification counters after the data transfer completes. The changes include: - Added function pointer types for notify variants in osc.h - Added function prototypes in osc_sm.h - Implemented the notify functions in osc_sm_comm.c - Updated the module template to register the new functions - Removed TODO comments that have been addressed Signed-off-by: Joseph Antony --- ompi/mca/osc/osc.h | 44 +++++++++- ompi/mca/osc/sm/osc_sm.h | 42 ++++++++++ ompi/mca/osc/sm/osc_sm_comm.c | 130 ++++++++++++++++++++++++++++- ompi/mca/osc/sm/osc_sm_component.c | 6 +- 4 files changed, 217 insertions(+), 5 deletions(-) diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index c8f77404c1c..bd05a6f11b7 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -216,6 +216,15 @@ typedef int (*ompi_osc_base_module_put_fn_t)(const void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); +typedef int (*ompi_osc_base_module_put_with_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); typedef int (*ompi_osc_base_module_get_fn_t)(void *origin_addr, size_t origin_count, @@ -226,6 +235,15 @@ typedef int (*ompi_osc_base_module_get_fn_t)(void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); +typedef int (*ompi_osc_base_module_get_with_notify_fn_t)(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); typedef int (*ompi_osc_base_module_accumulate_fn_t)(const void *origin_addr, size_t origin_count, @@ -276,6 +294,17 @@ typedef int (*ompi_osc_base_module_rput_fn_t)(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +typedef int (*ompi_osc_base_module_rput_with_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + typedef int (*ompi_osc_base_module_rget_fn_t)(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -286,6 +315,16 @@ typedef int (*ompi_osc_base_module_rget_fn_t)(void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +typedef int (*ompi_osc_base_module_rget_with_notify_fn_t)(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); typedef int (*ompi_osc_base_module_raccumulate_fn_t)(const void *origin_addr, size_t origin_count, @@ -371,7 +410,6 @@ typedef int (*ompi_osc_base_module_flush_local_all_fn_t)(struct ompi_win_t *win) * module structure. */ - // TODO: extend the struct and add pointers to put/get_with_notify functions struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_win_shared_query_fn_t osc_win_shared_query; @@ -380,14 +418,18 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_free_fn_t osc_free; ompi_osc_base_module_put_fn_t osc_put; + ompi_osc_base_module_put_with_notify_fn_t osc_put_with_notify; ompi_osc_base_module_get_fn_t osc_get; + ompi_osc_base_module_get_with_notify_fn_t osc_get_with_notify; ompi_osc_base_module_accumulate_fn_t osc_accumulate; ompi_osc_base_module_compare_and_swap_fn_t osc_compare_and_swap; ompi_osc_base_module_fetch_and_op_fn_t osc_fetch_and_op; ompi_osc_base_module_get_accumulate_fn_t osc_get_accumulate; ompi_osc_base_module_rput_fn_t osc_rput; + ompi_osc_base_module_rput_with_notify_fn_t osc_rput_with_notify; ompi_osc_base_module_rget_fn_t osc_rget; + ompi_osc_base_module_rget_with_notify_fn_t osc_rget_with_notify; ompi_osc_base_module_raccumulate_fn_t osc_raccumulate; ompi_osc_base_module_rget_accumulate_fn_t osc_rget_accumulate; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index 23afacd7d49..b7d6dadfd49 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -118,6 +118,16 @@ int ompi_osc_sm_put(const void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); + int ompi_osc_sm_put_with_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); + int ompi_osc_sm_get(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -127,6 +137,16 @@ int ompi_osc_sm_get(void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); +int ompi_osc_sm_get_with_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); + int ompi_osc_sm_accumulate(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -176,6 +196,17 @@ int ompi_osc_sm_rput(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +int ompi_osc_sm_rput_with_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + int ompi_osc_sm_rget(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -186,6 +217,17 @@ int ompi_osc_sm_rget(void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +int ompi_osc_sm_rget_with_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + int ompi_osc_sm_raccumulate(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index f9bae370870..a2e3a5cce1f 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -59,6 +59,49 @@ ompi_osc_sm_rput(const void *origin_addr, return OMPI_SUCCESS; } +int +ompi_osc_sm_rput_with_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **ompi_req) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rput_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* the only valid field of RMA request status is the MPI_ERROR field. + * ompi_request_empty has status MPI_SUCCESS and indicates the request is + * complete. */ + *ompi_req = &ompi_request_empty; + + opal_atomic_wmb(); + opal_atomic_add(&module->notify_counters[target][notify], 1); + + return OMPI_SUCCESS; +} int ompi_osc_sm_rget(void *origin_addr, @@ -99,6 +142,49 @@ ompi_osc_sm_rget(void *origin_addr, return OMPI_SUCCESS; } +int +ompi_osc_sm_rget_with_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **ompi_req) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt, + origin_addr, origin_count, origin_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* the only valid field of RMA request status is the MPI_ERROR field. + * ompi_request_empty has status MPI_SUCCESS and indicates the request is + * complete. */ + *ompi_req = &ompi_request_empty; + + opal_atomic_rmb(); + opal_atomic_add(&module->notify_counters[target][notify], 1); + + return OMPI_SUCCESS; +} int ompi_osc_sm_raccumulate(const void *origin_addr, @@ -236,6 +322,44 @@ ompi_osc_sm_put(const void *origin_addr, } +int +ompi_osc_sm_put_with_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win) +{ +int ret; +ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; +void *remote_address; + +OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "put_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + +remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + +ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); +if (OMPI_SUCCESS != ret) { + return ret; +} + +opal_atomic_wmb(); +opal_atomic_add(&module->notify_counters[target][notify], 1); + +return ret; +} + int ompi_osc_sm_get(void *origin_addr, size_t origin_count, @@ -294,7 +418,9 @@ ompi_osc_sm_get_with_notify(void *origin_addr, ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt, origin_addr, origin_count, origin_dt); - // TODO: do the same for put_with_notify + if (OMPI_SUCCESS != ret) { + return ret; + } opal_atomic_rmb(); opal_atomic_add(&module->notify_counters[target][notify], 1); @@ -473,4 +599,4 @@ ompi_osc_sm_fetch_and_op(const void *origin_addr, opal_atomic_unlock(&module->node_states[target].accumulate_lock); return OMPI_SUCCESS;; -} +} \ No newline at end of file diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index 1ad9a48cfd2..11f0ccc2e47 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -70,8 +70,6 @@ ompi_osc_sm_component_t mca_osc_sm_component = { MCA_BASE_COMPONENT_INIT(ompi, osc, sm) -// TODO: extend the struct and add pointers to put/get_with_notify functions -// TODO: extend it to rput/rget_with_notify as well ompi_osc_sm_module_t ompi_osc_sm_module_template = { { .osc_win_shared_query = ompi_osc_sm_shared_query, @@ -81,14 +79,18 @@ ompi_osc_sm_module_t ompi_osc_sm_module_template = { .osc_free = ompi_osc_sm_free, .osc_put = ompi_osc_sm_put, + .osc_put_with_notify = ompi_osc_sm_put_with_notify, .osc_get = ompi_osc_sm_get, + .osc_get_with_notify = ompi_osc_sm_get_with_notify, .osc_accumulate = ompi_osc_sm_accumulate, .osc_compare_and_swap = ompi_osc_sm_compare_and_swap, .osc_fetch_and_op = ompi_osc_sm_fetch_and_op, .osc_get_accumulate = ompi_osc_sm_get_accumulate, .osc_rput = ompi_osc_sm_rput, + .osc_rput_with_notify = ompi_osc_sm_rput_with_notify, .osc_rget = ompi_osc_sm_rget, + .osc_rget_with_notify = ompi_osc_sm_rget_with_notify, .osc_raccumulate = ompi_osc_sm_raccumulate, .osc_rget_accumulate = ompi_osc_sm_rget_accumulate, From 3bbbb4819a06a1a3046dcbb7aba91cc8fca47e4b Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Tue, 4 Nov 2025 17:03:59 +0000 Subject: [PATCH 02/15] osc/sm: Nit picking edits Signed-off-by: Joseph Antony --- ompi/mca/osc/sm/osc_sm_comm.c | 52 +++++++++++++++++------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index a2e3a5cce1f..ba19d8c08cf 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -333,31 +333,31 @@ ompi_osc_sm_put_with_notify(const void *origin_addr, int notify, struct ompi_win_t *win) { -int ret; -ompi_osc_sm_module_t *module = - (ompi_osc_sm_module_t*) win->w_osc_module; -void *remote_address; - -OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, - "put_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", - (unsigned long) origin_addr, origin_count, - origin_dt->name, target, (int) target_disp, - target_count, target_dt->name, - notify, - (unsigned long) win)); - -remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; - -ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, - remote_address, target_count, target_dt); -if (OMPI_SUCCESS != ret) { - return ret; -} + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "put_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } -opal_atomic_wmb(); -opal_atomic_add(&module->notify_counters[target][notify], 1); + opal_atomic_wmb(); + opal_atomic_add(&module->notify_counters[target][notify], 1); -return ret; + return ret; } int @@ -419,7 +419,7 @@ ompi_osc_sm_get_with_notify(void *origin_addr, ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt, origin_addr, origin_count, origin_dt); if (OMPI_SUCCESS != ret) { - return ret; + return ret; } opal_atomic_rmb(); opal_atomic_add(&module->notify_counters[target][notify], 1); @@ -598,5 +598,5 @@ ompi_osc_sm_fetch_and_op(const void *origin_addr, done: opal_atomic_unlock(&module->node_states[target].accumulate_lock); - return OMPI_SUCCESS;; -} \ No newline at end of file + return OMPI_SUCCESS; +} From 5e895cf31af78219cbebdf24d9b8c04466f66761 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Wed, 19 Nov 2025 12:29:14 -0500 Subject: [PATCH 03/15] Public APIs for: put_with_notify get_with_notify Signed-off-by: Joseph Antony --- ompi/include/mpi.h.in | 29 ++++++++ ompi/include/mpif-values.py | 1 + ompi/mca/osc/osc.h | 16 ++--- ompi/mca/osc/sm/osc_sm.h | 10 +-- ompi/mca/osc/sm/osc_sm_comm.c | 8 +-- ompi/mca/osc/sm/osc_sm_component.c | 8 +-- ompi/mpi/bindings/ompi_bindings/consts.py | 1 + ompi/mpi/c/Makefile.am | 2 + ompi/mpi/c/get_notify.c.in | 77 ++++++++++++++++++++++ ompi/mpi/c/put_notify.c.in | 80 +++++++++++++++++++++++ ompi/runtime/ompi_spc.c | 2 + ompi/runtime/ompi_spc.h | 2 + 12 files changed, 215 insertions(+), 21 deletions(-) create mode 100644 ompi/mpi/c/get_notify.c.in create mode 100644 ompi/mpi/c/put_notify.c.in diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in index e06865b182f..d34624cd1d2 100644 --- a/ompi/include/mpi.h.in +++ b/ompi/include/mpi.h.in @@ -764,6 +764,7 @@ enum { #define MPI_ERR_SESSION 78 #define MPI_ERR_VALUE_TOO_LARGE 79 #define MPI_ERR_ERRHANDLER 80 +#define MPI_ERR_NOTIFY_IDX 81 /* Per MPI-3 p349 47, MPI_ERR_LASTCODE must be >= the last predefined MPI_ERR_ code. Set the last code to allow some room for adding @@ -1917,6 +1918,14 @@ OMPI_DECLSPEC int MPI_Get_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int MPI_Get_notify(void *origin_addr, int origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int MPI_Get_notify_c(void *origin_addr, MPI_Count origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int MPI_Get_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, void *result_addr, int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -2180,6 +2189,12 @@ OMPI_DECLSPEC int MPI_Put(const void *origin_addr, int origin_count, MPI_Dataty OMPI_DECLSPEC int MPI_Put_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int MPI_Put_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int MPI_Put_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int MPI_Query_thread(int *provided); OMPI_DECLSPEC int MPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -3091,6 +3106,14 @@ OMPI_DECLSPEC int PMPI_Get_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int PMPI_Get_notify(void *origin_addr, int origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int PMPI_Get_notify_c(void *origin_addr, MPI_Count origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int PMPI_Get_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, void *result_addr, int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -3354,6 +3377,12 @@ OMPI_DECLSPEC int PMPI_Put(const void *origin_addr, int origin_count, MPI_Datat OMPI_DECLSPEC int PMPI_Put_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int PMPI_Put_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int PMPI_Put_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int PMPI_Query_thread(int *provided); OMPI_DECLSPEC int PMPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_count, diff --git a/ompi/include/mpif-values.py b/ompi/include/mpif-values.py index 53159d5d8dd..b74fbcbaf1f 100755 --- a/ompi/include/mpif-values.py +++ b/ompi/include/mpif-values.py @@ -301,6 +301,7 @@ 'MPI_ERR_SESSION': 78, 'MPI_ERR_VALUE_TOO_LARGE': 79, 'MPI_ERR_ERRHANDLER': 80, + 'MPI_ERR_NOTIFY_IDX': 81, 'MPI_ERR_LASTCODE': 92, 'MPI_IDENT': 0, 'MPI_CONGRUENT': 1, diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index bd05a6f11b7..83c7af9305e 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -216,7 +216,7 @@ typedef int (*ompi_osc_base_module_put_fn_t)(const void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); -typedef int (*ompi_osc_base_module_put_with_notify_fn_t)(const void *origin_addr, +typedef int (*ompi_osc_base_module_put_notify_fn_t)(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -235,7 +235,7 @@ typedef int (*ompi_osc_base_module_get_fn_t)(void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); -typedef int (*ompi_osc_base_module_get_with_notify_fn_t)(void *origin_addr, +typedef int (*ompi_osc_base_module_get_notify_fn_t)(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -294,7 +294,7 @@ typedef int (*ompi_osc_base_module_rput_fn_t)(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); -typedef int (*ompi_osc_base_module_rput_with_notify_fn_t)(const void *origin_addr, +typedef int (*ompi_osc_base_module_rput_notify_fn_t)(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -315,7 +315,7 @@ typedef int (*ompi_osc_base_module_rget_fn_t)(void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); -typedef int (*ompi_osc_base_module_rget_with_notify_fn_t)(void *origin_addr, +typedef int (*ompi_osc_base_module_rget_notify_fn_t)(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -418,18 +418,18 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_free_fn_t osc_free; ompi_osc_base_module_put_fn_t osc_put; - ompi_osc_base_module_put_with_notify_fn_t osc_put_with_notify; + ompi_osc_base_module_put_notify_fn_t osc_put_notify; ompi_osc_base_module_get_fn_t osc_get; - ompi_osc_base_module_get_with_notify_fn_t osc_get_with_notify; + ompi_osc_base_module_get_notify_fn_t osc_get_notify; ompi_osc_base_module_accumulate_fn_t osc_accumulate; ompi_osc_base_module_compare_and_swap_fn_t osc_compare_and_swap; ompi_osc_base_module_fetch_and_op_fn_t osc_fetch_and_op; ompi_osc_base_module_get_accumulate_fn_t osc_get_accumulate; ompi_osc_base_module_rput_fn_t osc_rput; - ompi_osc_base_module_rput_with_notify_fn_t osc_rput_with_notify; + ompi_osc_base_module_rput_notify_fn_t osc_rput_notify; ompi_osc_base_module_rget_fn_t osc_rget; - ompi_osc_base_module_rget_with_notify_fn_t osc_rget_with_notify; + ompi_osc_base_module_rget_notify_fn_t osc_rget_notify; ompi_osc_base_module_raccumulate_fn_t osc_raccumulate; ompi_osc_base_module_rget_accumulate_fn_t osc_rget_accumulate; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index b7d6dadfd49..200ec8b3de8 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -107,7 +107,7 @@ int ompi_osc_sm_detach(struct ompi_win_t *win, const void *base); int ompi_osc_sm_free(struct ompi_win_t *win); -// TODO: add put/get_with_notify prototypes +// TODO: add put/get_notify prototypes int ompi_osc_sm_put(const void *origin_addr, size_t origin_count, @@ -118,7 +118,7 @@ int ompi_osc_sm_put(const void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); - int ompi_osc_sm_put_with_notify(const void *origin_addr, + int ompi_osc_sm_put_notify(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -137,7 +137,7 @@ int ompi_osc_sm_get(void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); -int ompi_osc_sm_get_with_notify(void *origin_addr, +int ompi_osc_sm_get_notify(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -196,7 +196,7 @@ int ompi_osc_sm_rput(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); -int ompi_osc_sm_rput_with_notify(const void *origin_addr, +int ompi_osc_sm_rput_notify(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -217,7 +217,7 @@ int ompi_osc_sm_rget(void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); -int ompi_osc_sm_rget_with_notify(void *origin_addr, +int ompi_osc_sm_rget_notify(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index ba19d8c08cf..4391a375ebc 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -60,7 +60,7 @@ ompi_osc_sm_rput(const void *origin_addr, } int -ompi_osc_sm_rput_with_notify(const void *origin_addr, +ompi_osc_sm_rput_notify(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -143,7 +143,7 @@ ompi_osc_sm_rget(void *origin_addr, } int -ompi_osc_sm_rget_with_notify(void *origin_addr, +ompi_osc_sm_rget_notify(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -323,7 +323,7 @@ ompi_osc_sm_put(const void *origin_addr, int -ompi_osc_sm_put_with_notify(const void *origin_addr, +ompi_osc_sm_put_notify(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, @@ -392,7 +392,7 @@ ompi_osc_sm_get(void *origin_addr, int -ompi_osc_sm_get_with_notify(void *origin_addr, +ompi_osc_sm_get_notify(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index 11f0ccc2e47..e7613c86f6e 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -79,18 +79,18 @@ ompi_osc_sm_module_t ompi_osc_sm_module_template = { .osc_free = ompi_osc_sm_free, .osc_put = ompi_osc_sm_put, - .osc_put_with_notify = ompi_osc_sm_put_with_notify, + .osc_put_notify = ompi_osc_sm_put_notify, .osc_get = ompi_osc_sm_get, - .osc_get_with_notify = ompi_osc_sm_get_with_notify, + .osc_get_notify = ompi_osc_sm_get_notify, .osc_accumulate = ompi_osc_sm_accumulate, .osc_compare_and_swap = ompi_osc_sm_compare_and_swap, .osc_fetch_and_op = ompi_osc_sm_fetch_and_op, .osc_get_accumulate = ompi_osc_sm_get_accumulate, .osc_rput = ompi_osc_sm_rput, - .osc_rput_with_notify = ompi_osc_sm_rput_with_notify, + .osc_rput_notify = ompi_osc_sm_rput_notify, .osc_rget = ompi_osc_sm_rget, - .osc_rget_with_notify = ompi_osc_sm_rget_with_notify, + .osc_rget_notify = ompi_osc_sm_rget_notify, .osc_raccumulate = ompi_osc_sm_raccumulate, .osc_rget_accumulate = ompi_osc_sm_rget_accumulate, diff --git a/ompi/mpi/bindings/ompi_bindings/consts.py b/ompi/mpi/bindings/ompi_bindings/consts.py index 43bca486b57..759b342f64a 100644 --- a/ompi/mpi/bindings/ompi_bindings/consts.py +++ b/ompi/mpi/bindings/ompi_bindings/consts.py @@ -23,6 +23,7 @@ 'MPI_SUCCESS', 'MPI_ERR_BUFFER', 'MPI_ERR_COUNT', + 'MPI_ERR_NOTIFY_IDX' 'MPI_ERR_TYPE', 'MPI_ERR_TAG', 'MPI_ERR_COMM', diff --git a/ompi/mpi/c/Makefile.am b/ompi/mpi/c/Makefile.am index 25b871fa7d4..f6757d669dd 100644 --- a/ompi/mpi/c/Makefile.am +++ b/ompi/mpi/c/Makefile.am @@ -223,6 +223,7 @@ prototype_sources = \ get_accumulate.c.in \ get_address.c.in \ get.c.in \ + get_notify.c.in \ get_count.c.in \ get_elements.c.in \ get_elements_x.c.in \ @@ -341,6 +342,7 @@ prototype_sources = \ psend_init.c.in \ publish_name.c.in \ put.c.in \ + put_notify.c.in \ query_thread.c.in \ raccumulate.c.in \ recv.c.in \ diff --git a/ompi/mpi/c/get_notify.c.in b/ompi/mpi/c/get_notify.c.in new file mode 100644 index 00000000000..1bad16944ab --- /dev/null +++ b/ompi/mpi/c/get_notify.c.in @@ -0,0 +1,77 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/runtime/ompi_spc.h" + +PROTOTYPE ERROR_CLASS get_notify(BUFFER_OUT origin_addr, COUNT origin_count, + DATATYPE origin_datatype, INT target_rank, + AINT target_disp, COUNT target_count, + DATATYPE target_datatype, INT notification_idx, WIN win) +{ + int rc; + + SPC_RECORD(OMPI_SPC_GET_NOTIFY, 1); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + rc = MPI_ERR_NOTIFY_IDX; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS; + + rc = win->w_osc_module->osc_get_notify(origin_addr, origin_count, origin_datatype, + target_rank, target_disp, target_count, + target_datatype, notification_idx, win); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/put_notify.c.in b/ompi/mpi/c/put_notify.c.in new file mode 100644 index 00000000000..14ee5c7e365 --- /dev/null +++ b/ompi/mpi/c/put_notify.c.in @@ -0,0 +1,80 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/runtime/ompi_spc.h" + +PROTOTYPE ERROR_CLASS put_notify(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype, + INT target_rank, AINT target_disp, COUNT target_count, + DATATYPE target_datatype, INT notification_idx, WIN win) +{ + int rc; + + SPC_RECORD(OMPI_SPC_PUT_NOTIFY, 1); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if (NULL == target_datatype || + MPI_DATATYPE_NULL == target_datatype) { + rc = MPI_ERR_TYPE; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + rc = MPI_ERR_NOTIFY_IDX; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS; + + rc = win->w_osc_module->osc_put_notify(origin_addr, origin_count, origin_datatype, + target_rank, target_disp, target_count, + target_datatype, notification_idx, win); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c index 6f1d8aa7d6a..1d25545c80b 100644 --- a/ompi/runtime/ompi_spc.c +++ b/ompi/runtime/ompi_spc.c @@ -71,8 +71,10 @@ static const ompi_spc_event_t ompi_spc_events_desc[OMPI_SPC_NUM_COUNTERS] = { SET_COUNTER_ARRAY(OMPI_SPC_SENDRECV, "The number of times MPI_Sendrecv was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_SENDRECV_REPLACE, "The number of times MPI_Sendrecv_replace was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_PUT, "The number of times MPI_Put was called.", false, false), + SET_COUNTER_ARRAY(OMPI_SPC_PUT_NOTIFY, "The number of times MPI_Put_notify was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_RPUT, "The number of times MPI_Rput was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_GET, "The number of times MPI_Get was called.", false, false), + SET_COUNTER_ARRAY(OMPI_SPC_GET_NOTIFY, "The number of times MPI_Get was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_RGET, "The number of times MPI_Rget was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_PROBE, "The number of times MPI_Probe was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_IPROBE, "The number of times MPI_Iprobe was called.", false, false), diff --git a/ompi/runtime/ompi_spc.h b/ompi/runtime/ompi_spc.h index 76ec7f25f16..3d0efd257b3 100644 --- a/ompi/runtime/ompi_spc.h +++ b/ompi/runtime/ompi_spc.h @@ -58,8 +58,10 @@ typedef enum ompi_spc_counters { OMPI_SPC_SENDRECV, OMPI_SPC_SENDRECV_REPLACE, OMPI_SPC_PUT, + OMPI_SPC_PUT_NOTIFY, OMPI_SPC_RPUT, OMPI_SPC_GET, + OMPI_SPC_GET_NOTIFY, OMPI_SPC_RGET, OMPI_SPC_PROBE, OMPI_SPC_IPROBE, From 088364cb39970c5e650494cf3e23235424e12ce8 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Mon, 22 Dec 2025 09:46:28 -0500 Subject: [PATCH 04/15] Edits for Public APIs: put_with_notify get_with_notify Signed-off-by: Joseph Antony --- ompi/mca/osc/sm/osc_sm.h | 1 - ompi/runtime/ompi_spc.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index 200ec8b3de8..0aca3b50892 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -107,7 +107,6 @@ int ompi_osc_sm_detach(struct ompi_win_t *win, const void *base); int ompi_osc_sm_free(struct ompi_win_t *win); -// TODO: add put/get_notify prototypes int ompi_osc_sm_put(const void *origin_addr, size_t origin_count, diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c index 1d25545c80b..dcbbe04b256 100644 --- a/ompi/runtime/ompi_spc.c +++ b/ompi/runtime/ompi_spc.c @@ -74,7 +74,7 @@ static const ompi_spc_event_t ompi_spc_events_desc[OMPI_SPC_NUM_COUNTERS] = { SET_COUNTER_ARRAY(OMPI_SPC_PUT_NOTIFY, "The number of times MPI_Put_notify was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_RPUT, "The number of times MPI_Rput was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_GET, "The number of times MPI_Get was called.", false, false), - SET_COUNTER_ARRAY(OMPI_SPC_GET_NOTIFY, "The number of times MPI_Get was called.", false, false), + SET_COUNTER_ARRAY(OMPI_SPC_GET_NOTIFY, "The number of times MPI_Get_notify was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_RGET, "The number of times MPI_Rget was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_PROBE, "The number of times MPI_Probe was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_IPROBE, "The number of times MPI_Iprobe was called.", false, false), From f0ee69f578e1b6e9d14ba98540ac96e75f2d94c0 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Tue, 3 Feb 2026 08:29:27 -0500 Subject: [PATCH 05/15] Notified RMA counters memory allocation in the shared memory segment for a single and multi rank window. Signed-off-by: Joseph Antony --- ompi/mca/osc/sm/osc_sm.h | 3 ++- ompi/mca/osc/sm/osc_sm_component.c | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index 0aca3b50892..bf80c082ac8 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -22,6 +22,7 @@ typedef uint64_t osc_sm_post_type_t; typedef opal_atomic_uint64_t osc_sm_post_atomic_type_t; #define OSC_SM_POST_BITS 6 #define OSC_SM_POST_MASK 0x3f +#define OSC_SM_MAX_NOTIFY_COUNTERS 16 /* data shared across all peers */ struct ompi_osc_sm_global_state_t { @@ -79,7 +80,7 @@ struct ompi_osc_sm_module_t { size_t *sizes; void **bases; ptrdiff_t *disp_units; - uint64_t **notify_counters; + uint64_t *notify_counters; ompi_group_t *start_group; diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index e7613c86f6e..5500a2bb412 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -255,12 +255,17 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis module->posts = calloc (1, sizeof(module->posts[0]) + sizeof (module->posts[0][0])); if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; module->posts[0] = (osc_sm_post_atomic_type_t *) (module->posts + 1); + + /* allocate notify counters for single process case */ + module->notify_counters = calloc(OSC_SM_MAX_NOTIFY_COUNTERS, sizeof(uint64_t)); + if (NULL == module->notify_counters) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; } else { unsigned long total, *rbuf; int i, flag; size_t pagesize; size_t state_size; size_t posts_size, post_size = (comm_size + OSC_SM_POST_MASK) / (OSC_SM_POST_MASK + 1); + size_t notify_counters_size; size_t data_base_size; opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output, @@ -316,7 +321,9 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis state_size += OPAL_ALIGN_PAD_AMOUNT(state_size, 64); posts_size = comm_size * post_size * sizeof (module->posts[0][0]); posts_size += OPAL_ALIGN_PAD_AMOUNT(posts_size, 64); - data_base_size = state_size + posts_size; + notify_counters_size = OSC_SM_MAX_NOTIFY_COUNTERS * sizeof(uint64_t); + notify_counters_size += OPAL_ALIGN_PAD_AMOUNT(notify_counters_size, 64); + data_base_size = state_size + posts_size + notify_counters_size; data_base_size += OPAL_ALIGN_PAD_AMOUNT(data_base_size, pagesize); if (0 == ompi_comm_rank (module->comm)) { char *data_file; @@ -377,6 +384,12 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size); module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1); + /* set up notify counters in shared memory after node_states */ + module->notify_counters = (uint64_t *) ((char *)(module->node_states + comm_size) + + OPAL_ALIGN_PAD_AMOUNT((uintptr_t)(module->node_states + comm_size), 64)); + /* zero out notify counters */ + memset(module->notify_counters, 0, OSC_SM_MAX_NOTIFY_COUNTERS * sizeof(uint64_t)); + for (i = 0, total = data_base_size ; i < comm_size ; ++i) { if (i > 0) { module->posts[i] = module->posts[i - 1] + post_size; @@ -555,6 +568,7 @@ ompi_osc_sm_free(struct ompi_win_t *win) module->comm->c_coll->coll_barrier_module); opal_shmem_segment_detach (&module->seg_ds); + /* notify_counters points into shared memory segment, no separate free needed */ } else { free(module->node_states); free(module->global_state); @@ -562,6 +576,8 @@ ompi_osc_sm_free(struct ompi_win_t *win) mca_mpool_base_default_module->mpool_free(mca_mpool_base_default_module, module->bases[0]); } + /* free notify_counters for single process case */ + free(module->notify_counters); } free(module->disp_units); free(module->outstanding_locks); From 645739a2a839c4b801ba6b54a1622897a4856163 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Wed, 4 Feb 2026 07:47:01 -0500 Subject: [PATCH 06/15] Editing Notified RMA implementation Signed-off-by: Joseph Antony --- ompi/mca/osc/sm/osc_sm_comm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index 4391a375ebc..6cc5384d750 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -98,7 +98,7 @@ ompi_osc_sm_rput_notify(const void *origin_addr, *ompi_req = &ompi_request_empty; opal_atomic_wmb(); - opal_atomic_add(&module->notify_counters[target][notify], 1); + opal_atomic_add(&module->notify_counters[notify], 1); return OMPI_SUCCESS; } @@ -181,7 +181,7 @@ ompi_osc_sm_rget_notify(void *origin_addr, *ompi_req = &ompi_request_empty; opal_atomic_rmb(); - opal_atomic_add(&module->notify_counters[target][notify], 1); + opal_atomic_add(&module->notify_counters[notify], 1); return OMPI_SUCCESS; } @@ -355,7 +355,7 @@ ompi_osc_sm_put_notify(const void *origin_addr, } opal_atomic_wmb(); - opal_atomic_add(&module->notify_counters[target][notify], 1); + opal_atomic_add(&module->notify_counters[notify], 1); return ret; } @@ -422,7 +422,7 @@ ompi_osc_sm_get_notify(void *origin_addr, return ret; } opal_atomic_rmb(); - opal_atomic_add(&module->notify_counters[target][notify], 1); + opal_atomic_add(&module->notify_counters[notify], 1); return ret; } From 7441b3ef8cf17bc5c31b623373de7336ab76e7ff Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Wed, 18 Feb 2026 22:17:07 -0500 Subject: [PATCH 07/15] Editing Notified RMA implementation with new design Signed-off-by: Joseph Antony --- ompi/mca/osc/sm/osc_sm.h | 3 +++ ompi/mca/osc/sm/osc_sm_comm.c | 35 +++++++++++++++++++++++++---- ompi/mca/osc/sm/osc_sm_component.c | 36 ++++++++++++++++++++---------- 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index bf80c082ac8..cb104e5df15 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -48,6 +48,9 @@ struct ompi_osc_sm_node_state_t { opal_atomic_int32_t complete_count; ompi_osc_sm_lock_t lock; opal_atomic_lock_t accumulate_lock; + uint32_t notify_counter_count; + uint64_t notify_counter_offset; /* offset from segment_base, not raw pointer */ + }; typedef struct ompi_osc_sm_node_state_t ompi_osc_sm_node_state_t; diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index 6cc5384d750..359cdf7147f 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -20,6 +20,17 @@ #include "osc_sm.h" +static inline uint64_t *osc_sm_target_notify_base(ompi_osc_sm_module_t *module, int target) +{ + if (NULL == module->segment_base) { + /* single-rank path: notify_counters is a regular local allocation */ + return module->notify_counters; + } + + return (uint64_t *) ((char *) module->segment_base + + module->node_states[target].notify_counter_offset); +} + int ompi_osc_sm_rput(const void *origin_addr, size_t origin_count, @@ -97,8 +108,12 @@ ompi_osc_sm_rput_notify(const void *origin_addr, * complete. */ *ompi_req = &ompi_request_empty; + if (notify < 0 || (uint32_t) notify >= module->node_states[target].notify_counter_count) { + return OMPI_ERR_BAD_PARAM; + } + opal_atomic_wmb(); - opal_atomic_add(&module->notify_counters[notify], 1); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); return OMPI_SUCCESS; } @@ -180,8 +195,12 @@ ompi_osc_sm_rget_notify(void *origin_addr, * complete. */ *ompi_req = &ompi_request_empty; + if (notify < 0 || (uint32_t) notify >= module->node_states[target].notify_counter_count) { + return OMPI_ERR_BAD_PARAM; + } + opal_atomic_rmb(); - opal_atomic_add(&module->notify_counters[notify], 1); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); return OMPI_SUCCESS; } @@ -354,8 +373,12 @@ ompi_osc_sm_put_notify(const void *origin_addr, return ret; } + if (notify < 0 || (uint32_t) notify >= module->node_states[target].notify_counter_count) { + return OMPI_ERR_BAD_PARAM; + } + opal_atomic_wmb(); - opal_atomic_add(&module->notify_counters[notify], 1); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); return ret; } @@ -421,8 +444,12 @@ ompi_osc_sm_get_notify(void *origin_addr, if (OMPI_SUCCESS != ret) { return ret; } + if (notify < 0 || (uint32_t) notify >= module->node_states[target].notify_counter_count) { + return OMPI_ERR_BAD_PARAM; + } + opal_atomic_rmb(); - opal_atomic_add(&module->notify_counters[notify], 1); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); return ret; } diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index 5500a2bb412..7954ef6963e 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -259,8 +259,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis /* allocate notify counters for single process case */ module->notify_counters = calloc(OSC_SM_MAX_NOTIFY_COUNTERS, sizeof(uint64_t)); if (NULL == module->notify_counters) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + module->node_states[0].notify_counter_count = OSC_SM_MAX_NOTIFY_COUNTERS; + module->node_states[0].notify_counter_offset = 0; } else { - unsigned long total, *rbuf; + unsigned long total, total_counters, gather_values[2], *rbuf; int i, flag; size_t pagesize; size_t state_size; @@ -274,7 +276,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis /* get the pagesize */ pagesize = opal_getpagesize(); - rbuf = malloc(sizeof(unsigned long) * comm_size); + rbuf = malloc(sizeof(unsigned long) * comm_size * 2 ); if (NULL == rbuf) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; /* Note that the alloc_shared_noncontig info key only has @@ -298,9 +300,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis "allocating window using contiguous strategy"); } - total = size; - ret = module->comm->c_coll->coll_allgather(&total, 1, MPI_UNSIGNED_LONG, - rbuf, 1, MPI_UNSIGNED_LONG, + gather_values[0] = size; + gather_values[1] = OSC_SM_MAX_NOTIFY_COUNTERS; + ret = module->comm->c_coll->coll_allgather(gather_values, 2, MPI_UNSIGNED_LONG, + rbuf, 2, MPI_UNSIGNED_LONG, module->comm, module->comm->c_coll->coll_allgather_module); if (OMPI_SUCCESS != ret) { @@ -309,8 +312,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis } total = 0; + total_counters = 0; for (i = 0 ; i < comm_size ; ++i) { - total += rbuf[i]; + total += rbuf[2 * i]; + total_counters += rbuf[2 * i + 1]; if (module->noncontig) { total += OPAL_ALIGN_PAD_AMOUNT(total, pagesize); } @@ -321,7 +326,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis state_size += OPAL_ALIGN_PAD_AMOUNT(state_size, 64); posts_size = comm_size * post_size * sizeof (module->posts[0][0]); posts_size += OPAL_ALIGN_PAD_AMOUNT(posts_size, 64); - notify_counters_size = OSC_SM_MAX_NOTIFY_COUNTERS * sizeof(uint64_t); + notify_counters_size = total_counters * sizeof(uint64_t); notify_counters_size += OPAL_ALIGN_PAD_AMOUNT(notify_counters_size, 64); data_base_size = state_size + posts_size + notify_counters_size; data_base_size += OPAL_ALIGN_PAD_AMOUNT(data_base_size, pagesize); @@ -388,17 +393,23 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis module->notify_counters = (uint64_t *) ((char *)(module->node_states + comm_size) + OPAL_ALIGN_PAD_AMOUNT((uintptr_t)(module->node_states + comm_size), 64)); /* zero out notify counters */ - memset(module->notify_counters, 0, OSC_SM_MAX_NOTIFY_COUNTERS * sizeof(uint64_t)); + memset(module->notify_counters, 0, total_counters * sizeof(uint64_t)); - for (i = 0, total = data_base_size ; i < comm_size ; ++i) { + for (i = 0, total = data_base_size, total_counters = 0 ; i < comm_size ; ++i) { if (i > 0) { module->posts[i] = module->posts[i - 1] + post_size; } - module->sizes[i] = rbuf[i]; + module->node_states[i].notify_counter_count = (uint32_t) rbuf[2 * i + 1]; + module->node_states[i].notify_counter_offset = + (uint64_t) ((char *) (module->notify_counters + total_counters) - + (char *) module->segment_base); + total_counters += rbuf[2 * i + 1]; + + module->sizes[i] = rbuf[2 * i]; if (module->sizes[i] || !module->noncontig) { module->bases[i] = ((char *) module->segment_base) + total; - total += rbuf[i]; + total += rbuf[2 * i]; if (module->noncontig) { total += OPAL_ALIGN_PAD_AMOUNT(total, pagesize); } @@ -412,7 +423,8 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis /* initialize my state shared */ module->my_node_state = &module->node_states[ompi_comm_rank(module->comm)]; - memset (module->my_node_state, 0, sizeof(*module->my_node_state)); + module->my_node_state->complete_count = 0; + memset (&module->my_node_state->lock, 0, sizeof(module->my_node_state->lock)); *base = module->bases[ompi_comm_rank(module->comm)]; From 6ee9b494a0a674584302ee9cf280669e0114242a Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Wed, 4 Mar 2026 13:19:28 -0500 Subject: [PATCH 08/15] Implementing Notify Query Signed-off-by: Joseph Antony --- ompi/include/mpi.h.in | 2 ++ ompi/mca/osc/osc.h | 5 ++++ ompi/mca/osc/sm/osc_sm.h | 4 +++ ompi/mca/osc/sm/osc_sm_comm.c | 19 +++++++++++++ ompi/mca/osc/sm/osc_sm_component.c | 1 + ompi/mca/osc/ubcl/osc_ubcl.c | 5 ++++ ompi/mpi/c/Makefile.am | 2 ++ ompi/mpi/c/win_get_notify_value.c.in | 41 ++++++++++++++++++++++++++++ 8 files changed, 79 insertions(+) create mode 100644 ompi/mpi/c/win_get_notify_value.c.in diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in index d34624cd1d2..ed6d69d6222 100644 --- a/ompi/include/mpi.h.in +++ b/ompi/include/mpi.h.in @@ -2619,6 +2619,7 @@ OMPI_DECLSPEC int MPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandle OMPI_DECLSPEC int MPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int MPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int MPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); +OMPI_DECLSPEC int MPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); OMPI_DECLSPEC int MPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); @@ -3807,6 +3808,7 @@ OMPI_DECLSPEC int PMPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandl OMPI_DECLSPEC int PMPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int PMPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int PMPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); +OMPI_DECLSPEC int PMPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); OMPI_DECLSPEC int PMPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index 83c7af9305e..b43757b9b5c 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -245,6 +245,10 @@ typedef int (*ompi_osc_base_module_get_notify_fn_t)(void *origin_addr, int notify, struct ompi_win_t *win); +typedef int (*ompi_osc_base_module_win_get_notify_value_fn_t)(struct ompi_win_t *win, + int notify, + MPI_Count *value); + typedef int (*ompi_osc_base_module_accumulate_fn_t)(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -421,6 +425,7 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_put_notify_fn_t osc_put_notify; ompi_osc_base_module_get_fn_t osc_get; ompi_osc_base_module_get_notify_fn_t osc_get_notify; + ompi_osc_base_module_win_get_notify_value_fn_t osc_win_get_notify_value; ompi_osc_base_module_accumulate_fn_t osc_accumulate; ompi_osc_base_module_compare_and_swap_fn_t osc_compare_and_swap; ompi_osc_base_module_fetch_and_op_fn_t osc_fetch_and_op; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index cb104e5df15..f80c0116d73 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -149,6 +149,10 @@ int ompi_osc_sm_get_notify(void *origin_addr, struct ompi_datatype_t *target_dt, int notify, struct ompi_win_t *win); + +int ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, + int notify, + MPI_Count *value); int ompi_osc_sm_accumulate(const void *origin_addr, size_t origin_count, diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index 359cdf7147f..0597ecf3095 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -17,6 +17,7 @@ #include "ompi/mca/osc/osc.h" #include "ompi/mca/osc/base/base.h" #include "ompi/mca/osc/base/osc_base_obj_convert.h" +#include "ompi/communicator/communicator.h" #include "osc_sm.h" @@ -31,6 +32,24 @@ static inline uint64_t *osc_sm_target_notify_base(ompi_osc_sm_module_t *module, module->node_states[target].notify_counter_offset); } +int +ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, + int notify, + MPI_Count *value) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + int rank = ompi_comm_rank(module->comm); + + if (notify < 0 || (uint32_t) notify >= module->node_states[rank].notify_counter_count) { + return OMPI_ERR_BAD_PARAM; + } + + opal_atomic_rmb(); + *value = (MPI_Count) osc_sm_target_notify_base(module, rank)[notify]; + + return OMPI_SUCCESS; +} + int ompi_osc_sm_rput(const void *origin_addr, size_t origin_count, diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index 7954ef6963e..0a3f7002337 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -82,6 +82,7 @@ ompi_osc_sm_module_t ompi_osc_sm_module_template = { .osc_put_notify = ompi_osc_sm_put_notify, .osc_get = ompi_osc_sm_get, .osc_get_notify = ompi_osc_sm_get_notify, + .osc_win_get_notify_value = ompi_osc_sm_win_get_notify_value, .osc_accumulate = ompi_osc_sm_accumulate, .osc_compare_and_swap = ompi_osc_sm_compare_and_swap, .osc_fetch_and_op = ompi_osc_sm_fetch_and_op, diff --git a/ompi/mca/osc/ubcl/osc_ubcl.c b/ompi/mca/osc/ubcl/osc_ubcl.c index 5e81ed1add3..d358605244b 100644 --- a/ompi/mca/osc/ubcl/osc_ubcl.c +++ b/ompi/mca/osc/ubcl/osc_ubcl.c @@ -80,14 +80,19 @@ mca_osc_ubcl_module_t mca_osc_ubcl_module_template = { win_free, ompi_osc_ubcl_put, + NULL, ompi_osc_ubcl_get, + NULL, + NULL, ompi_osc_ubcl_accumulate, ompi_osc_ubcl_compare_and_swap, ompi_osc_ubcl_fetch_and_op, ompi_osc_ubcl_get_accumulate, ompi_osc_ubcl_rput, + NULL, ompi_osc_ubcl_rget, + NULL, ompi_osc_ubcl_raccumulate, ompi_osc_ubcl_rget_accumulate, diff --git a/ompi/mpi/c/Makefile.am b/ompi/mpi/c/Makefile.am index f6757d669dd..12c8752869c 100644 --- a/ompi/mpi/c/Makefile.am +++ b/ompi/mpi/c/Makefile.am @@ -486,6 +486,7 @@ prototype_sources = \ win_get_group.c.in \ win_get_info.c.in \ win_get_name.c.in \ + win_get_notify_value.c.in \ win_lock_all.c.in \ win_lock.c.in \ win_post.c.in \ @@ -956,6 +957,7 @@ interface_profile_sources = \ win_get_group_generated.c \ win_get_info_generated.c \ win_get_name_generated.c \ + win_get_notify_value_generated.c \ win_lock_all_generated.c \ win_lock_generated.c \ win_post_generated.c \ diff --git a/ompi/mpi/c/win_get_notify_value.c.in b/ompi/mpi/c/win_get_notify_value.c.in new file mode 100644 index 00000000000..228999c13ea --- /dev/null +++ b/ompi/mpi/c/win_get_notify_value.c.in @@ -0,0 +1,41 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_get_notify_value(WIN win, INT notification_idx, ELEMENT_COUNT value) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (notification_idx < 0) { + rc = MPI_ERR_NOTIFY_IDX; + } else if (NULL == value) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_get_notify_value(win, notification_idx, value); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} From d2d71f8a7019296cbd418f0c358cf0e8a1f3cb35 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Wed, 4 Mar 2026 13:39:55 -0500 Subject: [PATCH 09/15] Changes to Notify Query Signed-off-by: Joseph Antony --- ompi/mca/osc/osc.h | 2 +- ompi/mca/osc/sm/osc_sm.h | 2 +- ompi/mca/osc/sm/osc_sm_comm.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index b43757b9b5c..8aebf7446b1 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -247,7 +247,7 @@ typedef int (*ompi_osc_base_module_get_notify_fn_t)(void *origin_addr, typedef int (*ompi_osc_base_module_win_get_notify_value_fn_t)(struct ompi_win_t *win, int notify, - MPI_Count *value); + OMPI_MPI_COUNT_TYPE *value); typedef int (*ompi_osc_base_module_accumulate_fn_t)(const void *origin_addr, size_t origin_count, diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index f80c0116d73..c294cc7d1f6 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -152,7 +152,7 @@ int ompi_osc_sm_get_notify(void *origin_addr, int ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, int notify, - MPI_Count *value); + OMPI_MPI_COUNT_TYPE *value); int ompi_osc_sm_accumulate(const void *origin_addr, size_t origin_count, diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index 0597ecf3095..b0b3bde37f8 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -35,7 +35,7 @@ static inline uint64_t *osc_sm_target_notify_base(ompi_osc_sm_module_t *module, int ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, int notify, - MPI_Count *value) + OMPI_MPI_COUNT_TYPE *value) { ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; int rank = ompi_comm_rank(module->comm); @@ -45,7 +45,7 @@ ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, } opal_atomic_rmb(); - *value = (MPI_Count) osc_sm_target_notify_base(module, rank)[notify]; + *value = (OMPI_MPI_COUNT_TYPE) osc_sm_target_notify_base(module, rank)[notify]; return OMPI_SUCCESS; } From bfe1797c5735521be9f97c30d09d475669e04760 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Wed, 4 Mar 2026 18:24:42 -0500 Subject: [PATCH 10/15] Implemented Set Notify Query Signed-off-by: Joseph Antony --- ompi/include/mpi.h.in | 2 ++ ompi/mca/osc/osc.h | 5 ++++ ompi/mca/osc/sm/osc_sm.h | 4 +++ ompi/mca/osc/sm/osc_sm_comm.c | 21 +++++++++++++- ompi/mca/osc/sm/osc_sm_component.c | 1 + ompi/mpi/c/Makefile.am | 2 ++ ompi/mpi/c/win_set_notify_value.c.in | 41 ++++++++++++++++++++++++++++ 7 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 ompi/mpi/c/win_set_notify_value.c.in diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in index ed6d69d6222..eff39e9b1f0 100644 --- a/ompi/include/mpi.h.in +++ b/ompi/include/mpi.h.in @@ -2620,6 +2620,7 @@ OMPI_DECLSPEC int MPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int MPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int MPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); OMPI_DECLSPEC int MPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int MPI_Win_set_notify_value(MPI_Win win, int notification_idx, MPI_Count value); OMPI_DECLSPEC int MPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); @@ -3809,6 +3810,7 @@ OMPI_DECLSPEC int PMPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int PMPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int PMPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); OMPI_DECLSPEC int PMPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int PMPI_Win_set_notify_value(MPI_Win win, int notification_idx, MPI_Count value); OMPI_DECLSPEC int PMPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index 8aebf7446b1..ca3b9aac3ef 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -248,6 +248,10 @@ typedef int (*ompi_osc_base_module_get_notify_fn_t)(void *origin_addr, typedef int (*ompi_osc_base_module_win_get_notify_value_fn_t)(struct ompi_win_t *win, int notify, OMPI_MPI_COUNT_TYPE *value); + +typedef int (*ompi_osc_base_module_win_set_notify_value_fn_t)(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE value); typedef int (*ompi_osc_base_module_accumulate_fn_t)(const void *origin_addr, size_t origin_count, @@ -426,6 +430,7 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_get_fn_t osc_get; ompi_osc_base_module_get_notify_fn_t osc_get_notify; ompi_osc_base_module_win_get_notify_value_fn_t osc_win_get_notify_value; + ompi_osc_base_module_win_set_notify_value_fn_t osc_win_set_notify_value; ompi_osc_base_module_accumulate_fn_t osc_accumulate; ompi_osc_base_module_compare_and_swap_fn_t osc_compare_and_swap; ompi_osc_base_module_fetch_and_op_fn_t osc_fetch_and_op; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index c294cc7d1f6..cec3fa44bb2 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -153,6 +153,10 @@ int ompi_osc_sm_get_notify(void *origin_addr, int ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, int notify, OMPI_MPI_COUNT_TYPE *value); + +int ompi_osc_sm_win_set_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE value); int ompi_osc_sm_accumulate(const void *origin_addr, size_t origin_count, diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index b0b3bde37f8..b3879c11949 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -44,8 +44,27 @@ ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, return OMPI_ERR_BAD_PARAM; } - opal_atomic_rmb(); + *value = (OMPI_MPI_COUNT_TYPE) osc_sm_target_notify_base(module, rank)[notify]; + opal_atomic_rmb(); + + return OMPI_SUCCESS; +} + +int +ompi_osc_sm_win_set_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE value) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + int rank = ompi_comm_rank(module->comm); + + if (notify < 0 || (uint32_t) notify >= module->node_states[rank].notify_counter_count) { + return OMPI_ERR_BAD_PARAM; + } + + opal_atomic_wmb(); + osc_sm_target_notify_base(module, rank)[notify] = (uint64_t) value; return OMPI_SUCCESS; } diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index 0a3f7002337..e64d04d6130 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -83,6 +83,7 @@ ompi_osc_sm_module_t ompi_osc_sm_module_template = { .osc_get = ompi_osc_sm_get, .osc_get_notify = ompi_osc_sm_get_notify, .osc_win_get_notify_value = ompi_osc_sm_win_get_notify_value, + .osc_win_set_notify_value = ompi_osc_sm_win_set_notify_value, .osc_accumulate = ompi_osc_sm_accumulate, .osc_compare_and_swap = ompi_osc_sm_compare_and_swap, .osc_fetch_and_op = ompi_osc_sm_fetch_and_op, diff --git a/ompi/mpi/c/Makefile.am b/ompi/mpi/c/Makefile.am index 12c8752869c..fef08119635 100644 --- a/ompi/mpi/c/Makefile.am +++ b/ompi/mpi/c/Makefile.am @@ -487,6 +487,7 @@ prototype_sources = \ win_get_info.c.in \ win_get_name.c.in \ win_get_notify_value.c.in \ + win_set_notify_value.c.in \ win_lock_all.c.in \ win_lock.c.in \ win_post.c.in \ @@ -958,6 +959,7 @@ interface_profile_sources = \ win_get_info_generated.c \ win_get_name_generated.c \ win_get_notify_value_generated.c \ + win_set_notify_value_generated.c \ win_lock_all_generated.c \ win_lock_generated.c \ win_post_generated.c \ diff --git a/ompi/mpi/c/win_set_notify_value.c.in b/ompi/mpi/c/win_set_notify_value.c.in new file mode 100644 index 00000000000..8a7d97567db --- /dev/null +++ b/ompi/mpi/c/win_set_notify_value.c.in @@ -0,0 +1,41 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_set_notify_value(WIN win, INT notification_idx, PARTITIONED_COUNT value) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (notification_idx < 0) { + rc = MPI_ERR_NOTIFY_IDX; + } else if (value < 0) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_set_notify_value(win, notification_idx, value); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} From 8ae3d484194dffe254d6522791d4d1b668778100 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Wed, 4 Mar 2026 18:25:43 -0500 Subject: [PATCH 11/15] Changes to Notify Query Signed-off-by: Joseph Antony --- ompi/mca/osc/sm/osc_sm_comm.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index b3879c11949..5d9032e19ea 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -44,10 +44,9 @@ ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, return OMPI_ERR_BAD_PARAM; } - - *value = (OMPI_MPI_COUNT_TYPE) osc_sm_target_notify_base(module, rank)[notify]; opal_atomic_rmb(); - + *value = (OMPI_MPI_COUNT_TYPE) osc_sm_target_notify_base(module, rank)[notify]; + return OMPI_SUCCESS; } From 409afbfe19c4f00a365df761d51ade1184fe6a4a Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Wed, 11 Mar 2026 18:20:13 -0400 Subject: [PATCH 12/15] Configuring memory barries to propagate the update Signed-off-by: Joseph Antony --- ompi/mca/osc/sm/osc_sm_comm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index 5d9032e19ea..6d17d914283 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -44,8 +44,8 @@ ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, return OMPI_ERR_BAD_PARAM; } - opal_atomic_rmb(); *value = (OMPI_MPI_COUNT_TYPE) osc_sm_target_notify_base(module, rank)[notify]; + opal_atomic_rmb(); return OMPI_SUCCESS; } @@ -62,8 +62,8 @@ ompi_osc_sm_win_set_notify_value(struct ompi_win_t *win, return OMPI_ERR_BAD_PARAM; } - opal_atomic_wmb(); osc_sm_target_notify_base(module, rank)[notify] = (uint64_t) value; + opal_atomic_wmb(); return OMPI_SUCCESS; } From f81e53f95afeefd1237eaf6a29861eb5bc42ab05 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Wed, 11 Mar 2026 19:26:11 -0400 Subject: [PATCH 13/15] Reset notify value Implementation Signed-off-by: Joseph Antony --- ompi/include/mpi.h.in | 2 ++ ompi/mca/osc/osc.h | 5 ++++ ompi/mca/osc/sm/osc_sm.h | 6 +++- ompi/mca/osc/sm/osc_sm_comm.c | 19 ++++++++++++ ompi/mca/osc/sm/osc_sm_component.c | 1 + ompi/mpi/c/Makefile.am | 2 ++ ompi/mpi/c/win_reset_notify_value.c.in | 41 ++++++++++++++++++++++++++ 7 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 ompi/mpi/c/win_reset_notify_value.c.in diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in index eff39e9b1f0..3a147a7a136 100644 --- a/ompi/include/mpi.h.in +++ b/ompi/include/mpi.h.in @@ -2621,6 +2621,7 @@ OMPI_DECLSPEC int MPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int MPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); OMPI_DECLSPEC int MPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); OMPI_DECLSPEC int MPI_Win_set_notify_value(MPI_Win win, int notification_idx, MPI_Count value); +OMPI_DECLSPEC int MPI_Win_reset_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); OMPI_DECLSPEC int MPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); @@ -3811,6 +3812,7 @@ OMPI_DECLSPEC int PMPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int PMPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); OMPI_DECLSPEC int PMPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); OMPI_DECLSPEC int PMPI_Win_set_notify_value(MPI_Win win, int notification_idx, MPI_Count value); +OMPI_DECLSPEC int PMPI_Win_reset_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); OMPI_DECLSPEC int PMPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index ca3b9aac3ef..6af72390f7f 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -253,6 +253,10 @@ typedef int (*ompi_osc_base_module_win_set_notify_value_fn_t)(struct ompi_win_t int notify, OMPI_MPI_COUNT_TYPE value); +typedef int (*ompi_osc_base_module_win_reset_notify_value_fn_t)(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + typedef int (*ompi_osc_base_module_accumulate_fn_t)(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -431,6 +435,7 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_get_notify_fn_t osc_get_notify; ompi_osc_base_module_win_get_notify_value_fn_t osc_win_get_notify_value; ompi_osc_base_module_win_set_notify_value_fn_t osc_win_set_notify_value; + ompi_osc_base_module_win_reset_notify_value_fn_t osc_win_reset_notify_value; ompi_osc_base_module_accumulate_fn_t osc_accumulate; ompi_osc_base_module_compare_and_swap_fn_t osc_compare_and_swap; ompi_osc_base_module_fetch_and_op_fn_t osc_fetch_and_op; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index cec3fa44bb2..ca776bd59a0 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -157,7 +157,11 @@ int ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, int ompi_osc_sm_win_set_notify_value(struct ompi_win_t *win, int notify, OMPI_MPI_COUNT_TYPE value); - + +int ompi_osc_sm_win_reset_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + int ompi_osc_sm_accumulate(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index 6d17d914283..40cbc9d2813 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -68,6 +68,25 @@ ompi_osc_sm_win_set_notify_value(struct ompi_win_t *win, return OMPI_SUCCESS; } +int +ompi_osc_sm_win_reset_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + int rank = ompi_comm_rank(module->comm); + + if (notify < 0 || (uint32_t) notify >= module->node_states[rank].notify_counter_count) { + return OMPI_ERR_BAD_PARAM; + } + + /* Atomically swap the counter to 0 and return the previous value */ + *value = (OMPI_MPI_COUNT_TYPE) opal_atomic_swap_64( + &osc_sm_target_notify_base(module, rank)[notify], 0); + + return OMPI_SUCCESS; +} + int ompi_osc_sm_rput(const void *origin_addr, size_t origin_count, diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index e64d04d6130..3ba892e0885 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -84,6 +84,7 @@ ompi_osc_sm_module_t ompi_osc_sm_module_template = { .osc_get_notify = ompi_osc_sm_get_notify, .osc_win_get_notify_value = ompi_osc_sm_win_get_notify_value, .osc_win_set_notify_value = ompi_osc_sm_win_set_notify_value, + .osc_win_reset_notify_value = ompi_osc_sm_win_reset_notify_value, .osc_accumulate = ompi_osc_sm_accumulate, .osc_compare_and_swap = ompi_osc_sm_compare_and_swap, .osc_fetch_and_op = ompi_osc_sm_fetch_and_op, diff --git a/ompi/mpi/c/Makefile.am b/ompi/mpi/c/Makefile.am index fef08119635..096da3ab171 100644 --- a/ompi/mpi/c/Makefile.am +++ b/ompi/mpi/c/Makefile.am @@ -488,6 +488,7 @@ prototype_sources = \ win_get_name.c.in \ win_get_notify_value.c.in \ win_set_notify_value.c.in \ + win_reset_notify_value.c.in \ win_lock_all.c.in \ win_lock.c.in \ win_post.c.in \ @@ -960,6 +961,7 @@ interface_profile_sources = \ win_get_name_generated.c \ win_get_notify_value_generated.c \ win_set_notify_value_generated.c \ + win_reset_notify_value_generated.c \ win_lock_all_generated.c \ win_lock_generated.c \ win_post_generated.c \ diff --git a/ompi/mpi/c/win_reset_notify_value.c.in b/ompi/mpi/c/win_reset_notify_value.c.in new file mode 100644 index 00000000000..99aa1755a76 --- /dev/null +++ b/ompi/mpi/c/win_reset_notify_value.c.in @@ -0,0 +1,41 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_reset_notify_value(WIN win, INT notification_idx, ELEMENT_COUNT value) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (notification_idx < 0) { + rc = MPI_ERR_NOTIFY_IDX; + } else if (NULL == value) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_reset_notify_value(win, notification_idx, value); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} From 80d8d03f22fd8053edcf4d502d624c0f37ebd4ad Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Fri, 13 Mar 2026 11:43:20 -0400 Subject: [PATCH 14/15] Addressing review comments and bug fixes Signed-off-by: Joseph Antony --- ompi/mca/osc/osc.h | 14 +++++++------- ompi/mca/osc/sm/osc_sm_comm.c | 12 ++++++------ ompi/mca/osc/ubcl/osc_ubcl.c | 5 ----- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index 6af72390f7f..002866a69da 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -430,21 +430,14 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_free_fn_t osc_free; ompi_osc_base_module_put_fn_t osc_put; - ompi_osc_base_module_put_notify_fn_t osc_put_notify; ompi_osc_base_module_get_fn_t osc_get; - ompi_osc_base_module_get_notify_fn_t osc_get_notify; - ompi_osc_base_module_win_get_notify_value_fn_t osc_win_get_notify_value; - ompi_osc_base_module_win_set_notify_value_fn_t osc_win_set_notify_value; - ompi_osc_base_module_win_reset_notify_value_fn_t osc_win_reset_notify_value; ompi_osc_base_module_accumulate_fn_t osc_accumulate; ompi_osc_base_module_compare_and_swap_fn_t osc_compare_and_swap; ompi_osc_base_module_fetch_and_op_fn_t osc_fetch_and_op; ompi_osc_base_module_get_accumulate_fn_t osc_get_accumulate; ompi_osc_base_module_rput_fn_t osc_rput; - ompi_osc_base_module_rput_notify_fn_t osc_rput_notify; ompi_osc_base_module_rget_fn_t osc_rget; - ompi_osc_base_module_rget_notify_fn_t osc_rget_notify; ompi_osc_base_module_raccumulate_fn_t osc_raccumulate; ompi_osc_base_module_rget_accumulate_fn_t osc_rget_accumulate; @@ -466,6 +459,13 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_flush_all_fn_t osc_flush_all; ompi_osc_base_module_flush_local_fn_t osc_flush_local; ompi_osc_base_module_flush_local_all_fn_t osc_flush_local_all; + ompi_osc_base_module_put_notify_fn_t osc_put_notify; + ompi_osc_base_module_get_notify_fn_t osc_get_notify; + ompi_osc_base_module_win_get_notify_value_fn_t osc_win_get_notify_value; + ompi_osc_base_module_win_set_notify_value_fn_t osc_win_set_notify_value; + ompi_osc_base_module_win_reset_notify_value_fn_t osc_win_reset_notify_value; + ompi_osc_base_module_rput_notify_fn_t osc_rput_notify; + ompi_osc_base_module_rget_notify_fn_t osc_rget_notify; }; typedef struct ompi_osc_base_module_4_0_0_t ompi_osc_base_module_4_0_0_t; typedef ompi_osc_base_module_4_0_0_t ompi_osc_base_module_t; diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index 40cbc9d2813..7d7501dcabf 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -41,7 +41,7 @@ ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, int rank = ompi_comm_rank(module->comm); if (notify < 0 || (uint32_t) notify >= module->node_states[rank].notify_counter_count) { - return OMPI_ERR_BAD_PARAM; + return MPI_ERR_NOTIFY_IDX; } *value = (OMPI_MPI_COUNT_TYPE) osc_sm_target_notify_base(module, rank)[notify]; @@ -59,7 +59,7 @@ ompi_osc_sm_win_set_notify_value(struct ompi_win_t *win, int rank = ompi_comm_rank(module->comm); if (notify < 0 || (uint32_t) notify >= module->node_states[rank].notify_counter_count) { - return OMPI_ERR_BAD_PARAM; + return MPI_ERR_NOTIFY_IDX; } osc_sm_target_notify_base(module, rank)[notify] = (uint64_t) value; @@ -77,7 +77,7 @@ ompi_osc_sm_win_reset_notify_value(struct ompi_win_t *win, int rank = ompi_comm_rank(module->comm); if (notify < 0 || (uint32_t) notify >= module->node_states[rank].notify_counter_count) { - return OMPI_ERR_BAD_PARAM; + return MPI_ERR_NOTIFY_IDX; } /* Atomically swap the counter to 0 and return the previous value */ @@ -165,7 +165,7 @@ ompi_osc_sm_rput_notify(const void *origin_addr, *ompi_req = &ompi_request_empty; if (notify < 0 || (uint32_t) notify >= module->node_states[target].notify_counter_count) { - return OMPI_ERR_BAD_PARAM; + return MPI_ERR_NOTIFY_IDX; } opal_atomic_wmb(); @@ -252,7 +252,7 @@ ompi_osc_sm_rget_notify(void *origin_addr, *ompi_req = &ompi_request_empty; if (notify < 0 || (uint32_t) notify >= module->node_states[target].notify_counter_count) { - return OMPI_ERR_BAD_PARAM; + return MPI_ERR_NOTIFY_IDX; } opal_atomic_rmb(); @@ -430,7 +430,7 @@ ompi_osc_sm_put_notify(const void *origin_addr, } if (notify < 0 || (uint32_t) notify >= module->node_states[target].notify_counter_count) { - return OMPI_ERR_BAD_PARAM; + return MPI_ERR_NOTIFY_IDX; } opal_atomic_wmb(); diff --git a/ompi/mca/osc/ubcl/osc_ubcl.c b/ompi/mca/osc/ubcl/osc_ubcl.c index d358605244b..5e81ed1add3 100644 --- a/ompi/mca/osc/ubcl/osc_ubcl.c +++ b/ompi/mca/osc/ubcl/osc_ubcl.c @@ -80,19 +80,14 @@ mca_osc_ubcl_module_t mca_osc_ubcl_module_template = { win_free, ompi_osc_ubcl_put, - NULL, ompi_osc_ubcl_get, - NULL, - NULL, ompi_osc_ubcl_accumulate, ompi_osc_ubcl_compare_and_swap, ompi_osc_ubcl_fetch_and_op, ompi_osc_ubcl_get_accumulate, ompi_osc_ubcl_rput, - NULL, ompi_osc_ubcl_rget, - NULL, ompi_osc_ubcl_raccumulate, ompi_osc_ubcl_rget_accumulate, From 93e9e82bf823a4bfef779e807b4bc4f03bba7524 Mon Sep 17 00:00:00 2001 From: Joseph Antony Date: Fri, 13 Mar 2026 17:23:08 -0400 Subject: [PATCH 15/15] Removing Duplicate changes Signed-off-by: Joseph Antony --- ompi/include/mpi.h.in | 2 -- ompi/mca/osc/osc.h | 5 ---- ompi/mca/osc/sm/osc_sm.h | 4 --- ompi/mca/osc/sm/osc_sm_comm.c | 18 ------------ ompi/mca/osc/sm/osc_sm_component.c | 1 - ompi/mpi/c/Makefile.am | 2 -- ompi/mpi/c/win_set_notify_value.c.in | 41 ---------------------------- 7 files changed, 73 deletions(-) delete mode 100644 ompi/mpi/c/win_set_notify_value.c.in diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in index 3a147a7a136..1422695ea37 100644 --- a/ompi/include/mpi.h.in +++ b/ompi/include/mpi.h.in @@ -2620,7 +2620,6 @@ OMPI_DECLSPEC int MPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int MPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int MPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); OMPI_DECLSPEC int MPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); -OMPI_DECLSPEC int MPI_Win_set_notify_value(MPI_Win win, int notification_idx, MPI_Count value); OMPI_DECLSPEC int MPI_Win_reset_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); OMPI_DECLSPEC int MPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_lock_all(int mpi_assert, MPI_Win win); @@ -3811,7 +3810,6 @@ OMPI_DECLSPEC int PMPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int PMPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int PMPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); OMPI_DECLSPEC int PMPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); -OMPI_DECLSPEC int PMPI_Win_set_notify_value(MPI_Win win, int notification_idx, MPI_Count value); OMPI_DECLSPEC int PMPI_Win_reset_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); OMPI_DECLSPEC int PMPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_lock_all(int mpi_assert, MPI_Win win); diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index 002866a69da..b43f34ac3c5 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -249,10 +249,6 @@ typedef int (*ompi_osc_base_module_win_get_notify_value_fn_t)(struct ompi_win_t int notify, OMPI_MPI_COUNT_TYPE *value); -typedef int (*ompi_osc_base_module_win_set_notify_value_fn_t)(struct ompi_win_t *win, - int notify, - OMPI_MPI_COUNT_TYPE value); - typedef int (*ompi_osc_base_module_win_reset_notify_value_fn_t)(struct ompi_win_t *win, int notify, OMPI_MPI_COUNT_TYPE *value); @@ -462,7 +458,6 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_put_notify_fn_t osc_put_notify; ompi_osc_base_module_get_notify_fn_t osc_get_notify; ompi_osc_base_module_win_get_notify_value_fn_t osc_win_get_notify_value; - ompi_osc_base_module_win_set_notify_value_fn_t osc_win_set_notify_value; ompi_osc_base_module_win_reset_notify_value_fn_t osc_win_reset_notify_value; ompi_osc_base_module_rput_notify_fn_t osc_rput_notify; ompi_osc_base_module_rget_notify_fn_t osc_rget_notify; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index ca776bd59a0..85d250bfa18 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -154,10 +154,6 @@ int ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, int notify, OMPI_MPI_COUNT_TYPE *value); -int ompi_osc_sm_win_set_notify_value(struct ompi_win_t *win, - int notify, - OMPI_MPI_COUNT_TYPE value); - int ompi_osc_sm_win_reset_notify_value(struct ompi_win_t *win, int notify, OMPI_MPI_COUNT_TYPE *value); diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index 7d7501dcabf..fbd4f17856c 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -50,24 +50,6 @@ ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, return OMPI_SUCCESS; } -int -ompi_osc_sm_win_set_notify_value(struct ompi_win_t *win, - int notify, - OMPI_MPI_COUNT_TYPE value) -{ - ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; - int rank = ompi_comm_rank(module->comm); - - if (notify < 0 || (uint32_t) notify >= module->node_states[rank].notify_counter_count) { - return MPI_ERR_NOTIFY_IDX; - } - - osc_sm_target_notify_base(module, rank)[notify] = (uint64_t) value; - opal_atomic_wmb(); - - return OMPI_SUCCESS; -} - int ompi_osc_sm_win_reset_notify_value(struct ompi_win_t *win, int notify, diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index 3ba892e0885..259c0826017 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -83,7 +83,6 @@ ompi_osc_sm_module_t ompi_osc_sm_module_template = { .osc_get = ompi_osc_sm_get, .osc_get_notify = ompi_osc_sm_get_notify, .osc_win_get_notify_value = ompi_osc_sm_win_get_notify_value, - .osc_win_set_notify_value = ompi_osc_sm_win_set_notify_value, .osc_win_reset_notify_value = ompi_osc_sm_win_reset_notify_value, .osc_accumulate = ompi_osc_sm_accumulate, .osc_compare_and_swap = ompi_osc_sm_compare_and_swap, diff --git a/ompi/mpi/c/Makefile.am b/ompi/mpi/c/Makefile.am index 096da3ab171..49619694d0b 100644 --- a/ompi/mpi/c/Makefile.am +++ b/ompi/mpi/c/Makefile.am @@ -487,7 +487,6 @@ prototype_sources = \ win_get_info.c.in \ win_get_name.c.in \ win_get_notify_value.c.in \ - win_set_notify_value.c.in \ win_reset_notify_value.c.in \ win_lock_all.c.in \ win_lock.c.in \ @@ -960,7 +959,6 @@ interface_profile_sources = \ win_get_info_generated.c \ win_get_name_generated.c \ win_get_notify_value_generated.c \ - win_set_notify_value_generated.c \ win_reset_notify_value_generated.c \ win_lock_all_generated.c \ win_lock_generated.c \ diff --git a/ompi/mpi/c/win_set_notify_value.c.in b/ompi/mpi/c/win_set_notify_value.c.in deleted file mode 100644 index 8a7d97567db..00000000000 --- a/ompi/mpi/c/win_set_notify_value.c.in +++ /dev/null @@ -1,41 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2026 Triad National Security, LLC. All rights - * reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ -#include "ompi_config.h" - -#include "ompi/mpi/c/bindings.h" -#include "ompi/runtime/params.h" -#include "ompi/errhandler/errhandler.h" -#include "ompi/win/win.h" -#include "ompi/mca/osc/osc.h" - -PROTOTYPE ERROR_CLASS win_set_notify_value(WIN win, INT notification_idx, PARTITIONED_COUNT value) -{ - int rc; - - if (MPI_PARAM_CHECK) { - rc = OMPI_SUCCESS; - - OMPI_ERR_INIT_FINALIZE(FUNC_NAME); - - if (ompi_win_invalid(win)) { - return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); - } else if (notification_idx < 0) { - rc = MPI_ERR_NOTIFY_IDX; - } else if (value < 0) { - rc = MPI_ERR_ARG; - } - - OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); - } - - rc = win->w_osc_module->osc_win_set_notify_value(win, notification_idx, value); - OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); -}