From c87517843b1edfa56125376b0ffba2b79fa5f16f Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sun, 28 Oct 2018 00:07:54 -0700 Subject: [PATCH 1/4] Avoid segfault during exit process closes https://github.com/JuliaPy/pyjulia/issues/208 --- src/PyCall.jl | 6 ++++-- src/pyinit.jl | 36 ++++++++++++++++++++++++++++++++++++ test/runtests.jl | 14 ++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/PyCall.jl b/src/PyCall.jl index 8e5ccc46..2031c220 100644 --- a/src/PyCall.jl +++ b/src/PyCall.jl @@ -98,9 +98,11 @@ it is equivalent to a `PyNULL()` object. """ ispynull(o::PyObject) = o.o == PyPtr_NULL -function pydecref_(o::Union{PyPtr,PyObject}) +function pydecref_(o::Union{PyPtr,PyObject}) :: Nothing + if _finalized[] + return + end ccall(@pysym(:Py_DecRef), Cvoid, (PyPtr,), o) - return o end function pydecref(o::PyObject) diff --git a/src/pyinit.jl b/src/pyinit.jl index 7515b12c..525679a7 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -154,4 +154,40 @@ function __init__() end end end + + # Configure finalization steps. + # + # * In julia/PyCall, `julia` needs to call `Py_Finalize` to + # finalize Python runtime to invoke Python functions registered + # in Python's exit hook. This is done by Julia's `atexit` exit + # hook. + # + # * In PyJulia, `python` needs to call `jl_atexit_hook` in its + # exit hook instead. + # + # In both cases, it is important to not invoke GC of the finalized + # runtime. This is ensured by: + @pycheckz ccall((@pysym :Py_AtExit), Cint, (Ptr{Cvoid},), + @cfunction($_set_finalized, Cvoid, ())) + if !already_inited + # Once `_set_finalized` is successfully registered to + # `Py_AtExit`, it is safe to call `Py_Finalize` during + # finalization of this Julia process. + atexit(Py_Finalize) + end +end + +const _finalized = Ref(false) +# This flag is set via `Py_AtExit` to avoid calling `pydecref_` after +# Python is finalized. + +function _set_finalized() + # This function MUST NOT invoke any Python APIs. + # https://docs.python.org/3/c-api/sys.html#c.Py_AtExit + _finalized[] = true + return nothing +end + +function Py_Finalize() + ccall(@pysym(:Py_Finalize), Cvoid, ()) end diff --git a/test/runtests.jl b/test/runtests.jl index a1951214..971d82c7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -666,4 +666,18 @@ def try_call(f): pybuiltin("Exception")) end +@testset "atexit" begin + script = """ + $(Base.load_path_setup_code()) + + using PyCall + + pyimport("atexit")[:register]() do + println("atexit called") + end + """ + out = read(`$(Base.julia_cmd()) -e $script`, String) + @test occursin("atexit called", out) +end + include("test_pyfncall.jl") From adf1a4f42ed02acd028961d09a7341f157fcd402 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sun, 28 Oct 2018 15:30:53 -0700 Subject: [PATCH 2/4] Support Julia 0.6 --- src/pyinit.jl | 32 ++++++++++++++++---------------- test/runtests.jl | 7 ++++++- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/pyinit.jl b/src/pyinit.jl index 525679a7..d3ec8409 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -62,6 +62,21 @@ end ######################################################################### +const _finalized = Ref(false) +# This flag is set via `Py_AtExit` to avoid calling `pydecref_` after +# Python is finalized. + +function _set_finalized() + # This function MUST NOT invoke any Python APIs. + # https://docs.python.org/3/c-api/sys.html#c.Py_AtExit + _finalized[] = true + return nothing +end + +function Py_Finalize() + ccall(@pysym(:Py_Finalize), Cvoid, ()) +end + function __init__() # sanity check: in Pkg for Julia 0.7+, the location of Conda can change # if e.g. you checkout Conda master, and we'll need to re-build PyCall @@ -168,7 +183,7 @@ function __init__() # In both cases, it is important to not invoke GC of the finalized # runtime. This is ensured by: @pycheckz ccall((@pysym :Py_AtExit), Cint, (Ptr{Cvoid},), - @cfunction($_set_finalized, Cvoid, ())) + @cfunction(_set_finalized, Cvoid, ())) if !already_inited # Once `_set_finalized` is successfully registered to # `Py_AtExit`, it is safe to call `Py_Finalize` during @@ -176,18 +191,3 @@ function __init__() atexit(Py_Finalize) end end - -const _finalized = Ref(false) -# This flag is set via `Py_AtExit` to avoid calling `pydecref_` after -# Python is finalized. - -function _set_finalized() - # This function MUST NOT invoke any Python APIs. - # https://docs.python.org/3/c-api/sys.html#c.Py_AtExit - _finalized[] = true - return nothing -end - -function Py_Finalize() - ccall(@pysym(:Py_Finalize), Cvoid, ()) -end diff --git a/test/runtests.jl b/test/runtests.jl index 971d82c7..a9d34404 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -667,8 +667,13 @@ def try_call(f): end @testset "atexit" begin + if VERSION < v"0.7-" + setup = "" + else + setup = Base.load_path_setup_code() + end script = """ - $(Base.load_path_setup_code()) + $setup using PyCall From f6d9da1d2bdee6ef0999bf0cddb75e7573346e6f Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Mon, 29 Oct 2018 12:49:23 -0700 Subject: [PATCH 3/4] Revert back pydecref_ return type --- src/PyCall.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/PyCall.jl b/src/PyCall.jl index 2031c220..2dec1f30 100644 --- a/src/PyCall.jl +++ b/src/PyCall.jl @@ -98,11 +98,12 @@ it is equivalent to a `PyNULL()` object. """ ispynull(o::PyObject) = o.o == PyPtr_NULL -function pydecref_(o::Union{PyPtr,PyObject}) :: Nothing +function pydecref_(o::Union{PyPtr,PyObject}) if _finalized[] - return + return o end ccall(@pysym(:Py_DecRef), Cvoid, (PyPtr,), o) + return o end function pydecref(o::PyObject) From a52ab44b3d09d84a51b74704bb67c0d4c91e49f9 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Sat, 3 Nov 2018 20:21:42 -0400 Subject: [PATCH 4/4] shorten code slightly --- src/PyCall.jl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/PyCall.jl b/src/PyCall.jl index 2dec1f30..46ee4643 100644 --- a/src/PyCall.jl +++ b/src/PyCall.jl @@ -99,10 +99,7 @@ it is equivalent to a `PyNULL()` object. ispynull(o::PyObject) = o.o == PyPtr_NULL function pydecref_(o::Union{PyPtr,PyObject}) - if _finalized[] - return o - end - ccall(@pysym(:Py_DecRef), Cvoid, (PyPtr,), o) + _finalized[] || ccall(@pysym(:Py_DecRef), Cvoid, (PyPtr,), o) return o end