在 ctypes 加载的共享库中卸载共享库
Unload shared library inside ctypes loaded shared library
我正在从我的 python 脚本中调用一个 so 文件。据我了解，我真的不需要使用 ctypes 释放在 python 中打开的共享库。但是，在我的 so 文件代码中，它 dlopens 另一个 so 文件并且不执行 dlclose()。
在这种情况下，从 python 端使用是否安全？我不必释放在 ctypes 加载 so 文件中打开的共享库吗？
规则自己清理始终适用（尽管现代技术会为您解决清洁方面的问题）。
[Python 3.5]: ctypes - A foreign function library for Python包含很多有用的信息，应该成为你的朋友。
ctypes 在加载 .dll 时使用 dlopen。正如我所注意到的，它 doesn't 调用相应的 dlclose 意味着 .dll （和 加载它时加载的所有依赖项) 将保留在内存中，直到进程终止（或直到显式卸载）。
If the object specified by filename has dependencies on other shared objects, then these are also automatically loaded by the dynamic linker using the same rules. (This process may occur recursively, if those objects in turn have dependencies, and so on.)
...
If the same shared object is loaded again with dlopen(), the same object handle is returned. The dynamic linker maintains reference counts for object handles, so a dynamically loaded shared object is not deallocated until dlclose() has been called on it as many times as dlopen() has succeeded on it. Any initialization returns (see below) are called just once.
所以，我认为您不会遇到问题（当然，一切都取决于上下文）。正如您所注意到的，多次加载一个库实际上并不是每次都加载它，因此 运行 内存不足的机会非常小（除非您正在加载大量不同的 。 dlls，每个都有很多不同的依赖项）。
我能想到的一个例子是加载一个 .dll，它使用另一个 .dll 的符号。如果该符号也在之前加载的另一个 (3rd) .dll 中定义，那么代码的行为将与预期不同。
无论如何，您可以手动卸载（或更好：减少其 refcount）一个 .dll（我不确定这是怎么回事符合推荐方式或最佳实践），如下例所示。
dll.c:
#include <stdio.h>
int test() {
printf("[%s] (%d) - [%s]\n", __FILE__, __LINE__, __FUNCTION__);
return 0;
}
code.py:
import sys
from ctypes import CDLL, \
c_int, c_void_p
DLL = "./dll.so"
dlclose_func = CDLL(None).dlclose # This WON'T work on Win
dlclose_func.argtypes = [c_void_p]
def _load_dll(dll_name):
dll_dll = CDLL(dll_name)
print("{:}".format(dll_dll))
return dll_dll
def _load_test_func(dll):
test_func = dll.test
test_func.restype = c_int
return test_func
def main():
print("Loading a dll via `ctypes`, then delete the object. The dll is not unloaded. Call `dlclose` to unload. A 2nd call will fail.")
dll_dll = _load_dll(DLL)
dll_handle = dll_dll._handle
del dll_dll
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll_handle))) # Even if the ctypes dll object was destroyed, the dll wasn't unloaded
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll_handle))) # A new dlclose call will fail
print("\nUse `ctypes` to load the dll twice. The dll is not actually loaded only the 1st time (both have the same handle), but its ref count is increased. `dlclose` must be also called twice.")
dll0_dll = _load_dll(DLL)
dll1_dll = _load_dll(DLL)
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll0_dll._handle)))
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll1_dll._handle)))
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll1_dll._handle)))
print("\nLoad a dll via `ctypes`, and load one of its funcs. Try calling it before and after unloading the dll.")
dll_dll = _load_dll(DLL)
test_func = _load_test_func(dll_dll)
print("{:} returned {:d}".format(test_func.__name__, test_func()))
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll_dll._handle)))
print("{:} returned {:d}".format(test_func.__name__, test_func())) # Comment this line as it would segfault !!!
if __name__ == "__main__":
print("Python {:s} on {:s}\n".format(sys.version, sys.platform))
main()
输出:
[cfati@cfati-ubtu16x64-0:~/Work/Dev/Whosebug/q052179325]> ls
code.py dll.c
[cfati@cfati-ubtu16x64-0:~/Work/Dev/Whosebug/q052179325]> gcc -fPIC -shared -o dll.so dll.c
[cfati@cfati-ubtu16x64-0:~/Work/Dev/Whosebug/q052179325]> python3 ./code.py
Python 3.5.2 (default, Nov 23 2017, 16:37:01)
[GCC 5.4.0 20160609] on linux
Loading a dll via `ctypes`, then delete the object. The dll is not unloaded. Call `dlclose` to unload. A 2nd call will fail.
<CDLL './dll.so', handle 1d7aa20 at 0x7fa38715f240>
dlclose returned 0
dlclose returned -1
Use `ctypes` to load the dll twice. The dll is not actually loaded only the 1st time (both have the same handle), but its ref count is increased. `dlclose` must be also called twice.
<CDLL './dll.so', handle 1de2c80 at 0x7fa38715f240>
<CDLL './dll.so', handle 1de2c80 at 0x7fa38715f278>
dlclose returned 0
dlclose returned 0
dlclose returned -1
Load a dll via `ctypes`, and load one of its funcs. Try calling it before and after unloading the dll.
<CDLL './dll.so', handle 1de39c0 at 0x7fa38715f8d0>
[dll.c] (5) - [test]
test returned 0
dlclose returned 0
Segmentation fault (core dumped)
卸载依赖项的示例
在 Linux Fedora 32、Python 3.7.6 (anaconda)、ctypes 1.1.0、g++ 10.2.1 上测试。
依赖项是 OpenCv（版本 4.2）。
此处有更多详细信息：How can I unload a DLL using ctypes in Python?
code.cpp
#include <opencv2/core/core.hpp>
#include <iostream>
extern "C" int my_fct(int n)
{
cv::Mat1b mat = cv::Mat1b(10,8,(unsigned char) 1 ); // change 1 to test unloading
return mat(0,1) * n;
}
编译为
g++ code.cpp -shared -fPIC -Wall -std=c++17 -I/usr/include/opencv4 -lopencv_core -o so_opencv.so
Python代码
from sys import platform
import ctypes
class CtypesLib:
def __init__(self, fp_lib, dependencies=[]):
self._dependencies = [CtypesLib(fp_dep) for fp_dep in dependencies]
if platform == "linux" or platform == "linux2": # Linux
self._dlclose_func = ctypes.cdll.LoadLibrary('').dlclose
self._dlclose_func.argtypes = [ctypes.c_void_p]
self._ctypes_lib = ctypes.cdll.LoadLibrary(fp_lib)
elif platform == "win32": # Windows
self._ctypes_lib = ctypes.WinDLL(fp_lib)
self._handle = self._ctypes_lib._handle
def __getattr__(self, attr):
return self._ctypes_lib.__getattr__(attr)
def __del__(self):
for dep in self._dependencies:
del dep
del self._ctypes_lib
if platform == "linux" or platform == "linux2": # Linux
self._dlclose_func(self._handle)
elif platform == "win32": # Windows
ctypes.windll.kernel32.FreeLibrary(self._handle)
fp_lib = './so_opencv.so'
ctypes_lib = CtypesLib(fp_lib, ['/usr/lib64/libopencv_core.so'])
valIn = 1
ctypes_lib.my_fct.argtypes = [ctypes.c_int]
ctypes_lib.my_fct.restype = ctypes.c_int
valOut = ctypes_lib.my_fct(valIn)
print(valIn, valOut)
del ctypes_lib
我正在从我的 python 脚本中调用一个 so 文件。据我了解，我真的不需要使用 ctypes 释放在 python 中打开的共享库。但是，在我的 so 文件代码中，它 dlopens 另一个 so 文件并且不执行 dlclose()。 在这种情况下，从 python 端使用是否安全？我不必释放在 ctypes 加载 so 文件中打开的共享库吗？
规则自己清理始终适用（尽管现代技术会为您解决清洁方面的问题）。
[Python 3.5]: ctypes - A foreign function library for Python包含很多有用的信息，应该成为你的朋友。
ctypes 在加载 .dll 时使用 dlopen。正如我所注意到的，它 doesn't 调用相应的 dlclose 意味着 .dll （和 加载它时加载的所有依赖项) 将保留在内存中，直到进程终止（或直到显式卸载）。
If the object specified by filename has dependencies on other shared objects, then these are also automatically loaded by the dynamic linker using the same rules. (This process may occur recursively, if those objects in turn have dependencies, and so on.)
...
If the same shared object is loaded again with dlopen(), the same object handle is returned. The dynamic linker maintains reference counts for object handles, so a dynamically loaded shared object is not deallocated until dlclose() has been called on it as many times as dlopen() has succeeded on it. Any initialization returns (see below) are called just once.
所以，我认为您不会遇到问题（当然，一切都取决于上下文）。正如您所注意到的，多次加载一个库实际上并不是每次都加载它，因此 运行 内存不足的机会非常小（除非您正在加载大量不同的 。 dlls，每个都有很多不同的依赖项）。
我能想到的一个例子是加载一个 .dll，它使用另一个 .dll 的符号。如果该符号也在之前加载的另一个 (3rd) .dll 中定义，那么代码的行为将与预期不同。
无论如何，您可以手动卸载（或更好：减少其 refcount）一个 .dll（我不确定这是怎么回事符合推荐方式或最佳实践），如下例所示。
dll.c:
#include <stdio.h>
int test() {
printf("[%s] (%d) - [%s]\n", __FILE__, __LINE__, __FUNCTION__);
return 0;
}
code.py:
import sys
from ctypes import CDLL, \
c_int, c_void_p
DLL = "./dll.so"
dlclose_func = CDLL(None).dlclose # This WON'T work on Win
dlclose_func.argtypes = [c_void_p]
def _load_dll(dll_name):
dll_dll = CDLL(dll_name)
print("{:}".format(dll_dll))
return dll_dll
def _load_test_func(dll):
test_func = dll.test
test_func.restype = c_int
return test_func
def main():
print("Loading a dll via `ctypes`, then delete the object. The dll is not unloaded. Call `dlclose` to unload. A 2nd call will fail.")
dll_dll = _load_dll(DLL)
dll_handle = dll_dll._handle
del dll_dll
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll_handle))) # Even if the ctypes dll object was destroyed, the dll wasn't unloaded
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll_handle))) # A new dlclose call will fail
print("\nUse `ctypes` to load the dll twice. The dll is not actually loaded only the 1st time (both have the same handle), but its ref count is increased. `dlclose` must be also called twice.")
dll0_dll = _load_dll(DLL)
dll1_dll = _load_dll(DLL)
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll0_dll._handle)))
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll1_dll._handle)))
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll1_dll._handle)))
print("\nLoad a dll via `ctypes`, and load one of its funcs. Try calling it before and after unloading the dll.")
dll_dll = _load_dll(DLL)
test_func = _load_test_func(dll_dll)
print("{:} returned {:d}".format(test_func.__name__, test_func()))
print("{:} returned {:d}".format(dlclose_func.__name__, dlclose_func(dll_dll._handle)))
print("{:} returned {:d}".format(test_func.__name__, test_func())) # Comment this line as it would segfault !!!
if __name__ == "__main__":
print("Python {:s} on {:s}\n".format(sys.version, sys.platform))
main()
输出:
[cfati@cfati-ubtu16x64-0:~/Work/Dev/Whosebug/q052179325]> ls code.py dll.c [cfati@cfati-ubtu16x64-0:~/Work/Dev/Whosebug/q052179325]> gcc -fPIC -shared -o dll.so dll.c [cfati@cfati-ubtu16x64-0:~/Work/Dev/Whosebug/q052179325]> python3 ./code.py Python 3.5.2 (default, Nov 23 2017, 16:37:01) [GCC 5.4.0 20160609] on linux Loading a dll via `ctypes`, then delete the object. The dll is not unloaded. Call `dlclose` to unload. A 2nd call will fail. <CDLL './dll.so', handle 1d7aa20 at 0x7fa38715f240> dlclose returned 0 dlclose returned -1 Use `ctypes` to load the dll twice. The dll is not actually loaded only the 1st time (both have the same handle), but its ref count is increased. `dlclose` must be also called twice. <CDLL './dll.so', handle 1de2c80 at 0x7fa38715f240> <CDLL './dll.so', handle 1de2c80 at 0x7fa38715f278> dlclose returned 0 dlclose returned 0 dlclose returned -1 Load a dll via `ctypes`, and load one of its funcs. Try calling it before and after unloading the dll. <CDLL './dll.so', handle 1de39c0 at 0x7fa38715f8d0> [dll.c] (5) - [test] test returned 0 dlclose returned 0 Segmentation fault (core dumped)
卸载依赖项的示例
在 Linux Fedora 32、Python 3.7.6 (anaconda)、ctypes 1.1.0、g++ 10.2.1 上测试。
依赖项是 OpenCv（版本 4.2）。
此处有更多详细信息：How can I unload a DLL using ctypes in Python?
code.cpp
#include <opencv2/core/core.hpp>
#include <iostream>
extern "C" int my_fct(int n)
{
cv::Mat1b mat = cv::Mat1b(10,8,(unsigned char) 1 ); // change 1 to test unloading
return mat(0,1) * n;
}
编译为
g++ code.cpp -shared -fPIC -Wall -std=c++17 -I/usr/include/opencv4 -lopencv_core -o so_opencv.so
Python代码
from sys import platform
import ctypes
class CtypesLib:
def __init__(self, fp_lib, dependencies=[]):
self._dependencies = [CtypesLib(fp_dep) for fp_dep in dependencies]
if platform == "linux" or platform == "linux2": # Linux
self._dlclose_func = ctypes.cdll.LoadLibrary('').dlclose
self._dlclose_func.argtypes = [ctypes.c_void_p]
self._ctypes_lib = ctypes.cdll.LoadLibrary(fp_lib)
elif platform == "win32": # Windows
self._ctypes_lib = ctypes.WinDLL(fp_lib)
self._handle = self._ctypes_lib._handle
def __getattr__(self, attr):
return self._ctypes_lib.__getattr__(attr)
def __del__(self):
for dep in self._dependencies:
del dep
del self._ctypes_lib
if platform == "linux" or platform == "linux2": # Linux
self._dlclose_func(self._handle)
elif platform == "win32": # Windows
ctypes.windll.kernel32.FreeLibrary(self._handle)
fp_lib = './so_opencv.so'
ctypes_lib = CtypesLib(fp_lib, ['/usr/lib64/libopencv_core.so'])
valIn = 1
ctypes_lib.my_fct.argtypes = [ctypes.c_int]
ctypes_lib.my_fct.restype = ctypes.c_int
valOut = ctypes_lib.my_fct(valIn)
print(valIn, valOut)
del ctypes_lib