# Pybind11 Starter
这是一篇笔记,保存 pybind11 项目的起始代码,便于下次使用 pybind11 的时候参考。
# 前言
Python 外部调用(以 c++ 为主)的几个方法:
| 功能 | Boost.Python | pybind11 | Cython | SWIG |
|---|---|---|---|---|
| 核心思想 | 通过宏定义和元编程来简化 Python 的 API 调用 | 受 Boost.Python 启发的 Header-only 的易用 Python 接口 | 基于 Python 的 C/C++ 代码封装器,语法是 Python 的超集 | 将 C++ 代码封装成多种语言的框架 |
| 优点 | 1. 兼容旧特性的 C++ 和 Boost 自定义类型 2. 对 Numpy 的支持完备 |
1. Header-only,安装方便 2. 社区活跃,文档齐全 3. 易用性好,上手快 |
1. 基于 Python,Python 开发者上手快 2. 环境配置简单 3. 支持模板(阉割版) 4. 封装后函数调用性能好 |
能够将 C++ 代码封装成多种语言(Python/C#/Java/Ruby 等) |
| 缺点 | 1. 需要先编译安装才能使用 2. 社区更新周期长 |
只支持标准 C 库和 C11 以上特性 | 1. 封装 C 类、重载函数、继承类比 pybind11 繁杂 2. 无法利用 C 的宏定义 |
对 C++ 的高级特性支持较差 |
| 适用场景 | 封装基于 Boost 的库,或需要深度操作 Numpy | 推荐在大多数情况下优先考虑 | 1. 需要保留模板参数 2. 有大量的封装函数调用 3. 绑定的对象是 C 语言 API |
有多语言绑定需求 |
# 环境准备
- PyCharm
- Clion
# C++ 项目构建
# CMake 项目初始化
需要下载 pybind11 库并引入到项目(这里使用 FetchContent )。使用 pybind11_add_module 添加一个要导出的 python 模块,作为一个 CMake 目标。这个函数会自动为该目标引入 pybind11 头文件、添加编译和链接选项。
为了方便测试,新建一个 CMake 目标,只引入 c++ 库的源代码和测试类的入口 test.cpp 。
覆盖率配置可以忽略。
cmake_minimum_required(VERSION 3.28) | |
project(demo_woa_cpp) | |
set(CMAKE_CXX_STANDARD 20) | |
set(BUILD_SHARED_LIBS OFF) | |
set(FETCHCONTENT_UPDATES_DISCONNECTED ON) | |
include(FetchContent) | |
# Download pybind11 | |
FetchContent_Declare( | |
pybind11 | |
GIT_REPOSITORY https://github.com/pybind/pybind11 | |
GIT_SHALLOW TRUE | |
GIT_TAG v2.13.6 | |
) | |
FetchContent_MakeAvailable(pybind11) | |
# Coverage configuration | |
add_library(coverage_config INTERFACE) | |
target_compile_options(coverage_config INTERFACE -fprofile-arcs -ftest-coverage) | |
target_link_libraries(coverage_config INTERFACE gcov) | |
target_link_options(coverage_config INTERFACE --coverage) | |
# pybind11 module | |
pybind11_add_module(ext_obj_func | |
export.cpp # define python bindings | |
# main.cpp ... | |
) | |
target_compile_options(ext_obj_func PUBLIC -O2) | |
target_link_options(ext_obj_func PRIVATE -static-libgcc -static-libstdc++ -Wl,-static) | |
# for testing | |
add_executable(module_test | |
test.cpp | |
# main.cpp ... | |
) | |
target_compile_options(module_test PUBLIC -O2) | |
target_link_libraries(module_test coverage_config) |
# 类的导出
export.h : 声明导出类。例如 ObjectiveFunction 是 c++ 下的类,其接口定义可能不方便 python 传参,可以在 export.h 中新建 ObjectiveFunctionExport 类继承 ObjectiveFunction ,在其中专门声明要导出的函数。
#ifndef EXPORT_H | |
#define EXPORT_H | |
#include <pybind11/numpy.h> | |
#include <pybind11/stl.h> | |
#include "ObjectiveFunction.h" | |
namespace py = pybind11; | |
/** | |
* `ObjectiveFunction` exported for pybind11 | |
*/ | |
class ObjectiveFunctionExport : public ObjectiveFunction | |
{ | |
// ObjectiveFunctionExport(const string& str_config, int anything); | |
public: | |
ObjectiveFunctionExport(); | |
double get_value(const py::array_t<int>& solution); | |
//py::array_t 是 python 中的 numpy 数组类型 | |
py::array_t<double> get_value_list(const py::array_t<int>& solution); | |
// friend auto __getstate__(const ObjectiveFunctionExport& p); | |
// friend auto __setstate__(const py::tuple& t); | |
}; | |
#endif //EXPORT_H |
exeport.cpp : 对 export.h 中函数的实现,同时使用 pybind11 宏为 python 绑定函数签名。
#include "export.h" | |
// ObjectiveFunctionExport::ObjectiveFunctionExport(const string& str_config, int anything) | |
// : ObjectiveFunction(str_config, 0) | |
// { | |
// } | |
ObjectiveFunctionExport::ObjectiveFunctionExport() | |
: ObjectiveFunction("") | |
{ | |
} | |
double ObjectiveFunctionExport::get_value(const py::array_t<int>& solution) | |
{ | |
const auto i_solution = solution.request(); | |
const auto ptr_solution = static_cast<int*>(i_solution.ptr); | |
return ObjectiveFunction::get_value(ptr_solution); | |
} | |
// auto __getstate__(const ObjectiveFunctionExport& p) | |
// { | |
// string str_yaml_config = YAML::Dump(p.config); | |
// return py::make_tuple(str_yaml_config); | |
// } | |
// auto __setstate__(const py::tuple& t) | |
// { | |
// const string str_yaml_config = t[0].cast<string>(); | |
// return ObjectiveFunctionExport(str_yaml_config, 0); | |
// } | |
py::array_t<double> ObjectiveFunctionExport::get_value_list(const py::array_t<int>& solution) | |
{ | |
// 读 numpy 数组 | |
const auto i_solution = solution.request(); | |
const auto ptr_solution = static_cast<int*>(i_solution.ptr); | |
calc_values(ptr_solution); | |
// 写 numpy 数组 | |
py::array_t<double> result; | |
result.resize({static_cast<long>(objectives.size() + constrains.size())}); | |
auto r = result.mutable_unchecked<1>(); | |
// 使用 r (i) 访问 result | |
for (int i = 0; i < objectives.size(); ++i) | |
r(i) = obj_values[i]; | |
for (int i = 0; i < constrains.size(); ++i) | |
r(i + objectives.size()) = con_values[i]; | |
return result; | |
} | |
PYBIND11_MODULE(ext_obj_func, m) | |
{ | |
py::class_<ObjectiveFunctionExport>(m, "ObjectiveFunction") | |
.def(py::init<>()) | |
.def("get_value", &ObjectiveFunctionExport::get_value) | |
.def("get_value_list", &ObjectiveFunctionExport::get_value_list) | |
.def("get_num_tasks", &ObjectiveFunctionExport::get_num_tasks) | |
.def("get_num_resources", &ObjectiveFunctionExport::get_num_resources); | |
// .def(py::pickle(&__getstate__, &__setstate__)); | |
} |
# 安装到 Python 环境
构建完 c++ 项目,绑定了函数接口,现在需要编译包并安装到 python 解释器环境中。
setup.py : 使用 setuptools 配置包的生成信息,定义 CMake 扩展即可自动构建 CMake 目标。
这里参考 pybind11 的官方 CMake 样例。
import os | |
import re | |
import subprocess | |
import sys | |
from pathlib import Path | |
from setuptools import Extension, setup | |
from setuptools.command.build_ext import build_ext | |
# A CMakeExtension needs a sourcedir instead of a file list. | |
# The name must be the _single_ output extension from the CMake build. | |
# If you need multiple extensions, see scikit-build. | |
class CMakeExtension(Extension): | |
def __init__(self, name: str, sourcedir: str = "") -> None: | |
super().__init__(name, sources=[]) | |
self.sourcedir = os.fspath(Path(sourcedir).resolve()) | |
class CMakeBuild(build_ext): | |
def build_extension(self, ext: CMakeExtension) -> None: | |
# Must be in this form due to bug in .resolve() only fixed in Python 3.10+ | |
ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) | |
extdir = ext_fullpath.parent.resolve() | |
# Using this requires trailing slash for auto-detection & inclusion of | |
# auxiliary "native" libs | |
debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug | |
cfg = "Debug" if debug else "Release" | |
# Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON | |
cmake_args = [ | |
"-DCMAKE_CXX_STANDARD=20", | |
f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}", | |
f"-DPYTHON_EXECUTABLE={sys.executable}", | |
f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm | |
] | |
build_args = [] | |
# Adding CMake arguments set as environment variable | |
# (needed e.g. to build for ARM OSx on conda-forge) | |
if "CMAKE_ARGS" in os.environ: | |
cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item] | |
# Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level | |
# across all generators. | |
if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: | |
# self.parallel is a Python 3 only way to set parallel jobs by hand | |
# using -j in the build_ext call, not supported by pip or PyPA-build. | |
if hasattr(self, "parallel") and self.parallel: | |
# CMake 3.12+ only. | |
build_args += [f"-j{self.parallel}"] | |
build_temp = Path(self.build_temp) / ext.name | |
if not build_temp.exists(): | |
build_temp.mkdir(parents=True) | |
print(["cmake", ext.sourcedir, *cmake_args]) | |
subprocess.run( | |
["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True | |
) | |
print(["cmake", "--build", ".", *build_args]) | |
subprocess.run( | |
["cmake", "--build", ".", *build_args], cwd=build_temp, check=True | |
) | |
# The information here can also be placed in setup.cfg - better separation of | |
# logic and declaration, and simpler if you include description/version in a file. | |
setup( | |
name="ext_obj_func", | |
version="0.3.0", | |
author="DuanYue", | |
author_email="imduanyue@qq.com", | |
description="Objective function for a scheduling problem", | |
long_description="", | |
ext_modules=[CMakeExtension("ext_obj_func")], | |
cmdclass={"build_ext": CMakeBuild}, | |
zip_safe=False, | |
) |
之后,使用 pip 即可将我们的包安装到 python 环境。
#在 setup.py 的父目录下 | |
pip install . |
# 生成 stub 文件
至此,我们已经可以在 python 中使用 import 导入自定义包,并成功运行。但是 IDE 的代码洞察还不知道自定义模块的信息(函数签名)。
使用 pybind11-stubgen 自动生成 stub 文件,获得自定义包的 .pyi 文件,将其放置在解释器的 site-packages 文件夹下,IDE 就可以识别、自动补全自定义包的函数签名了。
# Install | |
pip install pybind11-stubgen | |
# Generate stubs for numpy | |
pybind11-stubgen ext_obj_func |