I'm trying to expose a C++ interface and callback system to Python using pybind11. In C++, everything works as expected: the callback is triggered in real time from a background thread. In Python, however, the callback is not triggered during execution. Only at the end, after the main program finishes, all the outputs appear at once. I tried to figure out how I might need to manage GIL acquireing but from what I found this should automatically be done by the trampoline. Still tried to manaully acquire the GIL with no success. Is there something that I am obviously missing? Is Python/Pybind able to process those kinds of asynchronous function calls?
Hopefully the following minimal example makes clear what I am trying to achieve.
Native C++ Code:
// Interface definition
class IMessageHandler {
public:
virtual ~IMessageHandler() = default;
virtual void onMessage(const std::string& msg) = 0;
};
// Class that stores an IMessageHandler and invokes it via callback
class MessageProcessor {
public:
MessageProcessor(IMessageHandler* handler)
: m_handler(handler), m_running(false) {}
// Usually this would be called from another thread
void simulateIncomingMessage(const std::string& msg) {
std::lock_guard<std::mutex> lock(m_mutex);
m_queue.push_back(msg);
}
// Start a thread that polls for new messages and calls onMessage
void startMessenger() {
m_running = true;
m_thread = std::thread([this]() {
while (m_running) {
std::vector<std::string> local_msgs;
{
std::lock_guard<std::mutex> lock(m_mutex);
local_msgs.swap(m_queue);
}
for (const auto& msg : local_msgs) {
if (m_handler) {
m_handler->onMessage(msg);
}
}
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
});
}
void stopMessenger() {
m_running = false;
if (m_thread.joinable())
m_thread.join();
}
~MessageProcessor() {
stopMessenger();
}
private:
IMessageHandler* m_handler;
std::thread m_thread;
std::atomic<bool> m_running;
std::vector<std::string> m_queue;
std::mutex m_mutex;
};
Pybind11:
// Pybind11 trampoline for IMessageHandler
class PyIMessageHandler : public IMessageHandler {
public:
using IMessageHandler::IMessageHandler;
void onMessage(const std::string& msg) override {
// py::gil_scoped_acquire gil; //This has no effect
PYBIND11_OVERLOAD_PURE(
void,
IMessageHandler,
onMessage,
msg
);
}
};
PYBIND11_MODULE(message_module, m) {
py::class_<IMessageHandler, PyIMessageHandler>(m, "IMessageHandler")
.def(py::init<>())
.def("onMessage", &IMessageHandler::onMessage);
py::class_<MessageProcessor>(m, "MessageProcessor")
.def(py::init<IMessageHandler*>())
.def("startMessenger", &MessageProcessor::startMessenger)
.def("stopMessenger", &MessageProcessor::stopMessenger);
}
Python:
import time
from message_module import IMessageHandler, MessageProcessor
class MyHandler(IMessageHandler):
def onMessage(self, msg):
print("Python received:", msg)
handler = MyHandler()
processor = MessageProcessor(handler)
processor.startMessenger()
time.sleep(2)
processor.stopMessenger()