diff options
Diffstat (limited to 'src/corelib')
| -rw-r--r-- | src/corelib/CMakeLists.txt | 14 | ||||
| -rw-r--r-- | src/corelib/configure.cmake | 22 | ||||
| -rw-r--r-- | src/corelib/io/qiooperation_p_p.h | 4 | ||||
| -rw-r--r-- | src/corelib/io/qioring.cpp | 79 | ||||
| -rw-r--r-- | src/corelib/io/qioring_linux.cpp | 743 | ||||
| -rw-r--r-- | src/corelib/io/qioring_p.h | 484 | ||||
| -rw-r--r-- | src/corelib/io/qrandomaccessasyncfile_p_p.h | 14 | ||||
| -rw-r--r-- | src/corelib/io/qrandomaccessasyncfile_qioring.cpp | 438 | ||||
| -rw-r--r-- | src/corelib/tools/qlist.h | 23 |
9 files changed, 1816 insertions, 5 deletions
diff --git a/src/corelib/CMakeLists.txt b/src/corelib/CMakeLists.txt index 32b70a1f288..e12d824cebb 100644 --- a/src/corelib/CMakeLists.txt +++ b/src/corelib/CMakeLists.txt @@ -583,6 +583,13 @@ if(QT_FEATURE_async_io) SOURCES io/qrandomaccessasyncfile_darwin.mm ) + elseif(LINUX AND QT_FEATURE_liburing) + qt_internal_extend_target(Core + SOURCES + io/qrandomaccessasyncfile_qioring.cpp + DEFINES + QT_RANDOMACCESSASYNCFILE_QIORING + ) elseif(QT_FEATURE_thread AND QT_FEATURE_future) # TODO: This should become the last (fallback) condition later. # We migth also want to rewrite it so that it does not depend on @@ -749,6 +756,13 @@ qt_internal_extend_target(Core CONDITION INTEGRITY --pending_instantiations=128 ) +qt_internal_extend_target(Core CONDITION QT_FEATURE_liburing + SOURCES + io/qioring.cpp io/qioring_linux.cpp io/qioring_p.h + LIBRARIES + uring +) + # Workaround for QTBUG-101411 # Remove if QCC (gcc version 8.3.0) for QNX 7.1.0 is no longer supported qt_internal_extend_target(Core CONDITION QCC AND (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL "8.3.0") diff --git a/src/corelib/configure.cmake b/src/corelib/configure.cmake index 08908082991..c1d15c75054 100644 --- a/src/corelib/configure.cmake +++ b/src/corelib/configure.cmake @@ -43,6 +43,8 @@ qt_find_package(JeMalloc MODULE PROVIDED_TARGETS PkgConfig::JeMalloc MODULE_NAME core QMAKE_LIB jemalloc) qt_find_package(Libsystemd MODULE PROVIDED_TARGETS PkgConfig::Libsystemd MODULE_NAME core QMAKE_LIB journald) +qt_find_package(Liburing MODULE + PROVIDED_TARGETS PkgConfig::Liburing MODULE_NAME global QMAKE_LIB liburing) qt_find_package(WrapAtomic MODULE PROVIDED_TARGETS WrapAtomic::WrapAtomic MODULE_NAME core QMAKE_LIB libatomic) qt_find_package(Libb2 MODULE PROVIDED_TARGETS Libb2::Libb2 MODULE_NAME core QMAKE_LIB libb2) @@ -402,6 +404,20 @@ int main(void) } ") +# liburing +qt_config_compile_test(liburing + LABEL "liburing" + LIBRARIES uring + CODE +"#include <liburing.h> + +int main(void) +{ + io_uring_enter(0, 0, 0, 0, nullptr); + return 0; +} +") + # linkat qt_config_compile_test(linkat LABEL "linkat()" @@ -809,6 +825,11 @@ qt_feature("linkat" PRIVATE AUTODETECT ( LINUX AND NOT ANDROID ) OR HURD CONDITION TEST_linkat ) +qt_feature("liburing" PRIVATE + LABEL "liburing" + AUTODETECT LINUX + CONDITION Liburing_FOUND +) qt_feature("std-atomic64" PUBLIC LABEL "64 bit atomic operations" CONDITION WrapAtomic_FOUND @@ -1250,6 +1271,7 @@ qt_configure_add_summary_entry(ARGS "forkfd_pidfd" CONDITION LINUX) qt_configure_add_summary_entry(ARGS "glib") qt_configure_add_summary_entry(ARGS "icu") qt_configure_add_summary_entry(ARGS "jemalloc") +qt_configure_add_summary_entry(ARGS "liburing") qt_configure_add_summary_entry(ARGS "timezone_tzdb") qt_configure_add_summary_entry(ARGS "system-libb2") qt_configure_add_summary_entry(ARGS "mimetype-database") diff --git a/src/corelib/io/qiooperation_p_p.h b/src/corelib/io/qiooperation_p_p.h index 470e0858fd3..be780d4c785 100644 --- a/src/corelib/io/qiooperation_p_p.h +++ b/src/corelib/io/qiooperation_p_p.h @@ -24,6 +24,10 @@ #include <QtCore/qspan.h> #include <QtCore/qvarlengtharray.h> +#ifdef QT_RANDOMACCESSASYNCFILE_QIORING +#include <QtCore/private/qioring_p.h> +#endif + #include <variant> QT_BEGIN_NAMESPACE diff --git a/src/corelib/io/qioring.cpp b/src/corelib/io/qioring.cpp new file mode 100644 index 00000000000..28849b49b04 --- /dev/null +++ b/src/corelib/io/qioring.cpp @@ -0,0 +1,79 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#include "qioring_p.h" + +QT_BEGIN_NAMESPACE + +Q_LOGGING_CATEGORY(lcQIORing, "qt.core.ioring", QtCriticalMsg) + +auto QIORing::queueRequestInternal(GenericRequestType &request) -> QueuedRequestStatus +{ + if (!ensureInitialized() || preparingRequests) { // preparingRequests protects against recursing + // inside callbacks of synchronous completions. + finishRequestWithError(request, QFileDevice::ResourceError); + addrItMap.remove(&request); + return QueuedRequestStatus::CompletedImmediately; + } + if (!lastUnqueuedIterator) + lastUnqueuedIterator.emplace(addrItMap[&request]); + + qCDebug(lcQIORing) << "Trying to submit request" << request.operation() + << "user data:" << std::addressof(request); + prepareRequests(); + // If this is now true we have, in some way, fulfilled the request: + const bool requestCompleted = !addrItMap.contains(&request); + const QueuedRequestStatus requestQueuedState = requestCompleted + ? QueuedRequestStatus::CompletedImmediately + : QueuedRequestStatus::Pending; + // We want to avoid notifying the kernel too often of tasks, so only do it if the queue is full, + // otherwise do it when we return to the event loop. + if (unstagedRequests == sqEntries && inFlightRequests <= cqEntries) { + submitRequests(); + return requestQueuedState; + } + if (stagePending || unstagedRequests == 0) + return requestQueuedState; + stagePending = true; + // We are not a QObject, but we always have the notifier, so use that for context: + QMetaObject::invokeMethod( + std::addressof(*notifier), [this] { submitRequests(); }, Qt::QueuedConnection); + return requestQueuedState; +} + +bool QIORing::waitForRequest(RequestHandle handle, QDeadlineTimer deadline) +{ + if (!handle || !addrItMap.contains(handle)) + return true; // : It was never there to begin with (so it is finished) + if (unstagedRequests) + submitRequests(); + completionReady(); // Try to process some pending completions + while (!deadline.hasExpired() && addrItMap.contains(handle)) { + if (!waitForCompletions(deadline)) + return false; + completionReady(); + } + return !addrItMap.contains(handle); +} + +namespace QtPrivate { +template <typename T> +using DetectResult = decltype(std::declval<const T &>().result); + +template <typename T> +constexpr bool HasResultMember = qxp::is_detected_v<DetectResult, T>; +} + +void QIORing::finishRequestWithError(QIORing::GenericRequestType &req, QFileDevice::FileError error) +{ + invokeOnOp(req, [error](auto *req) { + if constexpr (QtPrivate::HasResultMember<decltype(*req)>) + req->result.template emplace<QFileDevice::FileError>(error); + invokeCallback(*req); + }); +} + +QT_END_NAMESPACE + +#include "moc_qioring_p.cpp" diff --git a/src/corelib/io/qioring_linux.cpp b/src/corelib/io/qioring_linux.cpp new file mode 100644 index 00000000000..b296b916c81 --- /dev/null +++ b/src/corelib/io/qioring_linux.cpp @@ -0,0 +1,743 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#include "qioring_p.h" + +QT_REQUIRE_CONFIG(liburing); + +#include <QtCore/qobject.h> +#include <QtCore/qscopedvaluerollback.h> +#include <QtCore/private/qcore_unix_p.h> +#include <QtCore/private/qfiledevice_p.h> + +#include <liburing.h> +#include <sys/mman.h> +#include <sys/eventfd.h> +#include <sys/stat.h> + +QT_BEGIN_NAMESPACE +// From man write.2: +// On Linux, write() (and similar system calls) will transfer at most 0x7ffff000 (2,147,479,552) +// bytes, returning the number of bytes actually transferred. (This is true on both 32-bit and +// 64-bit systems.) +constexpr qsizetype MaxReadWriteLen = 0x7ffff000; // aka. MAX_RW_COUNT + +// We pretend that iovec and QSpans are the same, assert that size and alignment match: +static_assert(sizeof(iovec) + == sizeof(decltype(std::declval<QIORingRequest<QIORing::Operation::VectoredRead>>() + .destinations))); +static_assert(alignof(iovec) + == alignof(decltype(std::declval<QIORingRequest<QIORing::Operation::VectoredRead>>() + .destinations))); + +static io_uring_op toUringOp(QIORing::Operation op); +static void prepareFileReadWrite(io_uring_sqe *sqe, const QIORingRequestOffsetFdBase &request, + const void *address, qsizetype size); + + +QIORing *QIORing::sharedInstance() +{ + thread_local QIORing instance; + if (!instance.initializeIORing()) + return nullptr; + return &instance; +} + +QIORing::QIORing(quint32 submissionQueueSize, quint32 completionQueueSize) + : sqEntries(submissionQueueSize), cqEntries(completionQueueSize) +{ +} +QIORing::~QIORing() +{ + if (eventDescriptor != -1) + close(eventDescriptor); + if (io_uringFd != -1) + close(io_uringFd); +} + +bool QIORing::initializeIORing() +{ + if (io_uringFd != -1) + return true; + + io_uring_params params{}; + params.flags = IORING_SETUP_CQSIZE; + params.cq_entries = cqEntries; + const int fd = io_uring_setup(sqEntries, ¶ms); + if (fd < 0) { + qErrnoWarning(-fd, "Failed to setup io_uring"); + return false; + } + io_uringFd = fd; + size_t submissionQueueSize = params.sq_off.array + (params.sq_entries * sizeof(quint32)); + size_t completionQueueSize = params.cq_off.cqes + (params.cq_entries * sizeof(io_uring_cqe)); + if (params.features & IORING_FEAT_SINGLE_MMAP) + submissionQueueSize = std::max(submissionQueueSize, completionQueueSize); + submissionQueue = mmap(nullptr, submissionQueueSize, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, io_uringFd, IORING_OFF_SQ_RING); + if (submissionQueue == MAP_FAILED) { + qErrnoWarning(errno, "Failed to mmap io_uring submission queue"); + close(io_uringFd); + io_uringFd = -1; + return false; + } + const size_t submissionQueueEntriesSize = params.sq_entries * sizeof(io_uring_sqe); + submissionQueueEntries = static_cast<io_uring_sqe *>( + mmap(nullptr, submissionQueueEntriesSize, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, io_uringFd, IORING_OFF_SQES)); + if (submissionQueueEntries == MAP_FAILED) { + qErrnoWarning(errno, "Failed to mmap io_uring submission queue entries"); + munmap(submissionQueue, submissionQueueSize); + close(io_uringFd); + io_uringFd = -1; + return false; + } + void *completionQueue = nullptr; + if (params.features & IORING_FEAT_SINGLE_MMAP) { + completionQueue = submissionQueue; + } else { + completionQueue = mmap(nullptr, completionQueueSize, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, io_uringFd, IORING_OFF_CQ_RING); + if (completionQueue == MAP_FAILED) { + qErrnoWarning(errno, "Failed to mmap io_uring completion queue"); + munmap(submissionQueue, submissionQueueSize); + munmap(submissionQueueEntries, submissionQueueEntriesSize); + close(io_uringFd); + io_uringFd = -1; + return false; + } + } + sqEntries = params.sq_entries; + cqEntries = params.cq_entries; + + char *sq = static_cast<char *>(submissionQueue); + sqHead = reinterpret_cast<quint32 *>(sq + params.sq_off.head); + sqTail = reinterpret_cast<quint32 *>(sq + params.sq_off.tail); + sqIndexMask = reinterpret_cast<quint32 *>(sq + params.sq_off.ring_mask); + sqIndexArray = reinterpret_cast<quint32 *>(sq + params.sq_off.array); + + char *cq = static_cast<char *>(completionQueue); + cqHead = reinterpret_cast<quint32 *>(cq + params.cq_off.head); + cqTail = reinterpret_cast<quint32 *>(cq + params.cq_off.tail); + cqIndexMask = reinterpret_cast<quint32 *>(cq + params.cq_off.ring_mask); + completionQueueEntries = reinterpret_cast<io_uring_cqe *>(cq + params.cq_off.cqes); + + eventDescriptor = eventfd(0, 0); + io_uring_register(io_uringFd, IORING_REGISTER_EVENTFD, &eventDescriptor, 1); + + notifier.emplace(eventDescriptor, QSocketNotifier::Read); + QObject::connect(std::addressof(*notifier), &QSocketNotifier::activated, + std::addressof(*notifier), [this]() { completionReady(); }); + return true; +} + +template <QIORing::Operation Op> +Q_ALWAYS_INLINE QIORing::ReadWriteStatus QIORing::handleReadCompletion(const io_uring_cqe *cqe, GenericRequestType *request) +{ + auto *readRequest = request->requestData<Op>(); + Q_ASSERT(readRequest); + auto *destinations = [&readRequest]() { + if constexpr (Op == Operation::Read) + return &readRequest->destination; + else + return &readRequest->destinations[0]; + }(); + + if (cqe->res < 0) { + if (-cqe->res == ECANCELED) + readRequest->result.template emplace<QFileDevice::FileError>(QFileDevice::AbortError); + else + readRequest->result.template emplace<QFileDevice::FileError>(QFileDevice::ReadError); + } else if (auto *extra = request->getExtra<QtPrivate::ReadWriteExtra>()) { + const qint32 bytesRead = cqe->res; + qCDebug(lcQIORing) << "Partial read of" << bytesRead << "bytes completed"; + auto &readResult = [&readRequest]() -> QIORingResult<Op> & { + if (auto *result = std::get_if<QIORingResult<Op>>(&readRequest->result)) + return *result; + return readRequest->result.template emplace<QIORingResult<Op>>(); + }(); + readResult.bytesRead += bytesRead; + extra->spanOffset += qsizetype(bytesRead); + qCDebug(lcQIORing) << "Read operation progress: span" << extra->spanIndex << "offset" + << extra->spanOffset << "of" << destinations[extra->spanIndex].size() + << "bytes. Total read:" << readResult.bytesRead << "bytes"; + // The while loop is in case there is an empty span, we skip over it: + while (extra->spanOffset == destinations[extra->spanIndex].size()) { + // Move to next span + if (++extra->spanIndex == extra->numSpans) { + --ongoingSplitOperations; + return ReadWriteStatus::Finished; + } + extra->spanOffset = 0; + } + + QSpan<std::byte> span = destinations[extra->spanIndex].subspan(extra->spanOffset); + if (span.size() > MaxReadWriteLen) + span = span.first(MaxReadWriteLen); + + // Move the request such that it is next in the list to be processed: + auto &it = addrItMap[request]; + const auto where = lastUnqueuedIterator.value_or(pendingRequests.end()); + pendingRequests.splice(where, pendingRequests, it); + it = std::prev(where); + lastUnqueuedIterator = it; + + return ReadWriteStatus::MoreToDo; + } else { + auto &result = readRequest->result.template emplace<QIORingResult<Op>>(); + result.bytesRead = cqe->res; + } + return ReadWriteStatus::Finished; +} + +template <QIORing::Operation Op> +Q_ALWAYS_INLINE QIORing::ReadWriteStatus QIORing::handleWriteCompletion(const io_uring_cqe *cqe, GenericRequestType *request) +{ + auto *writeRequest = request->requestData<Op>(); + Q_ASSERT(writeRequest); + auto *sources = [&writeRequest]() { + if constexpr (Op == Operation::Write) + return &writeRequest->source; + else + return &writeRequest->sources[0]; + }(); + + if (cqe->res < 0) { + if (-cqe->res == ECANCELED) + writeRequest->result.template emplace<QFileDevice::FileError>(QFileDevice::AbortError); + else + writeRequest->result.template emplace<QFileDevice::FileError>(QFileDevice::WriteError); + } else if (auto *extra = request->getExtra<QtPrivate::ReadWriteExtra>()) { + const qint32 bytesWritten = cqe->res; + qCDebug(lcQIORing) << "Partial write of" << bytesWritten << "bytes completed"; + auto &writeResult = [&writeRequest]() -> QIORingResult<Op> & { + if (auto *result = std::get_if<QIORingResult<Op>>(&writeRequest->result)) + return *result; + return writeRequest->result.template emplace<QIORingResult<Op>>(); + }(); + writeResult.bytesWritten += bytesWritten; + extra->spanOffset += qsizetype(bytesWritten); + qCDebug(lcQIORing) << "Write operation progress: span" << extra->spanIndex << "offset" + << extra->spanOffset << "of" << sources[extra->spanIndex].size() + << "bytes. Total written:" << writeResult.bytesWritten << "bytes"; + // The while loop is in case there is an empty span, we skip over it: + while (extra->spanOffset == sources[extra->spanIndex].size()) { + // Move to next span + if (++extra->spanIndex == extra->numSpans) { + --ongoingSplitOperations; + return ReadWriteStatus::Finished; + } + extra->spanOffset = 0; + } + + QSpan<const std::byte> span = sources[extra->spanIndex].subspan(extra->spanOffset); + if (span.size() > MaxReadWriteLen) + span = span.first(MaxReadWriteLen); + + // Move the request such that it is next in the list to be processed: + auto &it = addrItMap[request]; + const auto where = lastUnqueuedIterator.value_or(pendingRequests.end()); + pendingRequests.splice(where, pendingRequests, it); + it = std::prev(where); + lastUnqueuedIterator = it; + + return ReadWriteStatus::MoreToDo; + } else { + auto &result = writeRequest->result.template emplace<QIORingResult<Op>>(); + result.bytesWritten = cqe->res; + } + return ReadWriteStatus::Finished; +} + +void QIORing::completionReady() +{ + // Drain the eventfd: + [[maybe_unused]] + quint64 ignored = 0; + std::ignore = read(eventDescriptor, &ignored, sizeof(ignored)); + + quint32 head = __atomic_load_n(cqHead, __ATOMIC_RELAXED); + const quint32 tail = __atomic_load_n(cqTail, __ATOMIC_ACQUIRE); + if (tail == head) + return; + + qCDebug(lcQIORing, + "Status of completion queue, total entries: %u, tail: %u, head: %u, to process: %u", + cqEntries, tail, head, (tail - head)); + while (head != tail) { + /* Get the entry */ + const io_uring_cqe *cqe = &completionQueueEntries[head & *cqIndexMask]; + ++head; + GenericRequestType *request = reinterpret_cast<GenericRequestType *>(cqe->user_data); + qCDebug(lcQIORing) << "Got completed entry. Operation:" << request->operation() + << "- user_data pointer:" << request; + switch (request->operation()) { + case Operation::Open: { + QIORingRequest<Operation::Open> + openRequest = request->template takeRequestData<Operation::Open>(); + if (cqe->res < 0) { + // qErrnoWarning(-cqe->res, "Failed to open"); + if (-cqe->res == ECANCELED) + openRequest.result.template emplace<QFileDevice::FileError>( + QFileDevice::AbortError); + else + openRequest.result.template emplace<QFileDevice::FileError>( + QFileDevice::OpenError); + } else { + auto &result = openRequest.result + .template emplace<QIORingResult<Operation::Open>>(); + result.fd = cqe->res; + } + invokeCallback(openRequest); + break; + } + case Operation::Close: { + QIORingRequest<Operation::Close> + closeRequest = request->template takeRequestData<Operation::Close>(); + if (cqe->res < 0) { + closeRequest.result.emplace<QFileDevice::FileError>(QFileDevice::OpenError); + } else { + closeRequest.result.emplace<QIORingResult<Operation::Close>>(); + } + invokeCallback(closeRequest); + break; + } + case Operation::Read: { + const ReadWriteStatus status = handleReadCompletion<Operation::Read>(cqe, request); + if (status == ReadWriteStatus::MoreToDo) + continue; + auto readRequest = request->takeRequestData<Operation::Read>(); + invokeCallback(readRequest); + break; + } + case Operation::Write: { + const ReadWriteStatus status = handleWriteCompletion<Operation::Write>(cqe, request); + if (status == ReadWriteStatus::MoreToDo) + continue; + auto writeRequest = request->takeRequestData<Operation::Write>(); + invokeCallback(writeRequest); + break; + } + case Operation::VectoredRead: { + const ReadWriteStatus status = handleReadCompletion<Operation::VectoredRead>(cqe, request); + if (status == ReadWriteStatus::MoreToDo) + continue; + auto readvRequest = request->takeRequestData<Operation::VectoredRead>(); + invokeCallback(readvRequest); + break; + } + case Operation::VectoredWrite: { + const ReadWriteStatus status = handleWriteCompletion<Operation::VectoredWrite>(cqe, request); + if (status == ReadWriteStatus::MoreToDo) + continue; + auto writevRequest = request->takeRequestData<Operation::VectoredWrite>(); + invokeCallback(writevRequest); + break; + } + case Operation::Flush: { + QIORingRequest<Operation::Flush> + flushRequest = request->template takeRequestData<Operation::Flush>(); + if (cqe->res < 0) { + flushRequest.result.emplace<QFileDevice::FileError>(QFileDevice::WriteError); + } else { + // No members to fill out, so just initialize to indicate success + flushRequest.result.emplace<QIORingResult<Operation::Flush>>(); + } + flushInProgress = false; + invokeCallback(flushRequest); + break; + } + case Operation::Cancel: { + QIORingRequest<Operation::Cancel> + cancelRequest = request->template takeRequestData<Operation::Cancel>(); + invokeCallback(cancelRequest); + break; + } + case Operation::Stat: { + QIORingRequest<Operation::Stat> + statRequest = request->template takeRequestData<Operation::Stat>(); + if (cqe->res < 0) { + statRequest.result.emplace<QFileDevice::FileError>(QFileDevice::OpenError); + } else { + struct statx *st = request->getExtra<struct statx>(); + Q_ASSERT(st); + auto &res = statRequest.result.emplace<QIORingResult<Operation::Stat>>(); + res.size = st->stx_size; + } + invokeCallback(statRequest); + break; + } + case Operation::NumOperations: + Q_UNREACHABLE_RETURN(); + break; + } + --inFlightRequests; + auto it = addrItMap.take(request); + pendingRequests.erase(it); + } + __atomic_store_n(cqHead, head, __ATOMIC_RELEASE); + qCDebug(lcQIORing, + "Done processing available completions, updated pointers, tail: %u, head: %u", tail, + head); + prepareRequests(); + if (!stagePending && unstagedRequests > 0) + submitRequests(); +} + +bool QIORing::waitForCompletions(QDeadlineTimer deadline) +{ + notifier->setEnabled(false); + auto reactivateNotifier = qScopeGuard([this]() { + notifier->setEnabled(true); + }); + + pollfd pfd = qt_make_pollfd(eventDescriptor, POLLIN); + return qt_safe_poll(&pfd, 1, deadline) > 0; +} + +bool QIORing::supportsOperation(Operation op) +{ + switch (op) { + case QtPrivate::Operation::Open: + case QtPrivate::Operation::Close: + case QtPrivate::Operation::Read: + case QtPrivate::Operation::Write: + case QtPrivate::Operation::VectoredRead: + case QtPrivate::Operation::VectoredWrite: + case QtPrivate::Operation::Flush: + case QtPrivate::Operation::Cancel: + case QtPrivate::Operation::Stat: + return true; + case QtPrivate::Operation::NumOperations: + return false; + } + return false; // May not always be unreachable! +} + +void QIORing::submitRequests() +{ + stagePending = false; + if (unstagedRequests == 0) + return; + + auto submitToRing = [this] { + int ret = io_uring_enter(io_uringFd, unstagedRequests, 0, 0, nullptr); + if (ret < 0) + qErrnoWarning("Error occurred notifying kernel about requests..."); + else + unstagedRequests -= ret; + qCDebug(lcQIORing) << "io_uring_enter returned" << ret; + return ret >= 0; + }; + if (submitToRing()) { + prepareRequests(); + if (unstagedRequests) + submitToRing(); + } +} + +namespace QtPrivate { +template <typename T> +using DetectFd = decltype(std::declval<const T &>().fd); + +template <typename T> +constexpr bool HasFdMember = qxp::is_detected_v<DetectFd, T>; +} // namespace QtPrivate + +bool QIORing::verifyFd(QIORing::GenericRequestType &req) +{ + bool result = true; + invokeOnOp(req, [&](auto *request) { + if constexpr (QtPrivate::HasFdMember<decltype(*request)>) { + result = request->fd > 0; + } + }); + return result; +} + +void QIORing::prepareRequests() +{ + if (!lastUnqueuedIterator) { + qCDebug(lcQIORing, "Nothing left to queue"); + return; + } + Q_ASSERT(!preparingRequests); + QScopedValueRollback<bool> prepareGuard(preparingRequests, true); + + quint32 tail = __atomic_load_n(sqTail, __ATOMIC_RELAXED); + const quint32 head = __atomic_load_n(sqHead, __ATOMIC_ACQUIRE); + qCDebug(lcQIORing, + "Status of submission queue, total entries: %u, tail: %u, head: %u, free: %u", + sqEntries, tail, head, sqEntries - (tail - head)); + + auto it = *lastUnqueuedIterator; + lastUnqueuedIterator.reset(); + const auto end = pendingRequests.end(); + bool anyQueued = false; + // Loop until we either: + // 1. Run out of requests to prepare for submission (it == end), + // 2. Have filled the submission queue (unstagedRequests == sqEntries) or, + // 3. The number of staged requests + currently processing/potentially finished requests is + // enough to fill the completion queue (inFlightRequests == cqEntries). + while (!flushInProgress && unstagedRequests != sqEntries && inFlightRequests != cqEntries + && it != end) { + const quint32 index = tail & *sqIndexMask; + io_uring_sqe *sqe = &submissionQueueEntries[index]; + *sqe = {}; + RequestPrepResult result = prepareRequest(sqe, *it); + + // QueueFull is unused on Linux: + Q_ASSERT(result != RequestPrepResult::QueueFull); + if (result == RequestPrepResult::Defer) { + qCDebug(lcQIORing) << "Request for" << it->operation() + << "had to be deferred, will not queue any more requests at the moment."; + break; + } + if (result == RequestPrepResult::RequestCompleted) { + addrItMap.remove(std::addressof(*it)); + it = pendingRequests.erase(it); // Completed synchronously, either failure or success. + continue; + } + anyQueued = true; + it->setQueued(true); + + sqIndexArray[index] = index; + ++inFlightRequests; + ++unstagedRequests; + ++tail; + ++it; + } + if (it != end) + lastUnqueuedIterator = it; + + if (anyQueued) { + qCDebug(lcQIORing, "Queued %u operation(s)", + tail - __atomic_load_n(sqTail, __ATOMIC_RELAXED)); + __atomic_store_n(sqTail, tail, __ATOMIC_RELEASE); + } +} + +static io_uring_op toUringOp(QIORing::Operation op) +{ + switch (op) { + case QIORing::Operation::Open: + return IORING_OP_OPENAT; + case QIORing::Operation::Read: + return IORING_OP_READ; + case QIORing::Operation::Close: + return IORING_OP_CLOSE; + case QIORing::Operation::Write: + return IORING_OP_WRITE; + case QIORing::Operation::VectoredRead: + return IORING_OP_READV; + case QIORing::Operation::VectoredWrite: + return IORING_OP_WRITEV; + case QIORing::Operation::Flush: + return IORING_OP_FSYNC; + case QIORing::Operation::Cancel: + return IORING_OP_ASYNC_CANCEL; + case QIORing::Operation::Stat: + return IORING_OP_STATX; + case QIORing::Operation::NumOperations: + break; + } + Q_UNREACHABLE_RETURN(IORING_OP_NOP); +} + +Q_ALWAYS_INLINE +static void prepareFileIOCommon(io_uring_sqe *sqe, const QIORingRequestOffsetFdBase &request) +{ + sqe->fd = qint32(request.fd); + sqe->off = request.offset; +} + +Q_ALWAYS_INLINE +static void prepareFileReadWrite(io_uring_sqe *sqe, const QIORingRequestOffsetFdBase &request, + const void *address, qsizetype size) +{ + prepareFileIOCommon(sqe, request); + sqe->len = quint32(size); + sqe->addr = quint64(address); +} + +// @todo: stolen from qfsfileengine_unix.cpp +static inline int openModeToOpenFlags(QIODevice::OpenMode mode) +{ + int oflags = QT_OPEN_RDONLY; +#ifdef QT_LARGEFILE_SUPPORT + oflags |= QT_OPEN_LARGEFILE; +#endif + + if ((mode & QIODevice::ReadWrite) == QIODevice::ReadWrite) + oflags = QT_OPEN_RDWR; + else if (mode & QIODevice::WriteOnly) + oflags = QT_OPEN_WRONLY; + + if ((mode & QIODevice::WriteOnly) + && !(mode & QIODevice::ExistingOnly)) // QFSFileEnginePrivate::openModeCanCreate(mode)) + oflags |= QT_OPEN_CREAT; + + if (mode & QIODevice::Truncate) + oflags |= QT_OPEN_TRUNC; + + if (mode & QIODevice::Append) + oflags |= QT_OPEN_APPEND; + + if (mode & QIODevice::NewOnly) + oflags |= QT_OPEN_EXCL; + + return oflags; +} + +auto QIORing::prepareRequest(io_uring_sqe *sqe, GenericRequestType &request) -> RequestPrepResult +{ + sqe->user_data = qint64(&request); + sqe->opcode = toUringOp(request.operation()); + + if (!verifyFd(request)) { + finishRequestWithError(request, QFileDevice::OpenError); + return RequestPrepResult::RequestCompleted; + } + + switch (request.operation()) { + case Operation::Open: { + const QIORingRequest<Operation::Open> + *openRequest = request.template requestData<Operation::Open>(); + sqe->fd = AT_FDCWD; // Could also support proper openat semantics + sqe->addr = reinterpret_cast<quint64>(openRequest->path.native().c_str()); + sqe->open_flags = openModeToOpenFlags(openRequest->flags); + auto &mode = sqe->len; + mode = 0666; // With an explicit API we can use QtPrivate::toMode_t() for this + break; + } + case Operation::Close: { + if (ongoingSplitOperations) + return Defer; + const QIORingRequest<Operation::Close> + *closeRequest = request.template requestData<Operation::Close>(); + sqe->fd = closeRequest->fd; + // Force all earlier entries in the sq to finish before this is processed: + sqe->flags |= IOSQE_IO_DRAIN; + break; + } + case Operation::Read: { + const QIORingRequest<Operation::Read> + *readRequest = request.template requestData<Operation::Read>(); + auto span = readRequest->destination; + if (span.size() >= MaxReadWriteLen) { + auto *extra = request.getOrInitializeExtra<QtPrivate::ReadWriteExtra>(); + qsizetype remaining = span.size() - extra->spanOffset; + span.slice(extra->spanOffset, std::min(remaining, MaxReadWriteLen)); + ++ongoingSplitOperations; + } + prepareFileReadWrite(sqe, *readRequest, span.data(), span.size()); + break; + } + case Operation::Write: { + const QIORingRequest<Operation::Write> + *writeRequest = request.template requestData<Operation::Write>(); + auto span = writeRequest->source; + if (span.size() >= MaxReadWriteLen) { + auto *extra = request.getOrInitializeExtra<QtPrivate::ReadWriteExtra>(); + qsizetype remaining = span.size() - extra->spanOffset; + span.slice(extra->spanOffset, std::min(remaining, MaxReadWriteLen)); + ++ongoingSplitOperations; + } + prepareFileReadWrite(sqe, *writeRequest, span.data(), span.size()); + break; + } + case Operation::VectoredRead: { + // @todo Apply the split read/write concept that will apply above to this too + const QIORingRequest<Operation::VectoredRead> + *readvRequest = request.template requestData<Operation::VectoredRead>(); + prepareFileReadWrite(sqe, *readvRequest, readvRequest->destinations.data(), + readvRequest->destinations.size()); + break; + } + case Operation::VectoredWrite: { + // @todo Apply the split read/write concept that will apply above to this too + const QIORingRequest<Operation::VectoredWrite> + *writevRequest = request.template requestData<Operation::VectoredWrite>(); + prepareFileReadWrite(sqe, *writevRequest, writevRequest->sources.data(), + writevRequest->sources.size()); + break; + } + case Operation::Flush: { + if (ongoingSplitOperations) + return Defer; + const QIORingRequest<Operation::Flush> + *flushRequest = request.template requestData<Operation::Flush>(); + sqe->fd = qint32(flushRequest->fd); + // Force all earlier entries in the sq to finish before this is processed: + sqe->flags |= IOSQE_IO_DRAIN; + flushInProgress = true; + break; + } + case Operation::Cancel: { + const QIORingRequest<Operation::Cancel> + *cancelRequest = request.template requestData<Operation::Cancel>(); + auto *otherOperation = reinterpret_cast<GenericRequestType *>(cancelRequest->handle); + auto it = std::as_const(addrItMap).find(otherOperation); + if (it == addrItMap.cend()) { // : The request to cancel doesn't exist + invokeCallback(*cancelRequest); + return RequestPrepResult::RequestCompleted; + } + if (!otherOperation->wasQueued()) { + // The request hasn't been queued yet, so we can just drop it from + // the pending requests and call the callback. + Q_ASSERT(!lastUnqueuedIterator); + finishRequestWithError(*otherOperation, QFileDevice::AbortError); + pendingRequests.erase(*it); // otherOperation is deleted + addrItMap.erase(it); + invokeCallback(*cancelRequest); + return RequestPrepResult::RequestCompleted; + } + sqe->addr = quint64(otherOperation); + break; + } + case Operation::Stat: { + const QIORingRequest<Operation::Stat> + *statRequest = request.template requestData<Operation::Stat>(); + // We need to store the statx struct somewhere: + struct statx *st = request.getOrInitializeExtra<struct statx>(); + + sqe->fd = statRequest->fd; + // We want to use the fd as the target of query instead of as the fd of the relative dir, + // so we set addr to an empty string, and specify the AT_EMPTY_PATH flag. + static const char emptystr[] = ""; + sqe->addr = qint64(emptystr); + sqe->statx_flags = AT_EMPTY_PATH; + sqe->len = STATX_ALL; // @todo configure somehow + sqe->off = quint64(st); + break; + } + case Operation::NumOperations: + Q_UNREACHABLE_RETURN(RequestPrepResult::RequestCompleted); + break; + } + return RequestPrepResult::Ok; +} + +void QIORing::GenericRequestType::cleanupExtra(Operation op, void *extra) +{ + switch (op) { + case Operation::Open: + case Operation::Close: + case Operation::VectoredRead: + case Operation::VectoredWrite: + case Operation::Cancel: + case Operation::Flush: + case Operation::NumOperations: + break; + case Operation::Read: + case Operation::Write: + delete static_cast<QtPrivate::ReadWriteExtra *>(extra); + return; + case Operation::Stat: + delete static_cast<struct statx *>(extra); + return; + } +} + +QT_END_NAMESPACE diff --git a/src/corelib/io/qioring_p.h b/src/corelib/io/qioring_p.h new file mode 100644 index 00000000000..d4c4308122e --- /dev/null +++ b/src/corelib/io/qioring_p.h @@ -0,0 +1,484 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#ifndef IOPROCESSOR_P_H +#define IOPROCESSOR_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtCore/private/qtcoreglobal_p.h> + +#include <QtCore/qstring.h> +#include <QtCore/qspan.h> +#include <QtCore/qhash.h> +#include <QtCore/qfiledevice.h> +#include <QtCore/qwineventnotifier.h> +#include <QtCore/qloggingcategory.h> +#include <QtCore/qdeadlinetimer.h> + +#ifdef Q_OS_LINUX +# include <QtCore/qsocketnotifier.h> +struct io_uring_sqe; +struct io_uring_cqe; +#endif + +#include <algorithm> +#include <filesystem> +#include <variant> +#include <optional> +#include <type_traits> + +/* + This file defines an interface for the backend of QRandomAccessFile. + The backends themselves are implemented in platform-specific files, such as + ioring_linux.cpp, ioring_win.cpp, etc. + And has a lower-level interface than the public interface will have, but the + separation hopefully makes it easier to implement the ioring backends, test + them, and tweak them without the higher-level interface needing to see + changes, and to make it possible to tweak the higher-level interface without + needing to touch the (somewhat similar) ioring backends. + + Most of the interface is just an enum QIORing::Operation + the + QIORingRequest template class, which is specialized for each operation so it + carries just the relevant data for that operation. And a small mechanism to + store the request in a generic manner so they can be used in the + implementation files at the cost of some overhead. + + There will be absolutely zero binary compatibility guarantees for this + interface. +*/ + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(lcQIORing); + +namespace QtPrivate { +Q_NAMESPACE + +#define FOREACH_IO_OPERATION(OP) \ + OP(Open) \ + OP(Close) \ + OP(Read) \ + OP(Write) \ + OP(VectoredRead) \ + OP(VectoredWrite) \ + OP(Flush) \ + OP(Stat) \ + OP(Cancel) \ + /**/ +#define DEFINE_ENTRY(OP) OP, + +// clang-format off +enum class Operation : quint8 { + FOREACH_IO_OPERATION(DEFINE_ENTRY) + + NumOperations, +}; +// clang-format on +Q_ENUM_NS(Operation); +#undef DEFINE_ENTRY +}; // namespace QtPrivate + +template <QtPrivate::Operation Op> +struct QIORingRequest; + +class QIORing final +{ + class GenericRequestType; + struct RequestHandleTag; // Just used as an opaque pointer +public: + static constexpr quint32 DefaultSubmissionQueueSize = 128; + static constexpr quint32 DefaultCompletionQueueSize = DefaultSubmissionQueueSize * 2; + using Operation = QtPrivate::Operation; + using RequestHandle = RequestHandleTag *; + + Q_CORE_EXPORT + explicit QIORing(quint32 submissionQueueSize = DefaultSubmissionQueueSize, + quint32 completionQueueSize = DefaultCompletionQueueSize); + Q_CORE_EXPORT + ~QIORing(); + Q_DISABLE_COPY_MOVE(QIORing) + + Q_CORE_EXPORT + static QIORing *sharedInstance(); + bool ensureInitialized() { return initializeIORing(); } + + Q_CORE_EXPORT + static bool supportsOperation(Operation op); + template <Operation Op> + QIORing::RequestHandle queueRequest(QIORingRequest<Op> &&request) + { + Q_ASSERT(supportsOperation(Op)); + auto &r = pendingRequests.emplace_back(std::move(request)); + addrItMap.emplace(&r, std::prev(pendingRequests.end())); + if (queueRequestInternal(r) == QueuedRequestStatus::CompletedImmediately) + return nullptr; // Return an invalid handle, to avoid ABA with following requests + return reinterpret_cast<RequestHandle>(&r); + } + Q_CORE_EXPORT + void submitRequests(); + Q_CORE_EXPORT + bool waitForRequest(RequestHandle handle, QDeadlineTimer deadline = QDeadlineTimer::Forever); + + quint32 submissionQueueSize() const noexcept { return sqEntries; } + quint32 completionQueueSize() const noexcept { return cqEntries; } + +private: + std::list<GenericRequestType> pendingRequests; + using PendingRequestsIterator = decltype(pendingRequests.begin()); + QHash<void *, PendingRequestsIterator> addrItMap; + std::optional<PendingRequestsIterator> lastUnqueuedIterator; + quint32 sqEntries = 0; + quint32 cqEntries = 0; + quint32 inFlightRequests = 0; + quint32 unstagedRequests = 0; + bool stagePending = false; + bool preparingRequests = false; + qsizetype ongoingSplitOperations = 0; + + Q_CORE_EXPORT + bool initializeIORing(); + + enum class QueuedRequestStatus : bool { + Pending = false, + CompletedImmediately = true, + }; + Q_CORE_EXPORT + QueuedRequestStatus queueRequestInternal(GenericRequestType &request); + void prepareRequests(); + void completionReady(); + bool waitForCompletions(QDeadlineTimer deadline); + + template <typename Fun> + static auto invokeOnOp(GenericRequestType &req, Fun fn); + + static void finishRequestWithError(GenericRequestType &req, QFileDevice::FileError error); + static bool verifyFd(GenericRequestType &req); + + enum RequestPrepResult : quint8 { + Ok, + QueueFull, + Defer, + RequestCompleted, + }; + enum class ReadWriteStatus : bool { + MoreToDo, + Finished, + }; +#ifdef Q_OS_LINUX + std::optional<QSocketNotifier> notifier; + // io_uring 'sq', 'sqe', 'cq', and 'cqe' pointers: + void *submissionQueue = nullptr; + io_uring_sqe *submissionQueueEntries = nullptr; + const io_uring_cqe *completionQueueEntries = nullptr; + + // Some pointers for working with the ring-buffer. + // The pointers to const are controlled by the kernel. + const quint32 *sqHead = nullptr; + quint32 *sqTail = nullptr; + const quint32 *sqIndexMask = nullptr; + quint32 *sqIndexArray = nullptr; + quint32 *cqHead = nullptr; + const quint32 *cqTail = nullptr; + const quint32 *cqIndexMask = nullptr; + // Because we want the flush to act as a barrier operation we need to track + // if there is one currently in progress. With kernel 6.16+ this seems to be + // fixed, but since we support older kernels we implement this deferring + // ourselves. + bool flushInProgress = false; + + int io_uringFd = -1; + int eventDescriptor = -1; + [[nodiscard]] + RequestPrepResult prepareRequest(io_uring_sqe *sqe, GenericRequestType &request); + template <Operation Op> + ReadWriteStatus handleReadCompletion(const io_uring_cqe *cqe, GenericRequestType *request); + template <Operation Op> + ReadWriteStatus handleWriteCompletion(const io_uring_cqe *cqe, GenericRequestType *request); +#endif +}; + +struct QIORingRequestEmptyBase +{ +}; + +template <QtPrivate::Operation Op> +struct QIORingResult; +template <QtPrivate::Operation Op> +struct QIORingRequest; + +// @todo: q23::expected once emplace() returns a reference +template <QtPrivate::Operation Op> +using ExpectedResultType = std::variant<std::monostate, QIORingResult<Op>, QFileDevice::FileError>; + +struct QIORingRequestOffsetFdBase : QIORingRequestEmptyBase +{ + qintptr fd; + quint64 offset; +}; + +template <QtPrivate::Operation Op, typename Base = QIORingRequestOffsetFdBase> +struct QIORingRequestBase : Base +{ + ExpectedResultType<Op> result; // To be filled in by the backend + QtPrivate::SlotObjUniquePtr callback; + template <typename Func> + Q_ALWAYS_INLINE void setCallback(Func &&func) + { + using Prototype = void (*)(const QIORingRequest<Op> &); + callback.reset(QtPrivate::makeCallableObject<Prototype>(std::forward<Func>(func))); + } +}; + +template <> +struct QIORingResult<QtPrivate::Operation::Open> +{ + qintptr fd; +}; +template <> +struct QIORingRequest<QtPrivate::Operation::Open> final + : QIORingRequestBase<QtPrivate::Operation::Open, QIORingRequestEmptyBase> +{ + std::filesystem::path path; + QFileDevice::OpenMode flags; +}; +template <> +struct QIORingResult<QtPrivate::Operation::Close> +{ +}; +template <> +struct QIORingRequest<QtPrivate::Operation::Close> final + : QIORingRequestBase<QtPrivate::Operation::Close, QIORingRequestEmptyBase> +{ + qintptr fd; +}; + +template <> +struct QIORingResult<QtPrivate::Operation::Write> +{ + qint64 bytesWritten; +}; +template <> +struct QIORingRequest<QtPrivate::Operation::Write> final + : QIORingRequestBase<QtPrivate::Operation::Write> +{ + QSpan<const std::byte> source; +}; +template <> +struct QIORingResult<QtPrivate::Operation::VectoredWrite> final + : QIORingResult<QtPrivate::Operation::Write> +{ +}; +template <> +struct QIORingRequest<QtPrivate::Operation::VectoredWrite> final + : QIORingRequestBase<QtPrivate::Operation::VectoredWrite> +{ + QSpan<const QSpan<const std::byte>> sources; +}; + +template <> +struct QIORingResult<QtPrivate::Operation::Read> +{ + qint64 bytesRead; +}; +template <> +struct QIORingRequest<QtPrivate::Operation::Read> final + : QIORingRequestBase<QtPrivate::Operation::Read> +{ + QSpan<std::byte> destination; +}; + +template <> +struct QIORingResult<QtPrivate::Operation::VectoredRead> final + : QIORingResult<QtPrivate::Operation::Read> +{ +}; +template <> +struct QIORingRequest<QtPrivate::Operation::VectoredRead> final + : QIORingRequestBase<QtPrivate::Operation::VectoredRead> +{ + QSpan<QSpan<std::byte>> destinations; +}; + +template <> +struct QIORingResult<QtPrivate::Operation::Flush> final +{ + // No value in the result, just a success or failure +}; +template <> +struct QIORingRequest<QtPrivate::Operation::Flush> final : QIORingRequestBase<QtPrivate::Operation::Flush, QIORingRequestEmptyBase> +{ + qintptr fd; +}; + +template <> +struct QIORingResult<QtPrivate::Operation::Stat> final +{ + quint64 size; +}; +template <> +struct QIORingRequest<QtPrivate::Operation::Stat> final + : QIORingRequestBase<QtPrivate::Operation::Stat, QIORingRequestEmptyBase> +{ + qintptr fd; +}; + +// This is not inheriting the QIORingRequestBase because it doesn't have a result, +// whether it was successful or not is indicated by whether the operation +// it was cancelling was successful or not. +template <> +struct QIORingRequest<QtPrivate::Operation::Cancel> final : QIORingRequestEmptyBase +{ + QIORing::RequestHandle handle; + QtPrivate::SlotObjUniquePtr callback; + template <typename Func> + Q_ALWAYS_INLINE void setCallback(Func &&func) + { + using Op = QtPrivate::Operation; + using Prototype = void (*)(const QIORingRequest<Op::Cancel> &); + callback.reset(QtPrivate::makeCallableObject<Prototype>(std::forward<Func>(func))); + } +}; + +template <QIORing::Operation Op> +Q_ALWAYS_INLINE void invokeCallback(const QIORingRequest<Op> &request) +{ + if (!request.callback) + return; + void *args[2] = { nullptr, const_cast<QIORingRequest<Op> *>(&request) }; + request.callback->call(nullptr, args); +} + +class QIORing::GenericRequestType +{ + friend class QIORing; + +#define POPULATE_VARIANT(Op) \ + QIORingRequest<Operation::Op>, \ + /**/ + + std::variant< + FOREACH_IO_OPERATION(POPULATE_VARIANT) + std::monostate + > taggedUnion; + +#undef POPULATE_VARIANT + + void *extraData = nullptr; + bool queued = false; + + template <Operation Op> + Q_ALWAYS_INLINE void initializeStorage(QIORingRequest<Op> &&t) noexcept + { + static_assert(Op < Operation::NumOperations); + taggedUnion.emplace<QIORingRequest<Op>>(std::move(t)); + } + + Q_CORE_EXPORT + static void cleanupExtra(Operation op, void *extra); + template <typename T> + T *getOrInitializeExtra() + { + if (!extraData) + extraData = new T(); + return static_cast<T *>(extraData); + } + template <typename T> + T *getExtra() const + { + return static_cast<T *>(extraData); + } + void reset() noexcept + { + Operation op = operation(); + taggedUnion.emplace<std::monostate>(); + if (extraData) + cleanupExtra(op, std::exchange(extraData, nullptr)); + } + +public: + template <Operation Op> + explicit GenericRequestType(QIORingRequest<Op> &&t) noexcept + { + initializeStorage(std::move(t)); + } + ~GenericRequestType() noexcept + { + reset(); + } + Q_DISABLE_COPY_MOVE(GenericRequestType) + // We have to provide equality operators. Since copying is disabled, we just check for equality + // based on the address in memory. Two requests could be constructed to be equal, but we don't + // actually care because the order in which they are added to the queue may also matter. + friend bool operator==(const GenericRequestType &l, const GenericRequestType &r) noexcept + { + return std::addressof(l) == std::addressof(r); + } + friend bool operator!=(const GenericRequestType &l, const GenericRequestType &r) noexcept + { + return !(l == r); + } + + Operation operation() const { return Operation(taggedUnion.index()); } + template <Operation Op> + QIORingRequest<Op> *requestData() + { + if (operation() == Op) + return std::get_if<QIORingRequest<Op>>(&taggedUnion); + Q_ASSERT("Wrong operation requested, see operation()"); + return nullptr; + } + template <Operation Op> + QIORingRequest<Op> takeRequestData() + { + if (operation() == Op) + return std::move(*std::get_if<QIORingRequest<Op>>(&taggedUnion)); + Q_ASSERT("Wrong operation requested, see operation()"); + return {}; + } + bool wasQueued() const { return queued; } + void setQueued(bool status) { queued = status; } +}; + +template <typename Fun> +auto QIORing::invokeOnOp(GenericRequestType &req, Fun fn) +{ +#define INVOKE_ON_OP(Op) \ +case QIORing::Operation::Op: \ + fn(req.template requestData<Operation::Op>()); \ + return; \ + /**/ + + switch (req.operation()) { + FOREACH_IO_OPERATION(INVOKE_ON_OP) + case QIORing::Operation::NumOperations: + break; + } + + Q_UNREACHABLE(); +#undef INVOKE_ON_OP +} + +namespace QtPrivate { +// The 'extra' struct for Read/Write operations that must be split up +struct ReadWriteExtra +{ + qsizetype spanIndex = 0; + qsizetype spanOffset = 0; + qsizetype numSpans = 1; +}; +} // namespace QtPrivate + +QT_END_NAMESPACE + +#endif // IOPROCESSOR_P_H diff --git a/src/corelib/io/qrandomaccessasyncfile_p_p.h b/src/corelib/io/qrandomaccessasyncfile_p_p.h index 924c9f9ed83..11ad788c884 100644 --- a/src/corelib/io/qrandomaccessasyncfile_p_p.h +++ b/src/corelib/io/qrandomaccessasyncfile_p_p.h @@ -43,6 +43,11 @@ #endif // Q_OS_DARWIN +#ifdef QT_RANDOMACCESSASYNCFILE_QIORING +#include <QtCore/private/qioring_p.h> +#include <QtCore/qlist.h> +#endif + QT_BEGIN_NAMESPACE class QRandomAccessAsyncFilePrivate : public QObjectPrivate @@ -114,6 +119,15 @@ private: void processFlush(); void processOpen(); void operationComplete(); +#elif defined(QT_RANDOMACCESSASYNCFILE_QIORING) + void queueCompletion(QIOOperationPrivate *priv, QIOOperation::Error error); + void startReadIntoSingle(QIOOperation *op, const QSpan<std::byte> &to); + void startWriteFromSingle(QIOOperation *op, const QSpan<const std::byte> &from); + QIORing::RequestHandle cancel(QIORing::RequestHandle handle); + QIORing *m_ioring = nullptr; + qintptr m_fd = -1; + QList<QPointer<QIOOperation>> m_operations; + QHash<QIOOperation *, QIORing::RequestHandle> m_opHandleMap; #endif #ifdef Q_OS_DARWIN using OperationId = quint64; diff --git a/src/corelib/io/qrandomaccessasyncfile_qioring.cpp b/src/corelib/io/qrandomaccessasyncfile_qioring.cpp new file mode 100644 index 00000000000..c9783ea2856 --- /dev/null +++ b/src/corelib/io/qrandomaccessasyncfile_qioring.cpp @@ -0,0 +1,438 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#include "qrandomaccessasyncfile_p_p.h" + +#include "qiooperation_p.h" +#include "qiooperation_p_p.h" + +#include <QtCore/qfile.h> // QtPrivate::toFilesystemPath +#include <QtCore/qtypes.h> +#include <QtCore/private/qioring_p.h> + +#include <QtCore/q26numeric.h> + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(lcQRandomAccessIORing, "qt.core.qrandomaccessasyncfile.ioring", + QtCriticalMsg); + +QRandomAccessAsyncFilePrivate::QRandomAccessAsyncFilePrivate() = default; + +QRandomAccessAsyncFilePrivate::~QRandomAccessAsyncFilePrivate() = default; + +void QRandomAccessAsyncFilePrivate::init() +{ + m_ioring = QIORing::sharedInstance(); + if (!m_ioring) + qCCritical(lcQRandomAccessIORing, "QRandomAccessAsyncFile: ioring failed to initialize"); +} + +QIORing::RequestHandle QRandomAccessAsyncFilePrivate::cancel(QIORing::RequestHandle handle) +{ + if (handle) { + QIORingRequest<QIORing::Operation::Cancel> cancelRequest; + cancelRequest.handle = handle; + return m_ioring->queueRequest(std::move(cancelRequest)); + } + return nullptr; +} + +void QRandomAccessAsyncFilePrivate::cancelAndWait(QIOOperation *op) +{ + auto *opHandle = m_opHandleMap.value(op); + if (auto *handle = cancel(opHandle)) { + m_ioring->waitForRequest(handle); + m_ioring->waitForRequest(opHandle); + } +} + +void QRandomAccessAsyncFilePrivate::queueCompletion(QIOOperationPrivate *priv, QIOOperation::Error error) +{ + // Remove the handle now in case the user cancels or deletes the io-operation + // before operationComplete is called - the null-handle will protect from + // nasty issues that may occur when trying to cancel an operation that's no + // longer in the queue: + m_opHandleMap.remove(priv->q_func()); + // @todo: Look into making it emit only if synchronously completed + QMetaObject::invokeMethod(priv->q_ptr, [priv, error](){ + priv->operationComplete(error); + }, Qt::QueuedConnection); +} + +QIOOperation *QRandomAccessAsyncFilePrivate::open(const QString &path, QIODeviceBase::OpenMode mode) +{ + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->type = QIOOperation::Type::Open; + + auto *op = new QIOOperation(*priv, q_ptr); + if (m_fileState != FileState::Closed) { + queueCompletion(priv, QIOOperation::Error::Open); + return op; + } + m_operations.append(op); + m_fileState = FileState::OpenPending; + + QIORingRequest<QIORing::Operation::Open> openOperation; + openOperation.path = QtPrivate::toFilesystemPath(path); + openOperation.flags = mode; + openOperation.setCallback([this, op, + priv](const QIORingRequest<QIORing::Operation::Open> &request) { + if (const auto *err = std::get_if<QFileDevice::FileError>(&request.result)) { + if (m_fileState != FileState::Opened) { + // We assume there was only one pending open() in flight. + m_fd = -1; + m_fileState = FileState::Closed; + } + if (priv->error == QIOOperation::Error::Aborted || *err == QFileDevice::AbortError) + queueCompletion(priv, QIOOperation::Error::Aborted); + else + queueCompletion(priv, QIOOperation::Error::Open); + } else if (const auto *result = std::get_if<QIORingResult<QIORing::Operation::Open>>( + &request.result)) { + if (m_fileState == FileState::OpenPending) { + m_fileState = FileState::Opened; + m_fd = result->fd; + queueCompletion(priv, QIOOperation::Error::None); + } else { // Something went wrong, we did not expect a callback: + // So we close the new handle: + QIORingRequest<QIORing::Operation::Close> closeRequest; + closeRequest.fd = result->fd; + QIORing::RequestHandle handle = m_ioring->queueRequest(std::move(closeRequest)); + // Since the user issued multiple open() calls they get to wait for the close() to + // finish: + m_ioring->waitForRequest(handle); + queueCompletion(priv, QIOOperation::Error::Open); + } + } + m_operations.removeOne(op); + }); + m_opHandleMap.insert(priv->q_func(), m_ioring->queueRequest(std::move(openOperation))); + + return op; +} + +void QRandomAccessAsyncFilePrivate::close() +{ + // all the operations should be aborted + const auto ops = std::exchange(m_operations, {}); + QList<QIORing::RequestHandle> tasksToAwait; + // Request to cancel all of the in-flight operations: + for (const auto &op : ops) { + if (op) { + op->d_func()->error = QIOOperation::Error::Aborted; + if (auto *opHandle = m_opHandleMap.value(op)) { + tasksToAwait.append(cancel(opHandle)); + tasksToAwait.append(opHandle); + } + } + } + + QIORingRequest<QIORing::Operation::Close> closeRequest; + closeRequest.fd = m_fd; + tasksToAwait.append(m_ioring->queueRequest(std::move(closeRequest))); + + // Wait for completion: + for (const QIORing::RequestHandle &handle : tasksToAwait) + m_ioring->waitForRequest(handle); + m_fileState = FileState::Closed; + m_fd = -1; +} + +qint64 QRandomAccessAsyncFilePrivate::size() const +{ + QIORingRequest<QIORing::Operation::Stat> statRequest; + statRequest.fd = m_fd; + qint64 finalSize = 0; + statRequest.setCallback([&finalSize](const QIORingRequest<QIORing::Operation::Stat> &request) { + if (const auto *err = std::get_if<QFileDevice::FileError>(&request.result)) { + Q_UNUSED(err); + finalSize = -1; + } else if (const auto *res = std::get_if<QIORingResult<QIORing::Operation::Stat>>(&request.result)) { + finalSize = q26::saturate_cast<qint64>(res->size); + } + }); + auto *handle = m_ioring->queueRequest(std::move(statRequest)); + m_ioring->waitForRequest(handle); + + return finalSize; +} + +QIOOperation *QRandomAccessAsyncFilePrivate::flush() +{ + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->type = QIOOperation::Type::Flush; + + auto *op = new QIOOperation(*priv, q_ptr); + m_operations.append(op); + + QIORingRequest<QIORing::Operation::Flush> flushRequest; + flushRequest.fd = m_fd; + flushRequest.setCallback([this, op](const QIORingRequest<QIORing::Operation::Flush> &request) { + auto *priv = QIOOperationPrivate::get(op); + if (const auto *err = std::get_if<QFileDevice::FileError>(&request.result)) { + if (priv->error == QIOOperation::Error::Aborted || *err == QFileDevice::AbortError) + queueCompletion(priv, QIOOperation::Error::Aborted); + else if (*err == QFileDevice::OpenError) + queueCompletion(priv, QIOOperation::Error::FileNotOpen); + else + queueCompletion(priv, QIOOperation::Error::Flush); + } else if (std::get_if<QIORingResult<QIORing::Operation::Flush>>(&request.result)) { + queueCompletion(priv, QIOOperation::Error::None); + } + m_operations.removeOne(op); + }); + m_opHandleMap.insert(priv->q_func(), m_ioring->queueRequest(std::move(flushRequest))); + + return op; +} + +void QRandomAccessAsyncFilePrivate::startReadIntoSingle(QIOOperation *op, + const QSpan<std::byte> &to) +{ + QIORingRequest<QIORing::Operation::Read> readRequest; + readRequest.fd = m_fd; + auto *priv = QIOOperationPrivate::get(op); + if (priv->offset < 0) { // The QIORing offset is unsigned, so error out now + queueCompletion(priv, QIOOperation::Error::IncorrectOffset); + m_operations.removeOne(op); + return; + } + readRequest.offset = priv->offset; + readRequest.destination = to; + readRequest.setCallback([this, op](const QIORingRequest<QIORing::Operation::Read> &request) { + auto *priv = QIOOperationPrivate::get(op); + if (const auto *err = std::get_if<QFileDevice::FileError>(&request.result)) { + if (priv->error == QIOOperation::Error::Aborted || *err == QFileDevice::AbortError) + queueCompletion(priv, QIOOperation::Error::Aborted); + else if (*err == QFileDevice::OpenError) + queueCompletion(priv, QIOOperation::Error::FileNotOpen); + else if (*err == QFileDevice::PositionError) + queueCompletion(priv, QIOOperation::Error::IncorrectOffset); + else + queueCompletion(priv, QIOOperation::Error::Read); + } else if (const auto *result = std::get_if<QIORingResult<QIORing::Operation::Read>>( + &request.result)) { + priv->appendBytesProcessed(result->bytesRead); + if (priv->dataStorage->containsReadSpans()) + priv->dataStorage->getReadSpans().first().slice(0, result->bytesRead); + else + priv->dataStorage->getByteArray().slice(0, result->bytesRead); + + queueCompletion(priv, QIOOperation::Error::None); + } + m_operations.removeOne(op); + }); + m_opHandleMap.insert(priv->q_func(), m_ioring->queueRequest(std::move(readRequest))); +} + +QIOReadOperation *QRandomAccessAsyncFilePrivate::read(qint64 offset, qint64 maxSize) +{ + QByteArray array; + array.resizeForOverwrite(maxSize); + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(std::move(array)); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Read; + + auto *op = new QIOReadOperation(*priv, q_ptr); + m_operations.append(op); + + startReadIntoSingle(op, as_writable_bytes(QSpan(dataStorage->getByteArray()))); + + return op; +} + +QIOWriteOperation *QRandomAccessAsyncFilePrivate::write(qint64 offset, const QByteArray &data) +{ + return write(offset, QByteArray(data)); +} + +void QRandomAccessAsyncFilePrivate::startWriteFromSingle(QIOOperation *op, + const QSpan<const std::byte> &from) +{ + QIORingRequest<QIORing::Operation::Write> writeRequest; + writeRequest.fd = m_fd; + auto *priv = QIOOperationPrivate::get(op); + if (priv->offset < 0) { // The QIORing offset is unsigned, so error out now + queueCompletion(priv, QIOOperation::Error::IncorrectOffset); + m_operations.removeOne(op); + return; + } + writeRequest.offset = priv->offset; + writeRequest.source = from; + writeRequest.setCallback([this, op](const QIORingRequest<QIORing::Operation::Write> &request) { + auto *priv = QIOOperationPrivate::get(op); + if (const auto *err = std::get_if<QFileDevice::FileError>(&request.result)) { + if (priv->error == QIOOperation::Error::Aborted || *err == QFileDevice::AbortError) + queueCompletion(priv, QIOOperation::Error::Aborted); + else if (*err == QFileDevice::OpenError) + queueCompletion(priv, QIOOperation::Error::FileNotOpen); + else if (*err == QFileDevice::PositionError) + queueCompletion(priv, QIOOperation::Error::IncorrectOffset); + else + queueCompletion(priv, QIOOperation::Error::Write); + } else if (const auto *result = std::get_if<QIORingResult<QIORing::Operation::Write>>( + &request.result)) { + priv->appendBytesProcessed(result->bytesWritten); + queueCompletion(priv, QIOOperation::Error::None); + } + m_operations.removeOne(op); + }); + m_opHandleMap.insert(priv->q_func(), m_ioring->queueRequest(std::move(writeRequest))); +} + +QIOWriteOperation *QRandomAccessAsyncFilePrivate::write(qint64 offset, QByteArray &&data) +{ + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(std::move(data)); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Write; + + auto *op = new QIOWriteOperation(*priv, q_ptr); + m_operations.append(op); + + startWriteFromSingle(op, as_bytes(QSpan(dataStorage->getByteArray()))); + + return op; +} + +QIOVectoredReadOperation *QRandomAccessAsyncFilePrivate::readInto(qint64 offset, + QSpan<std::byte> buffer) +{ + auto *dataStorage = new QtPrivate::QIOOperationDataStorage( + QSpan<const QSpan<std::byte>>{ buffer }); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Read; + + auto *op = new QIOVectoredReadOperation(*priv, q_ptr); + m_operations.append(op); + + startReadIntoSingle(op, dataStorage->getReadSpans().first()); + + return op; +} + +QIOVectoredWriteOperation *QRandomAccessAsyncFilePrivate::writeFrom(qint64 offset, + QSpan<const std::byte> buffer) +{ + auto *dataStorage = new QtPrivate::QIOOperationDataStorage( + QSpan<const QSpan<const std::byte>>{ buffer }); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Write; + + auto *op = new QIOVectoredWriteOperation(*priv, q_ptr); + m_operations.append(op); + + startWriteFromSingle(op, dataStorage->getWriteSpans().first()); + + return op; +} + +QIOVectoredReadOperation * +QRandomAccessAsyncFilePrivate::readInto(qint64 offset, QSpan<const QSpan<std::byte>> buffers) +{ + if (!QIORing::supportsOperation(QtPrivate::Operation::VectoredRead)) + return nullptr; + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(buffers); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Read; + + auto *op = new QIOVectoredReadOperation(*priv, q_ptr); + if (priv->offset < 0) { // The QIORing offset is unsigned, so error out now + queueCompletion(priv, QIOOperation::Error::IncorrectOffset); + return op; + } + m_operations.append(op); + + QIORingRequest<QIORing::Operation::VectoredRead> readRequest; + readRequest.fd = m_fd; + readRequest.offset = priv->offset; + readRequest.destinations = dataStorage->getReadSpans(); + readRequest.setCallback([this, + op](const QIORingRequest<QIORing::Operation::VectoredRead> &request) { + auto *priv = QIOOperationPrivate::get(op); + if (const auto *err = std::get_if<QFileDevice::FileError>(&request.result)) { + if (priv->error == QIOOperation::Error::Aborted || *err == QFileDevice::AbortError) + queueCompletion(priv, QIOOperation::Error::Aborted); + else + queueCompletion(priv, QIOOperation::Error::Read); + } else if (const auto + *result = std::get_if<QIORingResult<QIORing::Operation::VectoredRead>>( + &request.result)) { + priv->appendBytesProcessed(result->bytesRead); + qint64 processed = result->bytesRead; + for (auto &span : priv->dataStorage->getReadSpans()) { + if (span.size() < processed) { + processed -= span.size(); + } else { // span.size >= processed + span.slice(0, processed); + processed = 0; + } + } + queueCompletion(priv, QIOOperation::Error::None); + } + m_operations.removeOne(op); + }); + m_opHandleMap.insert(priv->q_func(), m_ioring->queueRequest(std::move(readRequest))); + + return op; +} + +QIOVectoredWriteOperation * +QRandomAccessAsyncFilePrivate::writeFrom(qint64 offset, QSpan<const QSpan<const std::byte>> buffers) +{ + if (!QIORing::supportsOperation(QtPrivate::Operation::VectoredWrite)) + return nullptr; + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(buffers); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Write; + + auto *op = new QIOVectoredWriteOperation(*priv, q_ptr); + if (priv->offset < 0) { // The QIORing offset is unsigned, so error out now + queueCompletion(priv, QIOOperation::Error::IncorrectOffset); + return op; + } + m_operations.append(op); + + QIORingRequest<QIORing::Operation::VectoredWrite> writeRequest; + writeRequest.fd = m_fd; + writeRequest.offset = priv->offset; + writeRequest.sources = dataStorage->getWriteSpans(); + writeRequest.setCallback( + [this, op](const QIORingRequest<QIORing::Operation::VectoredWrite> &request) { + auto *priv = QIOOperationPrivate::get(op); + if (const auto *err = std::get_if<QFileDevice::FileError>(&request.result)) { + if (priv->error == QIOOperation::Error::Aborted || *err == QFileDevice::AbortError) + queueCompletion(priv, QIOOperation::Error::Aborted); + else + queueCompletion(priv, QIOOperation::Error::Write); + } else if (const auto *result = std::get_if< + QIORingResult<QIORing::Operation::VectoredWrite>>( + &request.result)) { + priv->appendBytesProcessed(result->bytesWritten); + queueCompletion(priv, QIOOperation::Error::None); + } + m_operations.removeOne(op); + }); + m_opHandleMap.insert(priv->q_func(), m_ioring->queueRequest(std::move(writeRequest))); + + return op; +} + +QT_END_NAMESPACE diff --git a/src/corelib/tools/qlist.h b/src/corelib/tools/qlist.h index a11f7913dc7..e69b9aebabb 100644 --- a/src/corelib/tools/qlist.h +++ b/src/corelib/tools/qlist.h @@ -301,21 +301,27 @@ public: explicit QList(qsizetype size) : d(size) { - if (size) + if (size) { + Q_CHECK_PTR(d.data()); d->appendInitialize(size); + } } QList(qsizetype size, parameter_type t) : d(size) { - if (size) + if (size) { + Q_CHECK_PTR(d.data()); d->copyAppend(size, t); + } } inline QList(std::initializer_list<T> args) : d(qsizetype(args.size())) { - if (args.size()) + if (args.size()) { + Q_CHECK_PTR(d.data()); d->copyAppend(args.begin(), args.end()); + } } QList<T> &operator=(std::initializer_list<T> args) @@ -332,6 +338,7 @@ public: const auto distance = std::distance(i1, i2); if (distance) { d = DataPointer(qsizetype(distance)); + Q_CHECK_PTR(d.data()); // appendIteratorRange can deal with contiguous iterators on its own, // this is an optimization for C++17 code. if constexpr (std::is_same_v<std::decay_t<InputIterator>, iterator> || @@ -352,8 +359,10 @@ public: QList(qsizetype size, Qt::Initialization) : d(size) { - if (size) + if (size) { + Q_CHECK_PTR(d.data()); d->appendUninitialized(size); + } } // compiler-generated special member functions are fine! @@ -823,7 +832,10 @@ void QList<T>::reserve(qsizetype asize) } } - DataPointer detached(qMax(asize, size())); + qsizetype newSize = qMax(asize, size()); + DataPointer detached(newSize); + if (newSize) + Q_CHECK_PTR(detached.data()); detached->copyAppend(d->begin(), d->end()); if (detached.d_ptr()) detached->setFlag(Data::CapacityReserved); @@ -839,6 +851,7 @@ inline void QList<T>::squeeze() // must allocate memory DataPointer detached(size()); if (size()) { + Q_CHECK_PTR(detached.data()); if (d.needsDetach()) detached->copyAppend(d.data(), d.data() + d.size); else |
