/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.

* Permission is hereby granted, free of charge, to any person obtaining a copy

* of this software and associated documentation files (the "Software"), to

* deal in the Software without restriction, including without limitation the

* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or

* sell copies of the Software, and to permit persons to whom the Software is

* furnished to do so, subject to the following conditions:

*

* The above copyright notice and this permission notice shall be included in

* all copies or substantial portions of the Software.

*

* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING

* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS

* IN THE SOFTWARE.

*/

/* We lean on the fact that POLL{IN,OUT,ERR,HUP} correspond with their

* EPOLL* counterparts. We use the POLL* variants in this file because that

* is what libuv uses elsewhere.

*/

#include "uv.h"

#include "internal.h"

#include <inttypes.h>

#include <stdatomic.h>

#include <stddef.h> /* offsetof */

#include <stdint.h>

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <assert.h>

#include <errno.h>

#include <fcntl.h>

#include <ifaddrs.h>

#include <net/ethernet.h>

#include <net/if.h>

#include <netpacket/packet.h>

#include <sys/epoll.h>

#include <sys/inotify.h>

#include <sys/mman.h>

#include <sys/param.h>

#include <sys/prctl.h>

#include <sys/socket.h>

#include <sys/stat.h>

#include <sys/syscall.h>

#include <sys/sysinfo.h>

#include <sys/sysmacros.h>

#include <sys/types.h>

#include <sys/utsname.h>

#include <time.h>

#include <unistd.h>

#ifndef __NR_io_uring_setup

# define __NR_io_uring_setup 425

#endif

#ifndef __NR_io_uring_enter

# define __NR_io_uring_enter 426

#endif

#ifndef __NR_io_uring_register

# define __NR_io_uring_register 427

#endif

#ifndef __NR_copy_file_range

# if defined(__x86_64__)

# define __NR_copy_file_range 326

# elif defined(__i386__)

# define __NR_copy_file_range 377

# elif defined(__s390__)

# define __NR_copy_file_range 375

# elif defined(__arm__)

# define __NR_copy_file_range 391

# elif defined(__aarch64__)

# define __NR_copy_file_range 285

# elif defined(__powerpc__)

# define __NR_copy_file_range 379

# elif defined(__arc__)

# define __NR_copy_file_range 285

# elif defined(__riscv)

# define __NR_copy_file_range 285

# endif

#endif /* __NR_copy_file_range */

#ifndef __NR_statx

# if defined(__x86_64__)

# define __NR_statx 332

# elif defined(__i386__)

# define __NR_statx 383

# elif defined(__aarch64__)

# define __NR_statx 397

# elif defined(__arm__)

# define __NR_statx 397

# elif defined(__ppc__)

# define __NR_statx 383

# elif defined(__s390__)

# define __NR_statx 379

# elif defined(__riscv)

# define __NR_statx 291

# endif

#endif /* __NR_statx */

#ifndef __NR_getrandom

# if defined(__x86_64__)

# define __NR_getrandom 318

# elif defined(__i386__)

# define __NR_getrandom 355

# elif defined(__aarch64__)

# define __NR_getrandom 384

# elif defined(__arm__)

# define __NR_getrandom 384

# elif defined(__ppc__)

# define __NR_getrandom 359

# elif defined(__s390__)

# define __NR_getrandom 349

# elif defined(__riscv)

# define __NR_getrandom 278

# endif

#endif /* __NR_getrandom */

enum {

UV__IORING_SETUP_SQPOLL = 2u,

UV__IORING_SETUP_NO_SQARRAY = 0x10000u,

};

enum {

UV__IORING_FEAT_SINGLE_MMAP = 1u,

UV__IORING_FEAT_NODROP = 2u,

UV__IORING_FEAT_RSRC_TAGS = 1024u, /* linux v5.13 */

};

enum {

UV__IORING_OP_READV = 1,

UV__IORING_OP_WRITEV = 2,

UV__IORING_OP_FSYNC = 3,

UV__IORING_OP_OPENAT = 18,

UV__IORING_OP_CLOSE = 19,

UV__IORING_OP_STATX = 21,

UV__IORING_OP_EPOLL_CTL = 29,

UV__IORING_OP_RENAMEAT = 35,

UV__IORING_OP_UNLINKAT = 36,

UV__IORING_OP_MKDIRAT = 37,

UV__IORING_OP_SYMLINKAT = 38,

UV__IORING_OP_LINKAT = 39,

UV__IORING_OP_FTRUNCATE = 55,

};

enum {

UV__IORING_ENTER_GETEVENTS = 1u,

UV__IORING_ENTER_SQ_WAKEUP = 2u,

};

enum {

UV__IORING_SQ_NEED_WAKEUP = 1u,

UV__IORING_SQ_CQ_OVERFLOW = 2u,

};

struct uv__io_cqring_offsets {

uint32_t head;

uint32_t tail;

uint32_t ring_mask;

uint32_t ring_entries;

uint32_t overflow;

uint32_t cqes;

uint64_t reserved0;

uint64_t reserved1;

};

STATIC_ASSERT(40 == sizeof(struct uv__io_cqring_offsets));

struct uv__io_sqring_offsets {

uint32_t head;

uint32_t tail;

uint32_t ring_mask;

uint32_t ring_entries;

uint32_t flags;

uint32_t dropped;

uint32_t array;

uint32_t reserved0;

uint64_t reserved1;

};

STATIC_ASSERT(40 == sizeof(struct uv__io_sqring_offsets));

struct uv__io_uring_cqe {

uint64_t user_data;

int32_t res;

uint32_t flags;

};

STATIC_ASSERT(16 == sizeof(struct uv__io_uring_cqe));

struct uv__io_uring_sqe {

uint8_t opcode;

uint8_t flags;

uint16_t ioprio;

int32_t fd;

union {

uint64_t off;

uint64_t addr2;

};

union {

uint64_t addr;

};

uint32_t len;

union {

uint32_t rw_flags;

uint32_t fsync_flags;

uint32_t open_flags;

uint32_t statx_flags;

};

uint64_t user_data;

union {

uint16_t buf_index;

uint64_t pad[3];

};

};

STATIC_ASSERT(64 == sizeof(struct uv__io_uring_sqe));

STATIC_ASSERT(0 == offsetof(struct uv__io_uring_sqe, opcode));

STATIC_ASSERT(1 == offsetof(struct uv__io_uring_sqe, flags));

STATIC_ASSERT(2 == offsetof(struct uv__io_uring_sqe, ioprio));

STATIC_ASSERT(4 == offsetof(struct uv__io_uring_sqe, fd));

STATIC_ASSERT(8 == offsetof(struct uv__io_uring_sqe, off));

STATIC_ASSERT(16 == offsetof(struct uv__io_uring_sqe, addr));

STATIC_ASSERT(24 == offsetof(struct uv__io_uring_sqe, len));

STATIC_ASSERT(28 == offsetof(struct uv__io_uring_sqe, rw_flags));

STATIC_ASSERT(32 == offsetof(struct uv__io_uring_sqe, user_data));

STATIC_ASSERT(40 == offsetof(struct uv__io_uring_sqe, buf_index));

struct uv__io_uring_params {

uint32_t sq_entries;

uint32_t cq_entries;

uint32_t flags;

uint32_t sq_thread_cpu;

uint32_t sq_thread_idle;

uint32_t features;

uint32_t reserved[4];

struct uv__io_sqring_offsets sq_off; /* 40 bytes */

struct uv__io_cqring_offsets cq_off; /* 40 bytes */

};

STATIC_ASSERT(40 + 40 + 40 == sizeof(struct uv__io_uring_params));

STATIC_ASSERT(40 == offsetof(struct uv__io_uring_params, sq_off));

STATIC_ASSERT(80 == offsetof(struct uv__io_uring_params, cq_off));

STATIC_ASSERT(EPOLL_CTL_ADD < 4);

STATIC_ASSERT(EPOLL_CTL_DEL < 4);

STATIC_ASSERT(EPOLL_CTL_MOD < 4);

struct watcher_list {

RB_ENTRY(watcher_list) entry;

struct uv__queue watchers;

int iterating;

char* path;

int wd;

};

struct watcher_root {

struct watcher_list* rbh_root;

};

static int uv__inotify_fork(uv_loop_t* loop, struct watcher_list* root);

static int compare_watchers(const struct watcher_list* a,

const struct watcher_list* b);

static void maybe_free_watcher_list(struct watcher_list* w,

uv_loop_t* loop);

static void uv__epoll_ctl_flush(int epollfd,

struct uv__iou* ctl,

struct epoll_event (*events)[256]);

static void uv__epoll_ctl_prep(int epollfd,

struct uv__iou* ctl,

struct epoll_event (*events)[256],

int op,

int fd,

struct epoll_event* e);

RB_GENERATE_STATIC(watcher_root, watcher_list, entry, compare_watchers)

static struct watcher_root* uv__inotify_watchers(uv_loop_t* loop) {

/* This cast works because watcher_root is a struct with a pointer as its

* sole member. Such type punning is unsafe in the presence of strict

* pointer aliasing (and is just plain nasty) but that is why libuv

* is compiled with -fno-strict-aliasing.

*/

return (struct watcher_root*) &loop->inotify_watchers;

}

unsigned uv__kernel_version(void) {

static _Atomic unsigned cached_version;

struct utsname u;

unsigned version;

unsigned major;

unsigned minor;

unsigned patch;

char v_sig[256];

char* needle;

version = atomic_load_explicit(&cached_version, memory_order_relaxed);

if (version != 0)

return version;

/* Check /proc/version_signature first as it's the way to get the mainline

* kernel version in Ubuntu. The format is:

* Ubuntu ubuntu_kernel_version mainline_kernel_version

* For example:

* Ubuntu 5.15.0-79.86-generic 5.15.111

*/

if (0 == uv__slurp("/proc/version_signature", v_sig, sizeof(v_sig)))

if (3 == sscanf(v_sig, "Ubuntu %*s %u.%u.%u", &major, &minor, &patch))

goto calculate_version;

if (-1 == uname(&u))

return 0;

/* In Debian we need to check `version` instead of `release` to extract the

* mainline kernel version. This is an example of how it looks like:

* #1 SMP Debian 5.10.46-4 (2021-08-03)

*/

needle = strstr(u.version, "Debian ");

if (needle != NULL)

if (3 == sscanf(needle, "Debian %u.%u.%u", &major, &minor, &patch))

goto calculate_version;

if (3 != sscanf(u.release, "%u.%u.%u", &major, &minor, &patch))

return 0;

/* Handle it when the process runs under the UNAME26 personality:

*

* - kernels >= 3.x identify as 2.6.40+x

* - kernels >= 4.x identify as 2.6.60+x

*

* UNAME26 is a poorly conceived hack that doesn't let us distinguish

* between 4.x kernels and 5.x/6.x kernels so we conservatively assume

* that 2.6.60+x means 4.x.

*

* Fun fact of the day: it's technically possible to observe the actual

* kernel version for a brief moment because uname() first copies out the

* real release string before overwriting it with the backcompat string.

*/

if (major == 2 && minor == 6) {

if (patch >= 60) {

major = 4;

minor = patch - 60;

patch = 0;

} else if (patch >= 40) {

major = 3;

minor = patch - 40;

patch = 0;

}

}

calculate_version:

version = major * 65536 + minor * 256 + patch;

atomic_store_explicit(&cached_version, version, memory_order_relaxed);

return version;

}

ssize_t

uv__fs_copy_file_range(int fd_in,

off_t* off_in,

int fd_out,

off_t* off_out,

size_t len,

unsigned int flags)

{

#ifdef __NR_copy_file_range

return syscall(__NR_copy_file_range,

fd_in,

off_in,

fd_out,

off_out,

len,

flags);

#else

return errno = ENOSYS, -1;

#endif

}

int uv__statx(int dirfd,

const char* path,

int flags,

unsigned int mask,

struct uv__statx* statxbuf) {

#if !defined(__NR_statx) || defined(__ANDROID_API__) && __ANDROID_API__ < 30

return errno = ENOSYS, -1;

#else

int rc;

rc = syscall(__NR_statx, dirfd, path, flags, mask, statxbuf);

if (rc >= 0)

uv__msan_unpoison(statxbuf, sizeof(*statxbuf));

return rc;

#endif

}

ssize_t uv__getrandom(void* buf, size_t buflen, unsigned flags) {

#if !defined(__NR_getrandom) || defined(__ANDROID_API__) && __ANDROID_API__ < 28

return errno = ENOSYS, -1;

#else

ssize_t rc;

rc = syscall(__NR_getrandom, buf, buflen, flags);

if (rc >= 0)

uv__msan_unpoison(buf, buflen);

return rc;

#endif

}

int uv__io_uring_setup(int entries, struct uv__io_uring_params* params) {

return syscall(__NR_io_uring_setup, entries, params);

}

int uv__io_uring_enter(int fd,

unsigned to_submit,

unsigned min_complete,

unsigned flags) {

/* io_uring_enter used to take a sigset_t but it's unused

* in newer kernels unless IORING_ENTER_EXT_ARG is set,

* in which case it takes a struct io_uring_getevents_arg.

*/

return syscall(__NR_io_uring_enter,

fd,

to_submit,

min_complete,

flags,

NULL,

0L);

}

int uv__io_uring_register(int fd, unsigned opcode, void* arg, unsigned nargs) {

return syscall(__NR_io_uring_register, fd, opcode, arg, nargs);

}

static int uv__use_io_uring(uint32_t flags) {

#if defined(__ANDROID_API__)

return 0; /* Possibly available but blocked by seccomp. */

#elif defined(__arm__) && __SIZEOF_POINTER__ == 4

/* See https://github.com/libuv/libuv/issues/4158. */

return 0; /* All 32 bits kernels appear buggy. */

#elif defined(__powerpc64__) || defined(__ppc64__)

/* See https://github.com/libuv/libuv/issues/4283. */

return 0; /* Random SIGSEGV in signal handler. */

#else

/* Ternary: unknown=0, yes=1, no=-1 */

static _Atomic int use_io_uring;

char* val;

int use;

#if defined(__hppa__)

/* io_uring first supported on parisc in 6.1, functional in .51

* https://lore.kernel.org/all/[email protected]/

*/

if (uv__kernel_version() < /*6.1.51*/0x060133)

return 0;

#endif

/* SQPOLL is all kinds of buggy but epoll batching should work fine. */

if (0 == (flags & UV__IORING_SETUP_SQPOLL))

return 1;

/* Older kernels have a bug where the sqpoll thread uses 100% CPU. */

if (uv__kernel_version() < /*5.10.186*/0x050ABA)

return 0;

use = atomic_load_explicit(&use_io_uring, memory_order_relaxed);

if (use == 0) {

val = getenv("UV_USE_IO_URING");

use = val != NULL && atoi(val) > 0 ? 1 : -1;

atomic_store_explicit(&use_io_uring, use, memory_order_relaxed);

}

return use > 0;

#endif

}

static void uv__iou_init(int epollfd,

struct uv__iou* iou,

uint32_t entries,

uint32_t flags) {

struct uv__io_uring_params params;

struct epoll_event e;

size_t cqlen;

size_t sqlen;

size_t maxlen;

size_t sqelen;

unsigned kernel_version;

uint32_t* sqarray;

uint32_t i;

char* sq;

char* sqe;

int ringfd;

int no_sqarray;

sq = MAP_FAILED;

sqe = MAP_FAILED;

if (!uv__use_io_uring(flags))

return;

kernel_version = uv__kernel_version();

no_sqarray =

UV__IORING_SETUP_NO_SQARRAY * (kernel_version >= /* 6.6 */0x060600);

/* SQPOLL required CAP_SYS_NICE until linux v5.12 relaxed that requirement.

* Mostly academic because we check for a v5.13 kernel afterwards anyway.

*/

memset(&params, 0, sizeof(params));

params.flags = flags | no_sqarray;

if (flags & UV__IORING_SETUP_SQPOLL)

params.sq_thread_idle = 10; /* milliseconds */

/* Kernel returns a file descriptor with O_CLOEXEC flag set. */

ringfd = uv__io_uring_setup(entries, &params);

if (ringfd == -1)

return;

/* IORING_FEAT_RSRC_TAGS is used to detect linux v5.13 but what we're

* actually detecting is whether IORING_OP_STATX works with SQPOLL.

*/

if (!(params.features & UV__IORING_FEAT_RSRC_TAGS))

goto fail;

/* Implied by IORING_FEAT_RSRC_TAGS but checked explicitly anyway. */

if (!(params.features & UV__IORING_FEAT_SINGLE_MMAP))

goto fail;

/* Implied by IORING_FEAT_RSRC_TAGS but checked explicitly anyway. */

if (!(params.features & UV__IORING_FEAT_NODROP))

goto fail;

sqlen = params.sq_off.array + params.sq_entries * sizeof(uint32_t);

cqlen =

params.cq_off.cqes + params.cq_entries * sizeof(struct uv__io_uring_cqe);

maxlen = sqlen < cqlen ? cqlen : sqlen;

sqelen = params.sq_entries * sizeof(struct uv__io_uring_sqe);

sq = mmap(0,

maxlen,

PROT_READ | PROT_WRITE,

MAP_SHARED | MAP_POPULATE,

ringfd,

0); /* IORING_OFF_SQ_RING */

sqe = mmap(0,

sqelen,

PROT_READ | PROT_WRITE,

MAP_SHARED | MAP_POPULATE,

ringfd,

0x10000000ull); /* IORING_OFF_SQES */

if (sq == MAP_FAILED || sqe == MAP_FAILED)

goto fail;

if (flags & UV__IORING_SETUP_SQPOLL) {

/* Only interested in completion events. To get notified when

* the kernel pulls items from the submission ring, add POLLOUT.

*/

memset(&e, 0, sizeof(e));

e.events = POLLIN;

e.data.fd = ringfd;

if (epoll_ctl(epollfd, EPOLL_CTL_ADD, ringfd, &e))

goto fail;

}

iou->sqhead = (uint32_t*) (sq + params.sq_off.head);

iou->sqtail = (uint32_t*) (sq + params.sq_off.tail);

iou->sqmask = *(uint32_t*) (sq + params.sq_off.ring_mask);

iou->sqflags = (uint32_t*) (sq + params.sq_off.flags);

iou->cqhead = (uint32_t*) (sq + params.cq_off.head);

iou->cqtail = (uint32_t*) (sq + params.cq_off.tail);

iou->cqmask = *(uint32_t*) (sq + params.cq_off.ring_mask);

iou->sq = sq;

iou->cqe = sq + params.cq_off.cqes;

iou->sqe = sqe;

iou->sqlen = sqlen;

iou->cqlen = cqlen;

iou->maxlen = maxlen;

iou->sqelen = sqelen;

iou->ringfd = ringfd;

iou->in_flight = 0;

if (no_sqarray)

return;

sqarray = (uint32_t*) (sq + params.sq_off.array);

for (i = 0; i <= iou->sqmask; i++)

sqarray[i] = i; /* Slot -> sqe identity mapping. */

return;

fail:

if (sq != MAP_FAILED)

munmap(sq, maxlen);

if (sqe != MAP_FAILED)

munmap(sqe, sqelen);

uv__close(ringfd);

}

static void uv__iou_delete(struct uv__iou* iou) {

if (iou->ringfd > -1) {

munmap(iou->sq, iou->maxlen);

munmap(iou->sqe, iou->sqelen);

uv__close(iou->ringfd);

iou->ringfd = -1;

}

}

int uv__platform_loop_init(uv_loop_t* loop) {

uv__loop_internal_fields_t* lfields;

lfields = uv__get_internal_fields(loop);

lfields->ctl.ringfd = -1;

lfields->iou.ringfd = -2; /* "uninitialized" */

loop->inotify_watchers = NULL;

loop->inotify_fd = -1;

loop->backend_fd = epoll_create1(O_CLOEXEC);

if (loop->backend_fd == -1)

return UV__ERR(errno);

uv__iou_init(loop->backend_fd, &lfields->ctl, 256, 0);

return 0;

}

int uv__io_fork(uv_loop_t* loop) {

int err;

struct watcher_list* root;

root = uv__inotify_watchers(loop)->rbh_root;

uv__close(loop->backend_fd);

loop->backend_fd = -1;

/* TODO(bnoordhuis) Loses items from the submission and completion rings. */

uv__platform_loop_delete(loop);

err = uv__platform_loop_init(loop);

if (err)

return err;

return uv__inotify_fork(loop, root);

}

void uv__platform_loop_delete(uv_loop_t* loop) {

uv__loop_internal_fields_t* lfields;

lfields = uv__get_internal_fields(loop);

uv__iou_delete(&lfields->ctl);

uv__iou_delete(&lfields->iou);

if (loop->inotify_fd != -1) {

uv__io_stop(loop, &loop->inotify_read_watcher, POLLIN);

uv__close(loop->inotify_fd);

loop->inotify_fd = -1;

}

}

struct uv__invalidate {

struct epoll_event (*prep)[256];

struct epoll_event* events;

int nfds;

};

void uv__platform_invalidate_fd(uv_loop_t* loop, int fd) {

uv__loop_internal_fields_t* lfields;

struct uv__invalidate* inv;

struct epoll_event dummy;

int i;

lfields = uv__get_internal_fields(loop);

inv = lfields->inv;

/* Invalidate events with same file descriptor */

if (inv != NULL)

for (i = 0; i < inv->nfds; i++)

if (inv->events[i].data.fd == fd)

inv->events[i].data.fd = -1;

/* Remove the file descriptor from the epoll.

* This avoids a problem where the same file description remains open

* in another process, causing repeated junk epoll events.

*

* Perform EPOLL_CTL_DEL immediately instead of going through

* io_uring's submit queue, otherwise the file descriptor may

* be closed by the time the kernel starts the operation.

*

* We pass in a dummy epoll_event, to work around a bug in old kernels.

*

* Work around a bug in kernels 3.10 to 3.19 where passing a struct that

* has the EPOLLWAKEUP flag set generates spurious audit syslog warnings.

*/

memset(&dummy, 0, sizeof(dummy));

epoll_ctl(loop->backend_fd, EPOLL_CTL_DEL, fd, &dummy);

}

int uv__io_check_fd(uv_loop_t* loop, int fd) {

struct epoll_event e;

int rc;

memset(&e, 0, sizeof(e));

e.events = POLLIN;

e.data.fd = -1;

rc = 0;

if (epoll_ctl(loop->backend_fd, EPOLL_CTL_ADD, fd, &e))

if (errno != EEXIST)

rc = UV__ERR(errno);

if (rc == 0)

if (epoll_ctl(loop->backend_fd, EPOLL_CTL_DEL, fd, &e))

abort();

return rc;

}

/* Caller must initialize SQE and call uv__iou_submit(). */

static struct uv__io_uring_sqe* uv__iou_get_sqe(struct uv__iou* iou,

uv_loop_t* loop,

uv_fs_t* req) {

struct uv__io_uring_sqe* sqe;

uint32_t head;

uint32_t tail;

uint32_t mask;

uint32_t slot;

/* Lazily create the ring. State machine: -2 means uninitialized, -1 means

* initialization failed. Anything else is a valid ring file descriptor.

*/

if (iou->ringfd == -2) {

/* By default, the SQPOLL is not created. Enable only if the loop is

* configured with UV_LOOP_USE_IO_URING_SQPOLL and the UV_USE_IO_URING

* environment variable is unset or a positive number.

*/

if (loop->flags & UV_LOOP_ENABLE_IO_URING_SQPOLL)

if (uv__use_io_uring(UV__IORING_SETUP_SQPOLL))

uv__iou_init(loop->backend_fd, iou, 64, UV__IORING_SETUP_SQPOLL);

if (iou->ringfd == -2)

iou->ringfd = -1; /* "failed" */

}

if (iou->ringfd == -1)

return NULL;

head = atomic_load_explicit((_Atomic uint32_t*) iou->sqhead,

memory_order_acquire);

tail = *iou->sqtail;

mask = iou->sqmask;

if ((head & mask) == ((tail + 1) & mask))

return NULL; /* No room in ring buffer. TODO(bnoordhuis) maybe flush it? */

slot = tail & mask;

sqe = iou->sqe;

sqe = &sqe[slot];

memset(sqe, 0, sizeof(*sqe));

sqe->user_data = (uintptr_t) req;

/* Pacify uv_cancel(). */

req->work_req.loop = loop;

req->work_req.work = NULL;

req->work_req.done = NULL;

uv__queue_init(&req->work_req.wq);

uv__req_register(loop);

iou->in_flight++;

return sqe;

}

static void uv__iou_submit(struct uv__iou* iou) {

uint32_t flags;

atomic_store_explicit((_Atomic uint32_t*) iou->sqtail,

*iou->sqtail + 1,

memory_order_release);

flags = atomic_load_explicit((_Atomic uint32_t*) iou->sqflags,

memory_order_acquire);

if (flags & UV__IORING_SQ_NEED_WAKEUP)

if (uv__io_uring_enter(iou->ringfd, 0, 0, UV__IORING_ENTER_SQ_WAKEUP))

if (errno != EOWNERDEAD) /* Kernel bug. Harmless, ignore. */

perror("libuv: io_uring_enter(wakeup)"); /* Can't happen. */

}

int uv__iou_fs_close(uv_loop_t* loop, uv_fs_t* req) {

struct uv__io_uring_sqe* sqe;

struct uv__iou* iou;

int kv;

kv = uv__kernel_version();

/* Work around a poorly understood bug in older kernels where closing a file

* descriptor pointing to /foo/bar results in ETXTBSY errors when trying to

* execve("/foo/bar") later on. The bug seems to have been fixed somewhere

* between 5.15.85 and 5.15.90. I couldn't pinpoint the responsible commit

* but good candidates are the several data race fixes. Interestingly, it

* seems to manifest only when running under Docker so the possibility of

* a Docker bug can't be completely ruled out either. Yay, computers.

* Also, disable on non-longterm versions between 5.16.0 (non-longterm) and

* 6.1.0 (longterm). Starting with longterm 6.1.x, the issue seems to be

* solved.

*/

if (kv < /* 5.15.90 */ 0x050F5A)

return 0;

if (kv >= /* 5.16.0 */ 0x050A00 && kv < /* 6.1.0 */ 0x060100)

return 0;

iou = &uv__get_internal_fields(loop)->iou;

sqe = uv__iou_get_sqe(iou, loop, req);

if (sqe == NULL)

return 0;

sqe->fd = req->file;

sqe->opcode = UV__IORING_OP_CLOSE;

uv__iou_submit(iou);

return 1;

}

int uv__iou_fs_ftruncate(uv_loop_t* loop, uv_fs_t* req) {

struct uv__io_uring_sqe* sqe;

struct uv__iou* iou;

if (uv__kernel_version() < /* 6.9 */0x060900)

return 0;

iou = &uv__get_internal_fields(loop)->iou;

sqe = uv__iou_get_sqe(iou, loop, req);

if (sqe == NULL)

return 0;

sqe->fd = req->file;

sqe->off = req->off;

sqe->opcode = UV__IORING_OP_FTRUNCATE;

uv__iou_submit(iou);

return 1;

}

int uv__iou_fs_fsync_or_fdatasync(uv_loop_t* loop,

uv_fs_t* req,

uint32_t fsync_flags) {

struct uv__io_uring_sqe* sqe;

struct uv__iou* iou;

iou = &uv__get_internal_fields(loop)->iou;

sqe = uv__iou_get_sqe(iou, loop, req);

if (sqe == NULL)

return 0;

/* Little known fact: setting seq->off and seq->len turns

* it into an asynchronous sync_file_range() operation.

*/

sqe->fd = req->file;

sqe->fsync_flags = fsync_flags;

sqe->opcode = UV__IORING_OP_FSYNC;

uv__iou_submit(iou);

return 1;

}

int uv__iou_fs_link(uv_loop_t* loop, uv_fs_t* req) {

struct uv__io_uring_sqe* sqe;

struct uv__iou* iou;

if (uv__kernel_version() < /* 5.15.0 */0x050F00)

return 0;

iou = &uv__get_internal_fields(loop)->iou;

sqe = uv__iou_get_sqe(iou, loop, req);

if (sqe == NULL)

return 0;

sqe->addr = (uintptr_t) req->path;

sqe->fd = AT_FDCWD;

sqe->addr2 = (uintptr_t) req->new_path;

sqe->len = AT_FDCWD;

sqe->opcode = UV__IORING_OP_LINKAT;

uv__iou_submit(iou);

return 1;

}

int uv__iou_fs_mkdir(uv_loop_t* loop, uv_fs_t* req) {

struct uv__io_uring_sqe* sqe;

struct uv__iou* iou;

if (uv__kernel_version() < /* 5.15.0 */0x050F00)

return 0;

iou = &uv__get_internal_fields(loop)->iou;

sqe = uv__iou_get_sqe(iou, loop, req);

if (sqe == NULL)

return 0;

sqe->addr = (uintptr_t) req->path;

sqe->fd = AT_FDCWD;

sqe->len = req->mode;

sqe->opcode = UV__IORING_OP_MKDIRAT;

uv__iou_submit(iou);

return 1;

}

int uv__iou_fs_open(uv_loop_t* loop, uv_fs_t* req) {

struct uv__io_uring_sqe* sqe;

struct uv__iou* iou;

iou = &uv__get_internal_fields(loop)->iou;

sqe = uv__iou_get_sqe(iou, loop, req);

if (sqe == NULL)

return 0;

sqe->addr = (uintptr_t) req->path;

sqe->fd = AT_FDCWD;

sqe->len = req->mode;

sqe->opcode = UV__IORING_OP_OPENAT;

sqe->open_flags = req->flags | O_CLOEXEC;

uv__iou_submit(iou);

return 1;

}

int uv__iou_fs_rename(uv_loop_t* loop, uv_fs_t* req) {

struct uv__io_uring_sqe* sqe;

struct uv__iou* iou;

iou = &uv__get_internal_fields(loop)->iou;

sqe = uv__iou_get_sqe(iou, loop, req);

if (sqe == NULL)

return 0;

sqe->addr = (uintptr_t) req->path;

sqe->fd = AT_FDCWD;

sqe->addr2 = (uintptr_t) req->new_path;

sqe->len = AT_FDCWD;

sqe->opcode = UV__IORING_OP_RENAMEAT;

uv__iou_submit(iou);

return 1;

}