Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 1 addition & 17 deletions .github/workflows/wasi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ jobs:
run: |
cargo test --target ${{ matrix.job.target }} --no-default-features --features fs -p uucore fs::tests
- name: Run integration tests via wasmtime
if: matrix.job.target == 'wasm32-wasip1'
env:
RUSTFLAGS: ${{ matrix.job.rust-flags }}
run: |
Expand All @@ -75,26 +74,11 @@ jobs:
# The binary above is built without default features, so it has no
# caret diagnostics; the host test crate does have them, hence the
# not(wasi_runner) in the cfg guarding every `mod diagnostics`.
# TODO: add integration tests for these tools as WASI support is extended:
# csplit ls rm seq sort
# nproc is not part of the default feature set, so it needs to be
# requested explicitly for the host test crate to compile its tests.
UUTESTS_BINARY_PATH="$(pwd)/target/${{ matrix.job.target }}/debug/coreutils.wasm" \
UUTESTS_WASM_RUNNER=wasmtime \
cargo test --test tests --features nproc -- \
test_arch:: test_b2sum:: test_base32:: test_base64:: test_basenc:: \
test_basename:: test_cat:: test_cksum:: test_comm:: test_cp:: test_cut:: test_date:: \
test_dir:: test_dircolors:: test_dirname:: test_echo:: test_expand:: test_expr:: \
test_factor:: test_false:: test_fmt:: test_fold:: test_head:: test_join:: test_link:: test_ln:: \
test_ls::test_ls_a_dotdot_no_error_on_wasi \
test_ls::test_ls_al_no_capabilities_insufficient_on_wasi \
test_md5sum:: test_mkdir:: test_mktemp:: test_mv:: test_nl:: test_nproc:: test_numfmt:: test_od:: \
test_paste:: test_pathchk:: test_pr:: test_printenv:: test_printf:: test_ptx:: \
test_pwd:: test_readlink:: test_realpath:: test_rmdir:: test_sha1sum:: test_sha224sum:: test_sha256sum:: \
test_sha384sum:: test_sha512sum:: test_shred:: test_shuf:: test_sleep:: \
test_sort::sort_multiple test_sort::test_check test_split:: test_sum:: test_tail:: test_tee:: \
test_test:: test_touch:: test_tr:: test_true:: test_truncate:: test_tsort:: test_uname:: \
test_uniq:: test_unexpand:: test_unlink:: test_vdir:: test_wc:: test_yes::
cargo test --test tests --features nproc

# WASI tests on non-Linux hosts. These catch host-specific wasmtime differences
test_wasi_other_host:
Expand Down
6 changes: 6 additions & 0 deletions .vscode/cspell.dictionaries/workspace.wordlist.txt
Original file line number Diff line number Diff line change
Expand Up @@ -124,11 +124,13 @@ ENODATA
ENOENT
ENOSPC
ENOSYS
ENOTDIR
ENOTEMPTY
EOPNOTSUPP
EPERM
EPIPE
EROFS
ETXTBSY

# * vars/fcntl
F_GETFL
Expand Down Expand Up @@ -183,8 +185,10 @@ addrlen
blocksize
canonname
chroot
cmdline
dlsym
execvp
fchmod
fdatasync
freeaddrinfo
getaddrinfo
Expand All @@ -204,6 +208,7 @@ inodes
isatty
lchown
pathlen
reflink
setgid
setgroups
settime
Expand All @@ -218,6 +223,7 @@ strerror
strlen
syncfs
umask
unflushed
waitpid

# * vars/nix
Expand Down
4 changes: 0 additions & 4 deletions docs/src/wasi-test-gaps.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,6 @@ Limitations that affect only `wasm32-wasip2` use `#[uutests::wasip2_ignore(Reaso

To find all annotated tests: `grep -rnE 'wasi_ignore|wasip2_ignore' tests/`. For the full, authoritative list of reason keys and their text, see `tests/uutests_procs/src/lib.rs`.

## Tools not yet covered by integration tests

arch, b2sum, cat, cp, csplit, date, dir, dircolors, fmt, join, ls, md5sum, mkdir, mv, nproc, pathchk, pr, printenv, ptx, pwd, readlink, realpath, rm, rmdir, seq, sha1sum, sha224sum, sha256sum, sha384sum, sha512sum, shred, sleep, sort, split, tail, touch, tsort, uname, uniq, vdir, yes

## WASI sandbox: host paths not visible

The WASI guest only sees directories explicitly mapped with `--dir`. Host paths like `/proc`, `/sys`, and `/dev` are not accessible. Affected tests include those that read `/proc/version`, `/proc/modules`, `/proc/cpuinfo`, `/proc/self/mem`, `/sys/kernel/profiling`, `/dev/null`, `/dev/zero`, `/dev/full`, and tests that rely on anonymous pipes or Linux-specific I/O error paths.
Expand Down
5 changes: 5 additions & 0 deletions src/bin/coreutils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,12 @@ fn main() -> ExitCode {
// #[cfg(not(feature = "only_english"))]
validation::setup_localization_or_exit(util);
let code = uumain(vec![util_os].into_iter().chain(args));
// WASI preview 2 maps every nonzero `ExitCode` returned from main to 1,
// so the code must go through exit to reach the host.
#[cfg(all(target_os = "wasi", target_env = "p2"))]
exit(code);
// ExitCode only holds a u8; larger codes (Windows) go through exit.
#[cfg(not(all(target_os = "wasi", target_env = "p2")))]
return u8::try_from(code).map_or_else(|_| exit(code), ExitCode::from);
}
// GNU coreutils --help string shows help for coreutils
Expand Down
12 changes: 11 additions & 1 deletion src/uu/cat/src/cat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,14 @@ fn cat_path(path: &OsString, options: &OutputOptions, state: &mut OutputState) -
#[cfg(unix)]
InputType::Socket => Err(CatError::NoSuchDeviceOrAddress),
_ => {
let file = File::open(path)?;
let file = File::open(path).map_err(|e| match e.kind() {
ErrorKind::IsADirectory => CatError::IsDirectory,
#[cfg(target_os = "wasi")]
_ if e.raw_os_error() == Some(rustix::io::Errno::LOOP.raw_os_error()) => {
CatError::TooManySymlinks
}
_ => CatError::from(e),
})?;
if !is_safe_overwrite(&file, &io::stdout()) {
return Err(CatError::OutputIsInput);
}
Expand Down Expand Up @@ -456,6 +463,9 @@ fn get_input_type(path: &OsString) -> CatResult<InputType> {
let ft = match metadata(path) {
Ok(md) => md.file_type(),
Err(e) => {
if e.kind() == ErrorKind::IsADirectory {
return Err(CatError::IsDirectory);
}
if let Some(raw_error) = e.raw_os_error() {
#[cfg(any(unix, target_os = "wasi"))]
let too_many_symlink_code = rustix::io::Errno::LOOP.raw_os_error();
Expand Down
5 changes: 4 additions & 1 deletion src/uu/csplit/src/csplit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,10 @@ where
T: BufRead,
{
let enumerated_input_lines = LinesWithNewlines::new(input)
.map(|line| line.map_err_context(|| translate!("csplit-read-error")))
.map(|line| {
line.map_err(uucore::error::wasi_normalize_read_error)
.map_err_context(|| translate!("csplit-read-error"))
})
.enumerate();
let mut input_iter = InputSplitter::new(enumerated_input_lines);
let mut split_writer = SplitWriter::new(options);
Expand Down
1 change: 1 addition & 0 deletions src/uu/factor/src/factor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
}
Err(e) => {
set_exit_code(1);
let e = uucore::error::wasi_normalize_read_error(e);
show_error!(
"{}",
translate!("factor-error-reading-input", "error" => strip_errno(&e))
Expand Down
1 change: 1 addition & 0 deletions src/uu/od/src/multifile_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,7 @@ impl io::Read for MultifileReader<'_> {
Ok(0) => break,
Ok(n) => n,
Err(e) => {
let e = uucore::error::wasi_normalize_read_error(e);
show_error!(
"{}: {}",
self.file_name
Expand Down
2 changes: 2 additions & 0 deletions src/uu/paste/src/paste.rs
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,7 @@ impl InputSource {
let us = match self {
Self::File(bu, name) => bu
.read(buf)
.map_err(uucore::error::wasi_normalize_read_error)
.map_err_context(|| name.maybe_quote().to_string())?,
Self::StandardInput(rc) => rc
.try_borrow()
Expand All @@ -404,6 +405,7 @@ impl InputSource {
let us = match self {
Self::File(bu, name) => bu
.read_until(byte, buf)
.map_err(uucore::error::wasi_normalize_read_error)
.map_err_context(|| name.maybe_quote().to_string())?,
Self::StandardInput(rc) => rc
.try_borrow()
Expand Down
4 changes: 2 additions & 2 deletions src/uu/sort/src/sort.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2178,7 +2178,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
for (line_num, line_res) in buf_reader.split(b'\0').enumerate() {
let line = line_res.map_err(|error| SortError::ReadFailed {
path: files0_from.clone(),
error,
error: uucore::error::wasi_normalize_read_error(error),
})?;
if line.as_slice() == STDIN_FILE.as_bytes() {
return Err(SortError::MinusInStdIn.into());
Expand Down Expand Up @@ -3360,7 +3360,7 @@ fn open_with_open_failed_error(path: impl AsRef<OsStr>) -> UResult<Box<dyn Read
Ok(f) => Ok(Box::new(f) as Box<dyn Read + Send>),
Err(error) => Err(SortError::OpenFailed {
path: path.to_owned(),
error,
error: uucore::error::wasi_normalize_open_error(error),
}
.into()),
}
Expand Down
5 changes: 4 additions & 1 deletion src/uu/tac/src/tac.rs
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,10 @@ fn tac(filenames: &[OsString], before: bool, regex: bool, separator: &OsStr) ->
&buf
}
Err(e) => {
show!(TacError::ReadError(filename.clone(), e));
show!(TacError::ReadError(
filename.clone(),
uucore::error::wasi_normalize_read_error(e)
));
continue;
}
}
Expand Down
60 changes: 59 additions & 1 deletion src/uu/tail/src/tail.rs
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,9 @@ fn tail_stdin(
);
}
} else {
#[cfg(target_os = "wasi")]
let mut reader = BufReader::new(NormalizeReadErrors(stdin().take(limit)));
#[cfg(not(target_os = "wasi"))]
let mut reader = BufReader::new(stdin().take(limit));
unbounded_tail(&mut reader, settings)?;
}
Expand Down Expand Up @@ -572,7 +575,58 @@ fn bounded_tail(file: &mut File, settings: &Settings, end: Option<u64>) -> UResu
Ok(())
}

/// Reader that reports a directory read on WASI as `ErrorKind::IsADirectory`.
#[cfg(target_os = "wasi")]
struct NormalizeReadErrors<R>(R);

#[cfg(target_os = "wasi")]
impl<R: Read> Read for NormalizeReadErrors<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
self.0
.read(buf)
.map_err(uucore::error::wasi_normalize_read_error)
}
}

/// Writer that ends the process silently once the reader of stdout has gone away.
///
/// Unix signals this case with SIGPIPE. WASI has no signals, so the process exits
/// with the status that a SIGPIPE death reports.
#[cfg(target_os = "wasi")]
struct ExitOnClosedPipe<W>(W);

#[cfg(target_os = "wasi")]
impl<W: Write> ExitOnClosedPipe<W> {
fn check<T>(result: io::Result<T>) -> io::Result<T> {
// Preview 1 reports a closed pipe as a generic EIO instead of BrokenPipe.
#[cfg(target_env = "p1")]
const EIO: Option<i32> = Some(29);
#[cfg(not(target_env = "p1"))]
const EIO: Option<i32> = None;
if let Err(err) = &result
&& (err.kind() == ErrorKind::BrokenPipe || (EIO.is_some() && err.raw_os_error() == EIO))
{
std::process::exit(13);
}
result
}
}

#[cfg(target_os = "wasi")]
impl<W: Write> Write for ExitOnClosedPipe<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
Self::check(self.0.write(buf))
}

fn flush(&mut self) -> io::Result<()> {
Self::check(self.0.flush())
}
}

fn unbounded_tail<T: Read>(reader: &mut BufReader<T>, settings: &Settings) -> UResult<()> {
#[cfg(target_os = "wasi")]
let mut writer = BufWriter::new(ExitOnClosedPipe(stdout().lock()));
#[cfg(not(target_os = "wasi"))]
let mut writer = BufWriter::new(stdout().lock());
match &settings.mode {
FilterMode::Lines(Signum::Negative(count), sep) => {
Expand Down Expand Up @@ -657,7 +711,11 @@ fn print_target_section<
limit: Option<u64>,
) -> UResult<()> {
let stdout = stdout();
let mut stdout = stdout.lock();
let stdout = stdout.lock();
#[cfg(target_os = "wasi")]
let mut stdout = ExitOnClosedPipe(stdout);
#[cfg(not(target_os = "wasi"))]
let mut stdout = stdout;
if let Some(limit) = limit {
#[cfg(any(target_os = "linux", target_os = "android"))]
uucore::pipes::send_n_bytes(file, &mut stdout, limit)?;
Expand Down
1 change: 1 addition & 0 deletions src/uu/tee/src/tee.rs
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@ impl MultiWriter {
Ok(slice) => self.write_flush(slice)?,
Err(e) if e.kind() == ErrorKind::Interrupted => {}
Err(e) => {
let e = uucore::error::wasi_normalize_read_error(e);
show_error!(
"{}",
translate!("tee-error-stdin", "error" => strip_errno(&e))
Expand Down
5 changes: 4 additions & 1 deletion src/uu/tr/src/operation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -993,7 +993,10 @@ where
Ok(0) => break, // EOF reached
Ok(len) => len,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e.map_err_context(|| translate!("tr-error-read-error"))),
Err(e) => {
let e = uucore::error::wasi_normalize_read_error(e);
return Err(e.map_err_context(|| translate!("tr-error-read-error")));
}
};

// Process the buffer and collect translated chars to output
Expand Down
5 changes: 4 additions & 1 deletion src/uu/tr/src/simd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,10 @@ where
Ok(0) => break,
Ok(len) => len,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e.map_err_context(|| translate!("tr-error-read-error"))),
Err(e) => {
let e = uucore::error::wasi_normalize_read_error(e);
return Err(e.map_err_context(|| translate!("tr-error-read-error")));
}
};

output_buf.clear();
Expand Down
7 changes: 5 additions & 2 deletions src/uu/wc/src/wc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -770,7 +770,10 @@ fn word_count_from_input(input: &Input<'_>, settings: &Settings) -> CountResult
};
match maybe_err {
None => CountResult::Success(total),
Some(err) => CountResult::Interrupted(total, err),
Some(err) => {
let err = uucore::error::wasi_normalize_open_error(err);
CountResult::Interrupted(total, err)
}
}
}

Expand Down Expand Up @@ -882,7 +885,7 @@ fn files0_iter<'a>(
Ok(Input::Path(PathBuf::from(s).into()))
}
}
Err(e) => Err(e.map_err_context(
Err(e) => Err(uucore::error::wasi_normalize_read_error(e).map_err_context(
|| translate!("wc-error-read-error", "path" => locale_aware_shell_escape(&err_path)),
) as Box<dyn UError>),
}),
Expand Down
18 changes: 18 additions & 0 deletions src/uucore/src/lib/features/fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,10 @@ impl FileInformation {
pub fn from_path(path: impl AsRef<Path>, dereference: bool) -> IOResult<Self> {
#[cfg(any(unix, target_os = "wasi"))]
{
#[cfg(all(target_os = "wasi", target_env = "p2"))]
if dereference && let Some(info) = Self::from_symlink_target_p2(path.as_ref()) {
return Ok(info);
}
let stat = if dereference {
rustix::fs::stat(path.as_ref())
} else {
Expand All @@ -112,6 +116,20 @@ impl FileInformation {
}
}

/// Returns the identity of a symlink target on WASI preview 2.
///
/// Path-based `stat` on a symlink reports the inode of the link itself, so two paths to
/// the same file compare as different. Statting an open descriptor reports the target.
#[cfg(all(target_os = "wasi", target_env = "p2"))]
fn from_symlink_target_p2(path: &Path) -> Option<Self> {
let link = rustix::fs::lstat(path).ok()?;
if rustix::fs::FileType::from_raw_mode(link.st_mode) != rustix::fs::FileType::Symlink {
return None;
}
let file = fs::File::open(path).ok()?;
Self::from_file(&file).ok()
}

pub fn file_size(&self) -> u64 {
#[cfg(any(unix, target_os = "wasi"))]
{
Expand Down
Loading
Loading