From 3bfa125b2ea8175947360322f1429a0205be6c76 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Mon, 28 Aug 2023 20:17:04 -0400 Subject: [PATCH] ci: replace mips with powerpc64, aarch64 and s390x We drop our MIPS target because it no longer works.[1] We were previously using it as a means of testing ripgrep in a big endian environment. So to achieve that without MIPS, we test on powerpc64 and s390x. (No particular reason to do both, but why not.) We also add aarch64 as a proxy for at least ensuring everything works for the same architecture as Apple silicon. It's not a guarantee that everything works, but it seems better than nothing until we can actually test Apple silicon in CI. [1]: https://github.com/rust-lang/regex/commit/c788378d6fe407f4774df98a78436cea5d98525b --- .github/workflows/ci.yml | 89 +++++++++++-------- Cross.toml | 12 +-- .../Dockerfile | 2 +- .../build | 2 +- .../Dockerfile | 2 +- .../build | 2 +- ci/docker/s390x-unknown-linux-gnu/Dockerfile | 4 + ci/docker/s390x-unknown-linux-gnu/build | 5 ++ tests/feature.rs | 3 + tests/misc.rs | 6 ++ tests/util.rs | 46 +++++++--- 11 files changed, 114 insertions(+), 59 deletions(-) rename ci/docker/{arm-unknown-linux-gnueabihf => aarch64-unknown-linux-gnu}/Dockerfile (56%) rename ci/docker/{arm-unknown-linux-gnueabihf => aarch64-unknown-linux-gnu}/build (51%) rename ci/docker/{mips64-unknown-linux-gnuabi64 => powerpc64-unknown-linux-gnu}/Dockerfile (55%) rename ci/docker/{mips64-unknown-linux-gnuabi64 => powerpc64-unknown-linux-gnu}/build (51%) create mode 100644 ci/docker/s390x-unknown-linux-gnu/Dockerfile create mode 100755 ci/docker/s390x-unknown-linux-gnu/build diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c1e563d..bf8c2004 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,27 @@ on: - master schedule: - cron: '00 01 * * *' + +# The section is needed to drop write-all permissions that are granted on +# `schedule` event. By specifying any permission explicitly all others are set +# to none. By using the principle of least privilege the damage a compromised +# workflow can do (because of an injection or compromised third party tool or +# action) is restricted. Currently the worklow doesn't need any additional +# permission except for pulling the code. Adding labels to issues, commenting +# on pull-requests, etc. may need additional permissions: +# +# Syntax for this section: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions +# +# Reference for how to assign permissions on a job-by-job basis: +# https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs +# +# Reference for available permissions that we can enable if needed: +# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token +permissions: + # to fetch code (actions/checkout) + contents: read + jobs: test: name: test @@ -14,32 +35,21 @@ jobs: # systems. CARGO: cargo # When CARGO is set to CROSS, this is set to `--target matrix.target`. + # Note that we only use cross on Linux, so setting a target on a + # different OS will just use normal cargo. TARGET_FLAGS: # When CARGO is set to CROSS, TARGET_DIR includes matrix.target. TARGET_DIR: ./target + # Bump this as appropriate. We pin to a version to make sure CI + # continues to work as cross releases in the past have broken things + # in subtle ways. + CROSS_VERSION: v0.2.5 # Emit backtraces on panics. RUST_BACKTRACE: 1 runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - build: - # We test ripgrep on a pinned version of Rust, along with the moving - # targets of 'stable' and 'beta' for good measure. - - pinned - - stable - - beta - # Our release builds are generated by a nightly compiler to take - # advantage of the latest optimizations/compile time improvements. So - # we test all of them here. (We don't do mips releases, but test on - # mips for big-endian coverage.) - - nightly - - nightly-musl - - nightly-32 - - nightly-mips - - nightly-arm - - macos - - win-msvc - - win-gnu include: - build: pinned os: ubuntu-latest @@ -53,27 +63,26 @@ jobs: - build: nightly os: ubuntu-latest rust: nightly - - build: nightly-musl + - build: stable-musl os: ubuntu-latest - rust: nightly + rust: stable target: x86_64-unknown-linux-musl - - build: nightly-32 + - build: stable-x86 os: ubuntu-latest - rust: nightly + rust: stable target: i686-unknown-linux-gnu - - build: nightly-mips + - build: stable-aarch64 os: ubuntu-latest - rust: nightly - target: mips64-unknown-linux-gnuabi64 - - build: nightly-arm + rust: stable + target: aarch64-unknown-linux-gnu + - build: stable-powerpc64 os: ubuntu-latest - rust: nightly - # For stripping release binaries: - # docker run --rm -v $PWD/target:/target:Z \ - # rustembedded/cross:arm-unknown-linux-gnueabihf \ - # arm-linux-gnueabihf-strip \ - # /target/arm-unknown-linux-gnueabihf/debug/rg - target: arm-unknown-linux-gnueabihf + rust: stable + target: powerpc64-unknown-linux-gnu + - build: stable-s390x + os: ubuntu-latest + rust: stable + target: s390x-unknown-linux-gnu - build: macos os: macos-latest rust: nightly @@ -103,9 +112,17 @@ jobs: toolchain: ${{ matrix.rust }} - name: Use Cross - if: matrix.target != '' + if: matrix.os == 'ubuntu-latest' && matrix.target != '' run: | - cargo install cross + # In the past, new releases of 'cross' have broken CI. So for now, we + # pin it. We also use their pre-compiled binary releases because cross + # has over 100 dependencies and takes a bit to compile. + dir="$RUNNER_TEMP/cross-download" + mkdir "$dir" + echo "$dir" >> $GITHUB_PATH + cd "$dir" + curl -LO "https://github.com/cross-rs/cross/releases/download/$CROSS_VERSION/cross-x86_64-unknown-linux-musl.tar.gz" + tar xf cross-x86_64-unknown-linux-musl.tar.gz echo "CARGO=cross" >> $GITHUB_ENV echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV @@ -177,7 +194,6 @@ jobs: run: ci/test-complete rustfmt: - name: rustfmt runs-on: ubuntu-latest steps: - name: Checkout repository @@ -191,7 +207,6 @@ jobs: run: cargo fmt --all --check docs: - name: Docs runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/Cross.toml b/Cross.toml index 220e304b..b334e5db 100644 --- a/Cross.toml +++ b/Cross.toml @@ -4,9 +4,11 @@ image = "burntsushi/cross:x86_64-unknown-linux-musl" [target.i686-unknown-linux-gnu] image = "burntsushi/cross:i686-unknown-linux-gnu" -[target.mips64-unknown-linux-gnuabi64] -image = "burntsushi/cross:mips64-unknown-linux-gnuabi64" -build-std = true +[target.aarch64-unknown-linux-gnu] +image = "burntsushi/cross:aarch64-unknown-linux-gnu" -[target.arm-unknown-linux-gnueabihf] -image = "burntsushi/cross:arm-unknown-linux-gnueabihf" +[target.powerpc64-unknown-linux-gnu] +image = "burntsushi/cross:powerpc64-unknown-linux-gnu" + +[target.s390x-unknown-linux-gnu] +image = "burntsushi/cross:s390x-unknown-linux-gnu" diff --git a/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/ci/docker/aarch64-unknown-linux-gnu/Dockerfile similarity index 56% rename from ci/docker/arm-unknown-linux-gnueabihf/Dockerfile rename to ci/docker/aarch64-unknown-linux-gnu/Dockerfile index 59901cb5..0b40440f 100644 --- a/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile +++ b/ci/docker/aarch64-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM rustembedded/cross:arm-unknown-linux-gnueabihf +FROM rustembedded/cross:aarch64-unknown-linux-gnu COPY stage/ubuntu-install-packages / RUN /ubuntu-install-packages diff --git a/ci/docker/arm-unknown-linux-gnueabihf/build b/ci/docker/aarch64-unknown-linux-gnu/build similarity index 51% rename from ci/docker/arm-unknown-linux-gnueabihf/build rename to ci/docker/aarch64-unknown-linux-gnu/build index 107b8369..f32c3133 100755 --- a/ci/docker/arm-unknown-linux-gnueabihf/build +++ b/ci/docker/aarch64-unknown-linux-gnu/build @@ -2,4 +2,4 @@ mkdir -p stage cp ../../ubuntu-install-packages ./stage/ -docker build -t burntsushi/cross:arm-unknown-linux-gnueabihf . +docker build -t burntsushi/cross:aarch64-unknown-linux-gnu . diff --git a/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile b/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile similarity index 55% rename from ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile rename to ci/docker/powerpc64-unknown-linux-gnu/Dockerfile index 4b14616b..9489aa2c 100644 --- a/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile +++ b/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM rustembedded/cross:mips64-unknown-linux-gnuabi64 +FROM rustembedded/cross:powerpc64-unknown-linux-gnu COPY stage/ubuntu-install-packages / RUN /ubuntu-install-packages diff --git a/ci/docker/mips64-unknown-linux-gnuabi64/build b/ci/docker/powerpc64-unknown-linux-gnu/build similarity index 51% rename from ci/docker/mips64-unknown-linux-gnuabi64/build rename to ci/docker/powerpc64-unknown-linux-gnu/build index a15ca352..b25960c2 100755 --- a/ci/docker/mips64-unknown-linux-gnuabi64/build +++ b/ci/docker/powerpc64-unknown-linux-gnu/build @@ -2,4 +2,4 @@ mkdir -p stage cp ../../ubuntu-install-packages ./stage/ -docker build -t burntsushi/cross:mips64-unknown-linux-gnuabi64 . +docker build -t burntsushi/cross:powerpc64-unknown-linux-gnu . diff --git a/ci/docker/s390x-unknown-linux-gnu/Dockerfile b/ci/docker/s390x-unknown-linux-gnu/Dockerfile new file mode 100644 index 00000000..65a20876 --- /dev/null +++ b/ci/docker/s390x-unknown-linux-gnu/Dockerfile @@ -0,0 +1,4 @@ +FROM rustembedded/cross:s390x-unknown-linux-gnu + +COPY stage/ubuntu-install-packages / +RUN /ubuntu-install-packages diff --git a/ci/docker/s390x-unknown-linux-gnu/build b/ci/docker/s390x-unknown-linux-gnu/build new file mode 100755 index 00000000..f2fcb438 --- /dev/null +++ b/ci/docker/s390x-unknown-linux-gnu/build @@ -0,0 +1,5 @@ +#!/bin/sh + +mkdir -p stage +cp ../../ubuntu-install-packages ./stage/ +docker build -t burntsushi/cross:s390x-unknown-linux-gnu . diff --git a/tests/feature.rs b/tests/feature.rs index 94bb62be..8021043e 100644 --- a/tests/feature.rs +++ b/tests/feature.rs @@ -791,6 +791,9 @@ rgtest!(f1466_no_ignore_files, |dir: Dir, mut cmd: TestCommand| { rgtest!(f2361_sort_nested_files, |dir: Dir, mut cmd: TestCommand| { use std::{thread::sleep, time::Duration}; + if crate::util::is_cross() { + return; + } dir.create("foo", "1"); sleep(Duration::from_millis(100)); dir.create_dir("dir"); diff --git a/tests/misc.rs b/tests/misc.rs index 40f900dd..4fa8632e 100644 --- a/tests/misc.rs +++ b/tests/misc.rs @@ -1100,12 +1100,18 @@ rgtest!(sort_files, |dir: Dir, mut cmd: TestCommand| { }); rgtest!(sort_accessed, |dir: Dir, mut cmd: TestCommand| { + if crate::util::is_cross() { + return; + } sort_setup(dir); let expected = "a:test\ndir/c:test\nb:test\ndir/d:test\n"; eqnice!(expected, cmd.args(["--sort", "accessed", "test"]).stdout()); }); rgtest!(sortr_accessed, |dir: Dir, mut cmd: TestCommand| { + if crate::util::is_cross() { + return; + } sort_setup(dir); let expected = "dir/d:test\nb:test\ndir/c:test\na:test\n"; eqnice!(expected, cmd.args(["--sortr", "accessed", "test"]).stdout()); diff --git a/tests/util.rs b/tests/util.rs index 75fe92c1..4f958eb8 100644 --- a/tests/util.rs +++ b/tests/util.rs @@ -464,19 +464,39 @@ fn dir_list>(dir: P) -> Vec { /// will have setup qemu to run it. While this is integrated into the Rust /// testing by default, we need to handle it ourselves for integration tests. /// -/// Thankfully, cross sets an environment variable that points to the proper -/// qemu binary that we want to run. So we just search for that env var and -/// return its value if we could find it. +/// Now thankfully, cross sets `CROSS_RUNNER` to point to the right qemu +/// executable. Or so one thinks. But it seems to always be set to `qemu-user` +/// and I cannot find `qemu-user` anywhere in the Docker image. Awesome. +/// +/// Thers is `/linux-runner` which seems to work sometimes? But not always. +/// +/// Instead, it looks like we have to use `qemu-aarch64` in the `aarch64` +/// case. Perfect, so just get the current target architecture and append it +/// to `qemu-`. Wrong. Cross (or qemu or whoever) uses `qemu-ppc64` for +/// `powerpc64`, so we can't just use the target architecture as Rust knows +/// it verbatim. +/// +/// So... we just manually handle these cases. So fucking fun. fn cross_runner() -> Option { - for (k, v) in std::env::vars_os() { - let (k, v) = (k.to_string_lossy(), v.to_string_lossy()); - if !k.starts_with("CARGO_TARGET_") && !k.ends_with("_RUNNER") { - continue; - } - if !v.starts_with("qemu-") { - continue; - } - return Some(v.into_owned()); + let runner = std::env::var("CROSS_RUNNER").ok()?; + if runner.is_empty() { + return None; + } + if cfg!(target_arch = "powerpc64") { + Some("qemu-ppc64".to_string()) + } else if cfg!(target_arch = "x86") { + Some("i386".to_string()) + } else { + // Make a guess... Sigh. + Some(format!("qemu-{}", std::env::consts::ARCH)) } - None +} + +/// Returns true if the test setup believes Cross is running and `qemu` is +/// needed to run ripgrep. +/// +/// This is useful because it has been difficult to get some tests to pass +/// under Cross. +pub fn is_cross() -> bool { + std::env::var("CROSS_RUNNER").ok().map_or(false, |v| !v.is_empty()) }