add initial cargo ci

.cargo/config.toml

@@ -3,6 +3,7 @@ rustflags = ["--cfg", "tokio_unstable"]
 
 [alias]
 bump-versions = "run -p upgrade-version --"
+ci = "run -p ci --"
 
 [target.x86_64-pc-windows-msvc]
 # Use a different linker. Otherwise, the build fails with some obscure linker error that

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
         include:
           - { runner: spacetimedb-runner, smoketest_args: --docker }
           - { runner: windows-latest, smoketest_args: --no-build-cli }
-        runner: [ spacetimedb-runner, windows-latest ]
+        runner: [spacetimedb-runner, windows-latest]
     runs-on: ${{ matrix.runner }}
     steps:
       - name: Find Git ref
@@ -70,8 +70,8 @@ jobs:
       - name: Install psycopg2
         run: python -m pip install psycopg2-binary
       - name: Run smoketests
-        # Note: clear_database and replication only work in private
-        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication
+        run: cargo ci smoketests -- ${{ matrix.smoketest_args }}
+
       - name: Stop containers (Linux)
         if: always() && runner.os == 'Linux'
         run: docker compose down
@@ -103,26 +103,9 @@ jobs:
         with:
           global-json-file: global.json
 
-      - name: Create /stdb dir
-        run: |
-          sudo mkdir /stdb
-          sudo chmod 777 /stdb
-
       - name: Run cargo test
         #Note: Unreal tests will be run separately
-        run: cargo test --all -- --skip unreal
-
-      - name: Check that the test outputs are up-to-date
-        run: bash tools/check-diff.sh
-
-      - name: Ensure C# autogen bindings are up-to-date
-        run: |
-          cargo run -p spacetimedb-codegen --example regen-csharp-moduledef
-          bash tools/check-diff.sh crates/bindings-csharp
-
-      - name: C# bindings tests
-        working-directory: crates/bindings-csharp
-        run: dotnet test -warnaserror
+        run: cargo ci test
 
   lints:
     name: Lints
@@ -139,29 +122,7 @@ jobs:
           global-json-file: global.json
 
       - name: Run cargo fmt
-        run: cargo fmt --all -- --check
-
-      - name: Run cargo clippy
-        run: cargo clippy --all --tests --benches -- -D warnings
-
-      - name: Run C# formatting check
-        working-directory: crates/bindings-csharp
-        run: |
-          dotnet tool restore
-          dotnet csharpier --check .
-
-      - name: Run `cargo doc` for bindings crate
-        # `bindings` is the only crate we care strongly about documenting,
-        # since we link to its docs.rs from our website.
-        # We won't pass `--no-deps`, though,
-        # since we want everything reachable through it to also work.
-        # This includes `sats` and `lib`.
-        working-directory: crates/bindings
-        env:
-          # Make `cargo doc` exit with error on warnings, most notably broken links
-          RUSTDOCFLAGS: '--deny warnings'
-        run: |
-          cargo doc
+        run: cargo ci lint
 
   wasm_bindings:
     name: Build and test wasm bindings
@@ -173,20 +134,7 @@ jobs:
       - run: echo ::add-matcher::.github/workflows/rust_matcher.json
 
       - name: Run bindgen tests
-        run: cargo test -p spacetimedb-codegen
-
-      # Make sure the `Cargo.lock` file reflects the latest available versions.
-      # This is what users would end up with on a fresh module, so we want to
-      # catch any compile errors arising from a different transitive closure
-      # of dependencies than what is in the workspace lock file.
-      #
-      # For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714
-      - name: Update dependencies
-        run: cargo update
-
-      - name: Build module-test
-        run: cargo run -p spacetimedb-cli -- build --project-path modules/module-test
-
+        run: cargo ci wasm-bindings
 
   publish_checks:
     name: Check that packages are publishable
@@ -239,19 +187,9 @@ jobs:
         run: sudo apt install -y libssl-dev
 
       - name: Build spacetimedb-update
-        run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update
-
-      - name: Run self-install
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        shell: bash
-        run: |
-          ROOT_DIR="$(mktemp -d)"
-          # NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`.
-          # My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it
-          # happens very frequently on the `macos-runner`, but we haven't seen it on any others).
-          cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes
-          "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help
+        run: cargo ci update-flow --target=${{ matrix.target }}
 
   unreal_engine_tests:
     name: Unreal Engine Tests
@@ -290,6 +228,15 @@ jobs:
         with:
           ref: ${{ env.GIT_REF }}
       - uses: dsherret/rust-toolchain-file@v1
+      - name: Install unreal engine test dependencies
+        working-directory: sdks/unreal
+        env:
+          UE_ROOT_PATH: /home/ue4/UnrealEngine
+        run: |
+
+          apt-get update
+          apt-get install -y acl curl ca-certificates
+
       - name: Run Unreal Engine tests
         working-directory: sdks/unreal
         env:

Cargo.toml

@@ -46,6 +46,7 @@ members = [
   "sdks/rust/tests/test-client",
   "sdks/rust/tests/test-counter",
   "sdks/rust/tests/connect_disconnect_client",
+  "tools/ci",
   "tools/upgrade-version",
   "tools/license-check",
   "crates/bindings-typescript/test-app/server",

tools/ci/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "ci"
+version = "0.1.0"
+edition.workspace = true
+
+[dependencies]
+log.workspace = true
+anyhow.workspace = true
+chrono = { workspace = true, features=["clock"] }
+clap.workspace = true
+regex.workspace = true
+duct.workspace = true

tools/ci/build.rs

@@ -0,0 +1,4 @@
+#[allow(clippy::disallowed_macros)]
+fn main() {
+    println!("cargo:rustc-env=TARGET={}", std::env::var("TARGET").unwrap_or_default());
+}

tools/ci/src/main.rs

@@ -0,0 +1,172 @@
+use anyhow::Result;
+use clap::{CommandFactory, Parser, Subcommand};
+use duct::cmd;
+use std::collections::HashMap;
+use std::env;
+
+#[derive(Parser)]
+#[command(
+    name = "spacetimedb-ci",
+    about = "SpacetimeDB CI tasks",
+    subcommand_required = false,
+    arg_required_else_help = false
+)]
+struct Cli {
+    #[command(subcommand)]
+    cmd: Option<CiCmd>,
+
+    #[arg(long)]
+    skip: Vec<String>,
+}
+
+#[derive(Subcommand)]
+enum CiCmd {
+    Test,
+    Lints,
+    WasmBindings,
+    Smoketests {
+        #[arg(trailing_var_arg = true)]
+        args: Vec<String>,
+    },
+    UpdateFlow {
+        #[arg(long)]
+        target: Option<String>,
+        #[arg(long, default_value = "true")]
+        github_token_auth: bool,
+    },
+    UnrealTests,
+    CliDocs,
+}
+
+macro_rules! run {
+    ($cmdline:expr) => {
+        run_command($cmdline, &Vec::new())
+    };
+    ($cmdline:expr, $envs:expr) => {
+        run_command($cmdline, $envs)
+    };
+}
+
+fn run_all_clap_subcommands(skips: &[String]) -> Result<()> {
+    let subcmds = Cli::command()
+        .get_subcommands()
+        .map(|sc| sc.get_name().to_string())
+        .collect::<Vec<_>>();
+
+    for subcmd in subcmds {
+        if skips.contains(&subcmd) {
+            log::info!("skipping {subcmd} as requested");
+            continue;
+        }
+        log::info!("executing cargo ci {subcmd}");
+        run!(&format!("cargo ci {subcmd}"))?;
+    }
+
+    Ok(())
+}
+
+fn run_command(cmdline: &str, additional_env: &[(&str, &str)]) -> Result<()> {
+    let mut env = env::vars().collect::<HashMap<_, _>>();
+    env.extend(additional_env.iter().map(|(k, v)| (k.to_string(), v.to_string())));
+    log::debug!("$ {cmdline}");
+    let status = cmd!("bash", "-lc", cmdline).full_env(env).run()?;
+    if !status.status.success() {
+        let e = anyhow::anyhow!("command failed: {cmdline}");
+        log::error!("{e}");
+        return Err(e);
+    }
+    Ok(())
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    match cli.cmd {
+        Some(CiCmd::Test) => {
+            run!("sudo mkdir -p /stdb && sudo chmod 777 /stdb")?;
+            run!("cargo test --all -- --skip unreal")?;
+            run!("bash tools/check-diff.sh")?;
+            run!("cargo run -p spacetimedb-codegen --example regen-csharp-moduledef && bash tools/check-diff.sh crates/bindings-csharp")?;
+            run!("(cd crates/bindings-csharp && dotnet test -warnaserror)")?;
+        }
+
+        Some(CiCmd::Lints) => {
+            run!("cargo fmt --all -- --check")?;
+            run!("cargo clippy --all --tests --benches -- -D warnings")?;
+            run!("(cd crates/bindings-csharp && dotnet tool restore && dotnet csharpier --check .)")?;
+            run!("cd crates/bindings && cargo doc", &[("RUSTDOCFLAGS", "hey")])?;
+        }
+
+        Some(CiCmd::WasmBindings) => {
+            run!("cargo test -p spacetimedb-codegen")?;
+            run!("cargo update")?;
+            run!("cargo run -p spacetimedb-cli -- build --project-path modules/module-test")?;
+        }
+
+        Some(CiCmd::Smoketests { args }) => {
+            // Note: clear_database and replication only work in private
+            run!(&format!("python -m smoketests {} -x clear_database replication", args.join(" ")))?;
+        }
+
+        Some(CiCmd::UpdateFlow {
+            target,
+            github_token_auth,
+        }) => {
+            let target = target.unwrap_or_else(|| env!("TARGET").to_string());
+            let github_token_auth_flag = if github_token_auth {
+                "--features github-token-auth "
+            } else {
+                ""
+            };
+
+            run!(&format!("echo 'checking update flow for target: {target}'"))?;
+            run!(&format!(
+                "cargo build {github_token_auth_flag}--target {target} -p spacetimedb-update"
+            ))?;
+            run!(&format!(
+                r#"
+ROOT_DIR="$(mktemp -d)"
+# NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`.
+# My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it
+# happens very frequently on the `macos-runner`, but we haven't seen it on any others).
+cargo run {github_token_auth_flag}--target {target} -p spacetimedb-update -- self-install --root-dir="${{ROOT_DIR}}" --yes
+"${{ROOT_DIR}}"/spacetime --root-dir="${{ROOT_DIR}}" help
+        "#
+            ))?;
+        }
+
+        Some(CiCmd::UnrealTests) => {
+            run!("for p in \"$GITHUB_WORKSPACE\" \"${RUNNER_TEMP:-/__t}\" \"${RUNNER_TOOL_CACHE:-/__t}\"; do [ -d \"$p\" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX \"$p\" || true; done")?;
+
+            run!("export CARGO_HOME=\"${RUNNER_TOOL_CACHE:-/__t}/cargo\"")?;
+            run!("export RUSTUP_HOME=\"${RUNNER_TOOL_CACHE:-/__t}/rustup\"")?;
+            run!("mkdir -p \"$CARGO_HOME\" \"$RUSTUP_HOME\"")?;
+
+            run!("chmod a+rx \"$UE_ROOT_PATH\" \"$UE_ROOT_PATH/Engine\" \"$UE_ROOT_PATH/Engine/Build\" \"$UE_ROOT_PATH/Engine/Build/BatchFiles/Linux\" || true")?;
+            run!("chmod a+rx \"$UE_ROOT_PATH/Engine/Build/BatchFiles/Linux/Build.sh\" || true")?;
+
+            run!("sudo -E -H -u ue4 env HOME=/home/ue4 CARGO_HOME=\"$CARGO_HOME\" RUSTUP_HOME=\"$RUSTUP_HOME\" PATH=\"$CARGO_HOME/bin:$PATH\" bash -lc 'set -euxo pipefail; if ! command -v cargo >/dev/null 2>&1; then curl -sSf https://sh.rustup.rs | sh -s -- -y; fi; rustup show >/dev/null; git config --global --add safe.directory \"$GITHUB_WORKSPACE\" || true; cd \"$GITHUB_WORKSPACE/sdks/unreal\"; cargo --version; cargo test'")?;
+        }
+
+        Some(CiCmd::CliDocs) => {
+            run!("pnpm install --recursive")?;
+            run!("cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md")?;
+            run!("pnpm format")?;
+            run!("git status")?;
+            run!(
+                r#"
+if git diff --exit-code HEAD; then
+  echo "No docs changes detected"
+else
+  echo "It looks like the CLI docs have changed:"
+  exit 1
+fi
+                "#
+            )?;
+        }
+
+        None => run_all_clap_subcommands(&cli.skip)?,
+    }
+
+    Ok(())
+}