Always configure user

Closes https://github.com/astral-sh/ruff/issues/8406

---------

Co-authored-by: Dhruv Manilawala <dhruvmanila@gmail.com>

.github/workflows/ci.yaml

@@ -209,7 +209,13 @@ jobs:
         run: |
           pip install ./python/ruff-ecosystem
 
-      - name: Run `ruff check` ecosystem check
+      - name: Restore cached checkouts
+        uses: actions/cache/restore@v3
+        with:
+          path: ./checkouts
+          key: ecosystem-checkouts
+
+      - name: Run `ruff check` stable ecosystem check
         if: ${{ needs.determine_changes.outputs.linter == 'true' }}
         run: |
           # Make executable, since artifact download doesn't preserve this
@@ -218,13 +224,30 @@ jobs:
           # Set pipefail to avoid hiding errors with tee
           set -eo pipefail
 
-          ruff-ecosystem check ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown | tee ecosystem-result-check
+          ruff-ecosystem check ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown | tee ecosystem-result-check-stable
 
-          cat ecosystem-result-check > $GITHUB_STEP_SUMMARY
-          cat ecosystem-result-check > ecosystem-result
+          cat ecosystem-result-check-stable > $GITHUB_STEP_SUMMARY
+          echo "### Linter (stable)"  > ecosystem-result
+          cat ecosystem-result-check-stable >> ecosystem-result
           echo "" >> ecosystem-result
 
-      - name: Run `ruff format` ecosystem check
+      - name: Run `ruff check` preview ecosystem check
+        if: ${{ needs.determine_changes.outputs.linter == 'true' }}
+        run: |
+          # Make executable, since artifact download doesn't preserve this
+          chmod +x ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff
+
+          # Set pipefail to avoid hiding errors with tee
+          set -eo pipefail
+
+          ruff-ecosystem check ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown --force-preview | tee ecosystem-result-check-preview
+
+          cat ecosystem-result-check-preview > $GITHUB_STEP_SUMMARY
+          echo "### Linter (preview)" >> ecosystem-result
+          cat ecosystem-result-check-preview >> ecosystem-result
+          echo "" >> ecosystem-result
+
+      - name: Run `ruff format` stable ecosystem check
         if: ${{ needs.determine_changes.outputs.formatter == 'true' }}
         run: |
           # Make executable, since artifact download doesn't preserve this
@@ -233,10 +256,34 @@ jobs:
           # Set pipefail to avoid hiding errors with tee
           set -eo pipefail
 
-          ruff-ecosystem format ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown | tee ecosystem-result-format
+          ruff-ecosystem format ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown | tee ecosystem-result-format-stable
 
-          cat ecosystem-result-format > $GITHUB_STEP_SUMMARY
-          cat ecosystem-result-format >> ecosystem-result
+          cat ecosystem-result-format-stable > $GITHUB_STEP_SUMMARY
+          echo "### Formatter (stable)" >> ecosystem-result
+          cat ecosystem-result-format-stable >> ecosystem-result
+          echo "" >> ecosystem-result
+
+      - name: Run `ruff format` preview ecosystem check
+        if: ${{ needs.determine_changes.outputs.formatter == 'true' }}
+        run: |
+          # Make executable, since artifact download doesn't preserve this
+          chmod +x ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff
+
+          # Set pipefail to avoid hiding errors with tee
+          set -eo pipefail
+
+          ruff-ecosystem format ./ruff ${{ steps.ruff-target.outputs.download-path }}/ruff --cache ./checkouts --output-format markdown --force-preview | tee ecosystem-result-format-preview
+
+          cat ecosystem-result-format-preview > $GITHUB_STEP_SUMMARY
+          echo "### Formatter (preview)" >> ecosystem-result
+          cat ecosystem-result-format-preview >> ecosystem-result
+          echo "" >> ecosystem-result
+
+      - name: Save cached checkouts
+        uses: actions/cache/save@v3
+        with:
+          path: ./checkouts
+          key: ecosystem-checkouts
 
       - name: Export pull request number
         run: |

.github/workflows/pr-comment.yaml

@@ -48,9 +48,7 @@ jobs:
         id: generate-comment
         if: steps.download-ecosystem-result.outputs.found_artifact == 'true'
         run: |
-          echo '## PR Check Results' >> comment.txt
-
-          echo "### Ecosystem" >> comment.txt
+          echo '## `ruff-ecosystem` results' >> comment.txt
           cat pr/ecosystem/ecosystem-result >> comment.txt
           echo "" >> comment.txt
 

.github/workflows/release.yaml

@@ -48,7 +48,6 @@ jobs:
           args: --out dist
       - name: "Test sdist"
         run: |
-          rustup default $(cat rust-toolchain)
           pip install dist/${{ env.PACKAGE_NAME }}-*.tar.gz --force-reinstall
           ruff --help
           python -m ruff --help

.pre-commit-config.yaml

@@ -47,10 +47,13 @@ repos:
         language: system
         types: [rust]
         pass_filenames: false # This makes it a lot faster
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.3
+    hooks:
+      - id: ruff-format
       - id: ruff
-        name: ruff
-        entry: cargo run --bin ruff -- check --no-cache --force-exclude --fix --exit-non-zero-on-fix
-        language: system
+        args: [--fix, --exit-non-zero-on-fix]
         types_or: [python, pyi]
         require_serial: true
         exclude: |
@@ -59,12 +62,6 @@ repos:
             crates/ruff_python_formatter/resources/.*
           )$
 
-  # Black
-  - repo: https://github.com/psf/black
-    rev: 23.1.0
-    hooks:
-      - id: black
-
   # Prettier
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.0.0

CONTRIBUTING.md

@@ -337,16 +337,15 @@ even patch releases may contain [non-backwards-compatible changes](https://semve
 ## Ecosystem CI
 
 GitHub Actions will run your changes against a number of real-world projects from GitHub and
-report on any diagnostic differences. You can also run those checks locally via:
+report on any linter or formatter differences. You can also run those checks locally via:
 
 ```shell
-python scripts/check_ecosystem.py path/to/your/ruff path/to/older/ruff
+pip install -e ./python/ruff-ecosystem
+ruff-ecosystem check ruff "./target/debug/ruff"
+ruff-ecosystem format ruff "./target/debug/ruff"
 ```
 
-You can also run the Ecosystem CI check in a Docker container across a larger set of projects by
-downloading the [`known-github-tomls.json`](https://github.com/akx/ruff-usage-aggregate/blob/master/data/known-github-tomls.jsonl)
-as `github_search.jsonl` and following the instructions in [scripts/Dockerfile.ecosystem](https://github.com/astral-sh/ruff/blob/main/scripts/Dockerfile.ecosystem).
-Note that this check will take a while to run.
+See the [ruff-ecosystem package](https://github.com/astral-sh/ruff/tree/main/python/ruff-ecosystem) for more details.
 
 ## Benchmarking and Profiling
 

crates/ruff_cli/src/args.rs

@@ -409,6 +409,9 @@ pub struct FormatCommand {
     force_exclude: bool,
     #[clap(long, overrides_with("force_exclude"), hide = true)]
     no_force_exclude: bool,
+    /// Set the line-length.
+    #[arg(long, help_heading = "Format configuration")]
+    pub line_length: Option<LineLength>,
     /// Ignore all configuration files.
     #[arg(long, conflicts_with = "config", help_heading = "Miscellaneous")]
     pub isolated: bool,
@@ -552,6 +555,7 @@ impl FormatCommand {
                 stdin_filename: self.stdin_filename,
             },
             CliOverrides {
+                line_length: self.line_length,
                 respect_gitignore: resolve_bool_arg(
                     self.respect_gitignore,
                     self.no_respect_gitignore,

crates/ruff_linter/resources/test/fixtures/pylint/bad_string_format_type.py

@@ -57,3 +57,9 @@ r'\%03o' % (ord(c),)
 '(%r, %r, %r, %r)' % (hostname, address, username, '$PASSWORD')
 '%r' % ({'server_school_roles': server_school_roles, 'is_school_multiserver_domain': is_school_multiserver_domain}, )
 "%d" % (1 if x > 0 else 2)
+
+# Special cases for %c allowing single character strings
+# https://github.com/astral-sh/ruff/issues/8406
+"%c" % ("x",)
+"%c" % "x"
+"%c" % "œ"

crates/ruff_linter/resources/test/fixtures/pyupgrade/UP014.py

@@ -23,3 +23,11 @@ MyType = typing.NamedTuple("MyType", a=int, b=tuple[str, ...])
 MyType = typing.NamedTuple("MyType", [("a", int)], [("b", str)])
 MyType = typing.NamedTuple("MyType", [("a", int)], b=str)
 MyType = typing.NamedTuple(typename="MyType", a=int, b=str)
+
+# Regression test for: https://github.com/astral-sh/ruff/issues/8402#issuecomment-1788787357
+S3File = NamedTuple(
+    "S3File",
+    [
+        ("dataHPK",* str),
+    ],
+)

crates/ruff_linter/resources/test/fixtures/refurb/FURB171.py

@@ -43,3 +43,6 @@ if 1 is {1}:
 
 if "a" == "a":
     pass
+
+if 1 in {*[1]}:
+    pass

crates/ruff_linter/src/rules/pylint/rules/bad_string_format_type.rs

@@ -119,12 +119,23 @@ fn collect_specs(formats: &[CFormatStrOrBytes<String>]) -> Vec<&CFormatSpec> {
 
 /// Return `true` if the format string is equivalent to the constant type
 fn equivalent(format: &CFormatSpec, value: &Expr) -> bool {
-    let format = FormatType::from(format.format_char);
+    let format_type = FormatType::from(format.format_char);
     match ResolvedPythonType::from(value) {
-        ResolvedPythonType::Atom(atom) => format.is_compatible_with(atom),
-        ResolvedPythonType::Union(atoms) => {
-            atoms.iter().all(|atom| format.is_compatible_with(*atom))
+        ResolvedPythonType::Atom(atom) => {
+            // Special case where `%c` allows single character strings to be formatted
+            if format.format_char == 'c' {
+                if let Expr::StringLiteral(string) = value {
+                    let mut chars = string.chars();
+                    if chars.next().is_some() && chars.next().is_none() {
+                        return true;
+                    }
+                }
+            }
+            format_type.is_compatible_with(atom)
         }
+        ResolvedPythonType::Union(atoms) => atoms
+            .iter()
+            .all(|atom| format_type.is_compatible_with(*atom)),
         ResolvedPythonType::Unknown => true,
         ResolvedPythonType::TypeError => true,
     }

crates/ruff_linter/src/rules/pyupgrade/rules/convert_named_tuple_functional_to_class.rs

@@ -181,6 +181,9 @@ fn create_fields_from_fields_arg(fields: &Expr) -> Option<Vec<Stmt>> {
                 let [field, annotation] = elts.as_slice() else {
                     return None;
                 };
+                if annotation.is_starred_expr() {
+                    return None;
+                }
                 let ast::ExprStringLiteral { value: field, .. } = field.as_string_literal_expr()?;
                 if !is_identifier(field) {
                     return None;

crates/ruff_linter/src/rules/refurb/rules/single_item_membership_test.rs

@@ -1,8 +1,7 @@
 use ruff_diagnostics::{Diagnostic, Edit, Fix, FixAvailability, Violation};
 use ruff_macros::{derive_message_formats, violation};
 use ruff_python_ast::helpers::generate_comparison;
-use ruff_python_ast::ExprStringLiteral;
-use ruff_python_ast::{CmpOp, Expr};
+use ruff_python_ast::{self as ast, CmpOp, Expr, ExprStringLiteral};
 use ruff_text_size::Ranged;
 
 use crate::checkers::ast::Checker;
@@ -95,13 +94,17 @@ pub(crate) fn single_item_membership_test(
     checker.diagnostics.push(diagnostic);
 }
 
-/// Return the single item wrapped in Some if the expression contains a single
-/// item, otherwise return None.
+/// Return the single item wrapped in `Some` if the expression contains a single
+/// item, otherwise return `None`.
 fn single_item(expr: &Expr) -> Option<&Expr> {
     match expr {
-        Expr::List(list) if list.elts.len() == 1 => Some(&list.elts[0]),
-        Expr::Tuple(tuple) if tuple.elts.len() == 1 => Some(&tuple.elts[0]),
-        Expr::Set(set) if set.elts.len() == 1 => Some(&set.elts[0]),
+        Expr::List(ast::ExprList { elts, .. })
+        | Expr::Tuple(ast::ExprTuple { elts, .. })
+        | Expr::Set(ast::ExprSet { elts, .. }) => match elts.as_slice() {
+            [Expr::Starred(_)] => None,
+            [item] => Some(item),
+            _ => None,
+        },
         string_expr @ Expr::StringLiteral(ExprStringLiteral { value: string, .. })
             if string.chars().count() == 1 =>
         {

crates/ruff_python_formatter/README.md

@@ -117,10 +117,11 @@ quote-style = "single"
 ```
 
 The Ruff formatter also respects Ruff's [`line-length`](https://docs.astral.sh/ruff/settings/#line-length)
-setting, which also can be provided via a `pyproject.toml` or `ruff.toml` file.
+setting, which also can be provided via a `pyproject.toml` or `ruff.toml` file, or on the CLI, as
+in:
 
-```toml
-line-length = 80
+```console
+ruff format --line-length 100 /path/to/file.py
 ```
 
 ### Excluding code from formatting

docs/configuration.md

@@ -402,6 +402,9 @@ File selection:
       --exclude <FILE_PATTERN>  List of paths, used to omit files and/or directories from analysis
       --force-exclude           Enforce exclusions, even for paths passed to Ruff directly on the command-line. Use `--no-force-exclude` to disable
 
+Format configuration:
+      --line-length <LINE_LENGTH>  Set the line-length
+
 Log levels:
   -v, --verbose  Enable verbose logging
   -q, --quiet    Print diagnostics, but nothing else

docs/linter.md

@@ -277,6 +277,8 @@ Ruff will also respect variants of these action comments with a `# ruff:` prefix
 convey that the action comment is intended for Ruff, but are functionally equivalent to the
 isort variants.
 
+Unlike isort, Ruff does not respect action comments within docstrings.
+
 See the [isort documentation](https://pycqa.github.io/isort/docs/configuration/action_comments.html)
 for more.
 

pyproject.toml

@@ -52,6 +52,9 @@ exclude = [
   "crates/ruff_linter/resources/test/fixtures/**/*",
   "crates/ruff_linter/src/rules/*/snapshots/**/*"
 ]
+include = [
+  "rust-toolchain.toml"
+]
 
 [tool.ruff]
 extend-exclude = [

python/ruff-ecosystem/README.md

@@ -31,6 +31,21 @@ Run `ruff format` ecosystem checks comparing your debug build to your system Ruf
 ruff-ecosystem format ruff "./target/debug/ruff"
 ```
 
+Run `ruff format` ecosystem checks comparing with changes to code that is already formatted:
+
+```shell
+ruff-ecosystem format ruff "./target/debug/ruff" --format-comparison ruff-then-ruff
+```
+
+Run `ruff format` ecosystem checks comparing with the Black formatter:
+
+```shell
+ruff-ecosystem format black ruff -v --cache python/checkouts --format-comparison black-and-ruff
+```
+
+The default output format is markdown, which includes nice summaries of the changes. You can use `--output-format json` to display the raw data — this is
+particularly useful when making changes to the ecosystem checks.
+
 ## Development
 
 When developing, it can be useful to set the `--pdb` flag to drop into a debugger on failure:

python/ruff-ecosystem/ruff_ecosystem/check.py

@@ -24,12 +24,11 @@ from ruff_ecosystem.types import (
     Comparison,
     Diff,
     Result,
-    RuffError,
-    Serializable,
+    ToolError,
 )
 
 if TYPE_CHECKING:
-    from ruff_ecosystem.projects import ClonedRepository, Project
+    from ruff_ecosystem.projects import CheckOptions, ClonedRepository, Project
 
 
 # Matches lines that are summaries rather than diagnostics
@@ -501,8 +500,8 @@ async def ruff_check(
     *, executable: Path, path: Path, name: str, options: CheckOptions
 ) -> Sequence[str]:
     """Run the given ruff binary against the specified path."""
-    logger.debug(f"Checking {name} with {executable}")
-    ruff_args = options.to_cli_args()
+    ruff_args = options.to_ruff_args()
+    logger.debug(f"Checking {name} with {executable} " + " ".join(ruff_args))
 
     start = time.time()
     proc = await create_subprocess_exec(
@@ -519,7 +518,7 @@ async def ruff_check(
     logger.debug(f"Finished checking {name} with {executable} in {end - start:.2f}s")
 
     if proc.returncode != 0:
-        raise RuffError(err.decode("utf8"))
+        raise ToolError(err.decode("utf8"))
 
     # Strip summary lines so the diff is only diagnostic lines
     lines = [
@@ -529,35 +528,3 @@ async def ruff_check(
     ]
 
     return lines
-
-
-@dataclass(frozen=True)
-class CheckOptions(Serializable):
-    """
-    Ruff check options
-    """
-
-    select: str = ""
-    ignore: str = ""
-    exclude: str = ""
-
-    # Generating fixes is slow and verbose
-    show_fixes: bool = False
-
-    # Limit the number of reported lines per rule
-    max_lines_per_rule: int | None = 50
-
-    def markdown(self) -> str:
-        return f"select {self.select} ignore {self.ignore} exclude {self.exclude}"
-
-    def to_cli_args(self) -> list[str]:
-        args = ["check", "--no-cache", "--exit-zero"]
-        if self.select:
-            args.extend(["--select", self.select])
-        if self.ignore:
-            args.extend(["--ignore", self.ignore])
-        if self.exclude:
-            args.extend(["--exclude", self.exclude])
-        if self.show_fixes:
-            args.extend(["--show-fixes", "--ecosystem-ci"])
-        return args

python/ruff-ecosystem/ruff_ecosystem/cli.py

@@ -12,6 +12,7 @@ from signal import SIGINT, SIGTERM
 
 from ruff_ecosystem import logger
 from ruff_ecosystem.defaults import DEFAULT_TARGETS
+from ruff_ecosystem.format import FormatComparison
 from ruff_ecosystem.main import OutputFormat, main
 from ruff_ecosystem.projects import RuffCommand
 
@@ -45,45 +46,58 @@ def entrypoint():
         tempfile.TemporaryDirectory() if not args.cache else nullcontext(args.cache)
     )
 
-    ruff_baseline = args.ruff_baseline
-    if not args.ruff_baseline.exists():
-        ruff_baseline = get_executable_path(str(args.ruff_baseline))
-        if not ruff_baseline:
+    baseline_executable = args.baseline_executable
+    if not args.baseline_executable.exists():
+        baseline_executable = get_executable_path(str(args.baseline_executable))
+        if not baseline_executable:
             print(
-                f"Could not find ruff baseline executable: {args.ruff_baseline}",
+                f"Could not find ruff baseline executable: {args.baseline_executable}",
                 sys.stderr,
             )
             exit(1)
         logger.info(
-            "Resolved baseline executable %s to %s", args.ruff_baseline, ruff_baseline
+            "Resolved baseline executable %s to %s",
+            args.baseline_executable,
+            baseline_executable,
         )
 
-    ruff_comparison = args.ruff_comparison
-    if not args.ruff_comparison.exists():
-        ruff_comparison = get_executable_path(str(args.ruff_comparison))
-        if not ruff_comparison:
+    comparison_executable = args.comparison_executable
+    if not args.comparison_executable.exists():
+        comparison_executable = get_executable_path(str(args.comparison_executable))
+        if not comparison_executable:
             print(
-                f"Could not find ruff comparison executable: {args.ruff_comparison}",
+                f"Could not find ruff comparison executable: {args.comparison_executable}",
                 sys.stderr,
             )
             exit(1)
         logger.info(
             "Resolved comparison executable %s to %s",
-            args.ruff_comparison,
-            ruff_comparison,
+            args.comparison_executable,
+            comparison_executable,
         )
 
+    targets = DEFAULT_TARGETS
+    if args.force_preview:
+        targets = [target.with_preview_enabled() for target in targets]
+
+    format_comparison = (
+        FormatComparison(args.format_comparison)
+        if args.ruff_command == RuffCommand.format.value
+        else None
+    )
+
     with cache_context as cache:
         loop = asyncio.get_event_loop()
         main_task = asyncio.ensure_future(
             main(
                 command=RuffCommand(args.ruff_command),
-                ruff_baseline_executable=ruff_baseline,
-                ruff_comparison_executable=ruff_comparison,
-                targets=DEFAULT_TARGETS,
+                baseline_executable=baseline_executable,
+                comparison_executable=comparison_executable,
+                targets=targets,
                 format=OutputFormat(args.output_format),
                 project_dir=Path(cache),
                 raise_on_failure=args.pdb,
+                format_comparison=format_comparison,
             )
         )
         # https://stackoverflow.com/a/58840987/3549270
@@ -116,8 +130,8 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--output-format",
-        choices=[option.name for option in OutputFormat],
-        default="json",
+        choices=[option.value for option in OutputFormat],
+        default="markdown",
         help="Location for caching cloned repositories",
     )
     parser.add_argument(
@@ -131,17 +145,28 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Enable debugging on failure",
     )
+    parser.add_argument(
+        "--force-preview",
+        action="store_true",
+        help="Force preview mode to be enabled for all projects",
+    )
+    parser.add_argument(
+        "--format-comparison",
+        choices=[option.value for option in FormatComparison],
+        default=FormatComparison.ruff_and_ruff,
+        help="Type of comparison to make when checking formatting.",
+    )
     parser.add_argument(
         "ruff_command",
-        choices=[option.name for option in RuffCommand],
+        choices=[option.value for option in RuffCommand],
         help="The Ruff command to test",
     )
     parser.add_argument(
-        "ruff_baseline",
+        "baseline_executable",
         type=Path,
     )
     parser.add_argument(
-        "ruff_comparison",
+        "comparison_executable",
         type=Path,
     )
 

python/ruff-ecosystem/ruff_ecosystem/format.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 
 import time
 from asyncio import create_subprocess_exec
-from dataclasses import dataclass
+from enum import Enum
 from pathlib import Path
 from subprocess import PIPE
 from typing import TYPE_CHECKING, Sequence
@@ -15,10 +15,10 @@ from unidiff import PatchSet
 
 from ruff_ecosystem import logger
 from ruff_ecosystem.markdown import markdown_project_section
-from ruff_ecosystem.types import Comparison, Diff, Result, RuffError
+from ruff_ecosystem.types import Comparison, Diff, Result, ToolError
 
 if TYPE_CHECKING:
-    from ruff_ecosystem.projects import ClonedRepository
+    from ruff_ecosystem.projects import ClonedRepository, FormatOptions
 
 
 def markdown_format_result(result: Result) -> str:
@@ -124,28 +124,89 @@ async def compare_format(
     ruff_comparison_executable: Path,
     options: FormatOptions,
     cloned_repo: ClonedRepository,
+    format_comparison: FormatComparison,
 ):
-    # Run format without diff to get the baseline
-    await ruff_format(
+    args = (ruff_baseline_executable, ruff_comparison_executable, options, cloned_repo)
+    match format_comparison:
+        case FormatComparison.ruff_then_ruff:
+            coro = format_then_format(Formatter.ruff, *args)
+        case FormatComparison.ruff_and_ruff:
+            coro = format_and_format(Formatter.ruff, *args)
+        case FormatComparison.black_then_ruff:
+            coro = format_then_format(Formatter.black, *args)
+        case FormatComparison.black_and_ruff:
+            coro = format_and_format(Formatter.black, *args)
+        case _:
+            raise ValueError(f"Unknown format comparison type {format_comparison!r}.")
+
+    diff = await coro
+    return Comparison(diff=Diff(diff), repo=cloned_repo)
+
+
+async def format_then_format(
+    baseline_formatter: Formatter,
+    ruff_baseline_executable: Path,
+    ruff_comparison_executable: Path,
+    options: FormatOptions,
+    cloned_repo: ClonedRepository,
+) -> Sequence[str]:
+    # Run format to get the baseline
+    await format(
+        formatter=baseline_formatter,
         executable=ruff_baseline_executable.resolve(),
         path=cloned_repo.path,
         name=cloned_repo.fullname,
         options=options,
     )
     # Then get the diff from stdout
-    diff = await ruff_format(
+    diff = await format(
+        formatter=Formatter.ruff,
         executable=ruff_comparison_executable.resolve(),
         path=cloned_repo.path,
         name=cloned_repo.fullname,
         options=options,
         diff=True,
     )
-
-    return Comparison(diff=Diff(diff), repo=cloned_repo)
+    return diff
 
 
-async def ruff_format(
+async def format_and_format(
+    baseline_formatter: Formatter,
+    ruff_baseline_executable: Path,
+    ruff_comparison_executable: Path,
+    options: FormatOptions,
+    cloned_repo: ClonedRepository,
+) -> Sequence[str]:
+    # Run format without diff to get the baseline
+    await format(
+        formatter=baseline_formatter,
+        executable=ruff_baseline_executable.resolve(),
+        path=cloned_repo.path,
+        name=cloned_repo.fullname,
+        options=options,
+    )
+    # Commit the changes
+    commit = await cloned_repo.commit(
+        message=f"Formatted with baseline {ruff_baseline_executable}"
+    )
+    # Then reset
+    await cloned_repo.reset()
+    # Then run format again
+    await format(
+        formatter=Formatter.ruff,
+        executable=ruff_comparison_executable.resolve(),
+        path=cloned_repo.path,
+        name=cloned_repo.fullname,
+        options=options,
+    )
+    # Then get the diff from the commit
+    diff = await cloned_repo.diff(commit)
+    return diff
+
+
+async def format(
     *,
+    formatter: Formatter,
     executable: Path,
     path: Path,
     name: str,
@@ -153,16 +214,20 @@ async def ruff_format(
     diff: bool = False,
 ) -> Sequence[str]:
     """Run the given ruff binary against the specified path."""
-    logger.debug(f"Formatting {name} with {executable}")
-    ruff_args = options.to_cli_args()
+    args = (
+        options.to_ruff_args()
+        if formatter == Formatter.ruff
+        else options.to_black_args()
+    )
+    logger.debug(f"Formatting {name} with {executable} " + " ".join(args))
 
     if diff:
-        ruff_args.append("--diff")
+        args.append("--diff")
 
     start = time.time()
     proc = await create_subprocess_exec(
         executable.absolute(),
-        *ruff_args,
+        *args,
         ".",
         stdout=PIPE,
         stderr=PIPE,
@@ -174,22 +239,34 @@ async def ruff_format(
     logger.debug(f"Finished formatting {name} with {executable} in {end - start:.2f}s")
 
     if proc.returncode not in [0, 1]:
-        raise RuffError(err.decode("utf8"))
+        raise ToolError(err.decode("utf8"))
 
     lines = result.decode("utf8").splitlines()
     return lines
 
 
-@dataclass(frozen=True)
-class FormatOptions:
+class FormatComparison(Enum):
+    ruff_then_ruff = "ruff-then-ruff"
     """
-    Ruff format options.
+    Run Ruff baseline then Ruff comparison; checks for changes in behavior when formatting previously "formatted" code
     """
 
-    exclude: str = ""
+    ruff_and_ruff = "ruff-and-ruff"
+    """
+    Run Ruff baseline then reset and run Ruff comparison; checks changes in behavior when formatting "unformatted" code
+    """
 
-    def to_cli_args(self) -> list[str]:
-        args = ["format"]
-        if self.exclude:
-            args.extend(["--exclude", self.exclude])
-        return args
+    black_then_ruff = "black-then-ruff"
+    """
+    Run Black baseline then Ruff comparison; checks for changes in behavior when formatting previously "formatted" code
+    """
+
+    black_and_ruff = "black-and-ruff"
+    """"
+    Run Black baseline then reset and run Ruff comparison; checks changes in behavior when formatting "unformatted" code
+    """
+
+
+class Formatter(Enum):
+    black = "black"
+    ruff = "ruff"

python/ruff-ecosystem/ruff_ecosystem/main.py

@@ -7,7 +7,11 @@ from typing import Awaitable, TypeVar
 
 from ruff_ecosystem import logger
 from ruff_ecosystem.check import compare_check, markdown_check_result
-from ruff_ecosystem.format import compare_format, markdown_format_result
+from ruff_ecosystem.format import (
+    FormatComparison,
+    compare_format,
+    markdown_format_result,
+)
 from ruff_ecosystem.projects import (
     Project,
     RuffCommand,
@@ -25,18 +29,21 @@ class OutputFormat(Enum):
 
 async def main(
     command: RuffCommand,
-    ruff_baseline_executable: Path,
-    ruff_comparison_executable: Path,
+    baseline_executable: Path,
+    comparison_executable: Path,
     targets: list[Project],
     project_dir: Path,
     format: OutputFormat,
+    format_comparison: FormatComparison | None,
     max_parallelism: int = 50,
     raise_on_failure: bool = False,
 ) -> None:
     logger.debug("Using command %s", command.value)
-    logger.debug("Using baseline executable at %s", ruff_baseline_executable)
-    logger.debug("Using comparison executable at %s", ruff_comparison_executable)
+    logger.debug("Using baseline executable at %s", baseline_executable)
+    logger.debug("Using comparison executable at %s", comparison_executable)
     logger.debug("Using checkout_dir directory %s", project_dir)
+    if format_comparison:
+        logger.debug("Using format comparison type %s", format_comparison.value)
     logger.debug("Checking %s targets", len(targets))
 
     # Limit parallelism to avoid high memory consumption
@@ -51,10 +58,11 @@ async def main(
             limited_parallelism(
                 clone_and_compare(
                     command,
-                    ruff_baseline_executable,
-                    ruff_comparison_executable,
+                    baseline_executable,
+                    comparison_executable,
                     target,
                     project_dir,
+                    format_comparison,
                 )
             )
             for target in targets
@@ -92,10 +100,11 @@ async def main(
 
 async def clone_and_compare(
     command: RuffCommand,
-    ruff_baseline_executable: Path,
-    ruff_comparison_executable: Path,
+    baseline_executable: Path,
+    comparison_executable: Path,
     target: Project,
     project_dir: Path,
+    format_comparison: FormatComparison | None,
 ) -> Comparison:
     """Check a specific repository against two versions of ruff."""
     assert ":" not in target.repo.owner
@@ -103,14 +112,12 @@ async def clone_and_compare(
 
     match command:
         case RuffCommand.check:
-            compare, options = (
-                compare_check,
-                target.check_options,
-            )
+            compare, options, kwargs = (compare_check, target.check_options, {})
         case RuffCommand.format:
-            compare, options = (
+            compare, options, kwargs = (
                 compare_format,
                 target.format_options,
+                {"format_comparison": format_comparison},
             )
         case _:
             raise ValueError(f"Unknown target Ruff command {command}")
@@ -120,10 +127,11 @@ async def clone_and_compare(
 
     try:
         return await compare(
-            ruff_baseline_executable,
-            ruff_comparison_executable,
+            baseline_executable,
+            comparison_executable,
             options,
             cloned_repo,
+            **kwargs,
         )
     except ExceptionGroup as e:
         raise e.exceptions[0] from e

python/ruff-ecosystem/ruff_ecosystem/markdown.py

@@ -12,7 +12,7 @@ def markdown_project_section(
     return markdown_details(
         summary=f'<a href="{project.repo.url}">{project.repo.fullname}</a> ({title})',
         # Show the command used for the check
-        preface="<pre>ruff " + " ".join(options.to_cli_args()) + "</pre>",
+        preface="<pre>ruff " + " ".join(options.to_ruff_args()) + "</pre>",
         content=content,
     )
 

python/ruff-ecosystem/ruff_ecosystem/projects.py

@@ -4,16 +4,16 @@ Abstractions and utilities for working with projects to run ecosystem checks on.
 
 from __future__ import annotations
 
+import abc
+import dataclasses
 from asyncio import create_subprocess_exec
 from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
-from subprocess import PIPE
+from subprocess import DEVNULL, PIPE
 from typing import Self
 
 from ruff_ecosystem import logger
-from ruff_ecosystem.check import CheckOptions
-from ruff_ecosystem.format import FormatOptions
 from ruff_ecosystem.types import Serializable
 
 
@@ -27,12 +27,90 @@ class Project(Serializable):
     check_options: CheckOptions = field(default_factory=lambda: CheckOptions())
     format_options: FormatOptions = field(default_factory=lambda: FormatOptions())
 
+    def with_preview_enabled(self: Self) -> Self:
+        return type(self)(
+            repo=self.repo,
+            check_options=self.check_options.with_options(preview=True),
+            format_options=self.format_options.with_options(preview=True),
+        )
+
 
 class RuffCommand(Enum):
     check = "check"
     format = "format"
 
 
+@dataclass(frozen=True)
+class CommandOptions(Serializable, abc.ABC):
+    def with_options(self: Self, **kwargs) -> Self:
+        """
+        Return a copy of self with the given options set.
+        """
+        return type(self)(**{**dataclasses.asdict(self), **kwargs})
+
+    @abc.abstractmethod
+    def to_ruff_args(self) -> list[str]:
+        pass
+
+
+@dataclass(frozen=True)
+class CheckOptions(CommandOptions):
+    """
+    Ruff check options
+    """
+
+    select: str = ""
+    ignore: str = ""
+    exclude: str = ""
+    preview: bool = False
+
+    # Generating fixes is slow and verbose
+    show_fixes: bool = False
+
+    # Limit the number of reported lines per rule
+    max_lines_per_rule: int | None = 50
+
+    def to_ruff_args(self) -> list[str]:
+        args = ["check", "--no-cache", "--exit-zero"]
+        if self.select:
+            args.extend(["--select", self.select])
+        if self.ignore:
+            args.extend(["--ignore", self.ignore])
+        if self.exclude:
+            args.extend(["--exclude", self.exclude])
+        if self.show_fixes:
+            args.extend(["--show-fixes", "--ecosystem-ci"])
+        if self.preview:
+            args.append("--preview")
+        return args
+
+
+@dataclass(frozen=True)
+class FormatOptions(CommandOptions):
+    """
+    Format ecosystem check options.
+    """
+
+    preview: bool = False
+    exclude: str = ""
+
+    def to_ruff_args(self) -> list[str]:
+        args = ["format"]
+        if self.exclude:
+            args.extend(["--exclude", self.exclude])
+        if self.preview:
+            args.append("--preview")
+        return args
+
+    def to_black_args(self) -> list[str]:
+        args = []
+        if self.exclude:
+            args.extend(["--exclude", self.exclude])
+        if self.preview:
+            args.append("--preview")
+        return args
+
+
 class ProjectSetupError(Exception):
     """An error setting up a project."""
 
@@ -60,10 +138,11 @@ class Repository(Serializable):
         Shallow clone this repository
         """
         if checkout_dir.exists():
-            logger.debug(f"Reusing {self.owner}:{self.name}")
+            logger.debug(f"Reusing cached {self.fullname}")
 
             if self.ref:
-                logger.debug(f"Checking out ref {self.ref}")
+                logger.debug(f"Checking out {self.fullname} @ {self.ref}")
+
                 process = await create_subprocess_exec(
                     *["git", "checkout", "-f", self.ref],
                     cwd=checkout_dir,
@@ -77,39 +156,68 @@ class Repository(Serializable):
                         f"Failed to checkout {self.ref}: {stderr.decode()}"
                     )
 
-            return await ClonedRepository.from_path(checkout_dir, self)
+            cloned_repo = await ClonedRepository.from_path(checkout_dir, self)
+            await cloned_repo.reset()
 
-        logger.debug(f"Cloning {self.owner}:{self.name} to {checkout_dir}")
-        command = [
-            "git",
-            "clone",
-            "--config",
-            "advice.detachedHead=false",
-            "--quiet",
-            "--depth",
-            "1",
-            "--no-tags",
-        ]
-        if self.ref:
-            command.extend(["--branch", self.ref])
+            logger.debug(f"Pulling latest changes for {self.fullname} @ {self.ref}")
+            await cloned_repo.pull()
 
-        command.extend(
-            [
-                f"https://github.com/{self.owner}/{self.name}",
-                str(checkout_dir),
-            ],
-        )
+        else:
+            logger.debug(f"Cloning {self.owner}:{self.name} to {checkout_dir}")
+            command = [
+                "git",
+                "clone",
+                "--config",
+                "advice.detachedHead=false",
+                "--quiet",
+                "--depth",
+                "1",
+                "--no-tags",
+            ]
+            if self.ref:
+                command.extend(["--branch", self.ref])
 
-        process = await create_subprocess_exec(
-            *command, env={"GIT_TERMINAL_PROMPT": "0"}
-        )
+            command.extend(
+                [
+                    f"https://github.com/{self.owner}/{self.name}",
+                    str(checkout_dir),
+                ],
+            )
 
-        status_code = await process.wait()
+            process = await create_subprocess_exec(
+                *command, env={"GIT_TERMINAL_PROMPT": "0"}
+            )
 
-        logger.debug(
-            f"Finished cloning {self.fullname} with status {status_code}",
-        )
-        return await ClonedRepository.from_path(checkout_dir, self)
+            status_code = await process.wait()
+
+            logger.debug(
+                f"Finished cloning {self.fullname} with status {status_code}",
+            )
+
+            cloned_repo = await ClonedRepository.from_path(checkout_dir, self)
+
+        # Configure git user — needed for `self.commit` to work
+        await (
+            await create_subprocess_exec(
+                *["git", "config", "user.email", "ecosystem@astral.sh"],
+                cwd=checkout_dir,
+                env={"GIT_TERMINAL_PROMPT": "0"},
+                stdout=DEVNULL,
+                stderr=DEVNULL,
+            )
+        ).wait()
+
+        await (
+            await create_subprocess_exec(
+                *["git", "config", "user.name", "Ecosystem Bot"],
+                cwd=checkout_dir,
+                env={"GIT_TERMINAL_PROMPT": "0"},
+                stdout=DEVNULL,
+                stderr=DEVNULL,
+            )
+        ).wait()
+
+        return cloned_repo
 
 
 @dataclass(frozen=True)
@@ -166,3 +274,73 @@ class ClonedRepository(Repository, Serializable):
             raise ProjectSetupError(f"Failed to retrieve commit sha at {checkout_dir}")
 
         return stdout.decode().strip()
+
+    async def reset(self: Self) -> None:
+        """
+        Reset the cloned repository to the ref it started at.
+        """
+        process = await create_subprocess_exec(
+            *["git", "reset", "--hard", "origin/" + self.ref] if self.ref else [],
+            cwd=self.path,
+            env={"GIT_TERMINAL_PROMPT": "0"},
+            stdout=PIPE,
+            stderr=PIPE,
+        )
+        _, stderr = await process.communicate()
+        if await process.wait() != 0:
+            raise RuntimeError(f"Failed to reset: {stderr.decode()}")
+
+    async def pull(self: Self) -> None:
+        """
+        Pull the latest changes.
+
+        Typically `reset` should be run first.
+        """
+        process = await create_subprocess_exec(
+            *["git", "pull"],
+            cwd=self.path,
+            env={"GIT_TERMINAL_PROMPT": "0"},
+            stdout=PIPE,
+            stderr=PIPE,
+        )
+        _, stderr = await process.communicate()
+        if await process.wait() != 0:
+            raise RuntimeError(f"Failed to pull: {stderr.decode()}")
+
+    async def commit(self: Self, message: str) -> str:
+        """
+        Commit all current changes.
+
+        Empty commits are allowed.
+        """
+        process = await create_subprocess_exec(
+            *["git", "commit", "--allow-empty", "-a", "-m", message],
+            cwd=self.path,
+            env={"GIT_TERMINAL_PROMPT": "0"},
+            stdout=PIPE,
+            stderr=PIPE,
+        )
+        _, stderr = await process.communicate()
+        if await process.wait() != 0:
+            raise RuntimeError(f"Failed to commit: {stderr.decode()}")
+
+        return await self._get_head_commit(self.path)
+
+    async def diff(self: Self, *args: str) -> list[str]:
+        """
+        Get the current diff from git.
+
+        Arguments are passed to `git diff ...`
+        """
+        process = await create_subprocess_exec(
+            *["git", "diff", *args],
+            cwd=self.path,
+            env={"GIT_TERMINAL_PROMPT": "0"},
+            stdout=PIPE,
+            stderr=PIPE,
+        )
+        stdout, stderr = await process.communicate()
+        if await process.wait() != 0:
+            raise RuntimeError(f"Failed to commit: {stderr.decode()}")
+
+        return stdout.decode().splitlines()

python/ruff-ecosystem/ruff_ecosystem/types.py

@@ -89,5 +89,5 @@ class Comparison(Serializable):
     repo: ClonedRepository
 
 
-class RuffError(Exception):
-    """An error reported by Ruff."""
+class ToolError(Exception):
+    """An error reported by the checked executable."""