diff --git a/.github/workflows/binstall.yml b/.github/workflows/binstall.yml
new file mode 100644
index 0000000..da83f82
--- /dev/null
+++ b/.github/workflows/binstall.yml
@@ -0,0 +1,34 @@
+name: binstall
+
+on: [push, pull_request]
+
+permissions: {}
+
+jobs:
+  binstall:
+    if: github.repository == 'Swatinem/rust-cache'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    name: Test `cargo binstall` on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+
+    env:
+      CARGO_TERM_COLOR: always
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - run: rustup toolchain install stable --profile minimal --no-self-update
+
+      - uses: cargo-bins/cargo-binstall@v1.17.9
+
+      - uses: ./
+
+      - run: cargo binstall -y cargo-deny
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/src/cleanup.ts b/src/cleanup.ts
index 67eafeb..8fe53c4 100644
--- a/src/cleanup.ts
+++ b/src/cleanup.ts
@@ -77,16 +77,16 @@ async function cleanProfileTarget(profileDir: string, packages: Packages, checkT
 
 export async function getCargoBins(): Promise<Set<string>> {
   const bins = new Set<string>();
+
   try {
-    const { installs }: { installs: { [key: string]: { bins: Array<string> } } } = JSON.parse(
-      await fs.promises.readFile(path.join(CARGO_HOME, ".crates2.json"), "utf8"),
-    );
-    for (const pkg of Object.values(installs)) {
-      for (const bin of pkg.bins) {
-        bins.add(bin);
+    const dir = await fs.promises.opendir(path.join(CARGO_HOME, "bin"));
+    for await (const dirent of dir) {
+      if (dirent.isFile()) {
+        bins.add(dirent.name);
       }
     }
   } catch {}
+
   return bins;
 }
 
@@ -97,15 +97,11 @@ export async function getCargoBins(): Promise<Set<string>> {
  * @param oldBins The binaries that existed when the action started.
  */
 export async function cleanBin(oldBins: Array<string>) {
-  const bins = await getCargoBins();
-
-  for (const bin of oldBins) {
-    bins.delete(bin);
-  }
+  const binsToRemove = new Set<string>(oldBins);
 
   const dir = await fs.promises.opendir(path.join(CARGO_HOME, "bin"));
   for await (const dirent of dir) {
-    if (dirent.isFile() && !bins.has(dirent.name)) {
+    if (dirent.isFile() && binsToRemove.has(dirent.name)) {
       await rm(dir.path, dirent);
     }
   }