diff --git a/.github/workflows/qf-s-benchmark.lock.yml b/.github/workflows/qf-s-benchmark.lock.yml
deleted file mode 100644
index e481c712dc..0000000000
--- a/.github/workflows/qf-s-benchmark.lock.yml
+++ /dev/null
@@ -1,1572 +0,0 @@
-# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d46f6f1f939df67ce5b7bf1e8073ba03c5631381df51d9711a1c2091400233f7","body_hash":"31e198b5f33dc3ac3830a2e6f90bb70ca3e7947b5ba4ee113ca0fdab61ac0467","compiler_version":"v0.81.6","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.65"}}
-# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"ba6380cc6e5be5d21677bebe04d52fb48e3abec7","version":"v0.81.6"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.11","digest":"sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.11@sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11","digest":"sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11@sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.11","digest":"sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.11@sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.30","digest":"sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.30@sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.4.0","digest":"sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036","pinned_image":"ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036"}]}
-# This file was automatically generated by gh-aw (v0.81.6). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md
-#
-# ___ _ _
-# / _ \ | | (_)
-# | |_| | __ _ ___ _ __ | |_ _ ___
-# | _ |/ _` |/ _ \ '_ \| __| |/ __|
-# | | | | (_| | __/ | | | |_| | (__
-# \_| |_/\__, |\___|_| |_|\__|_|\___|
-# __/ |
-# _ _ |___/
-# | | | | / _| |
-# | | | | ___ _ __ _ __| |_| | _____ ____
-# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___|
-# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \
-# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/
-#
-#
-# To update this file, edit the corresponding .md file and run:
-# gh aw compile
-# Not all edits will cause changes to this file.
-#
-# For more information: https://github.github.com/gh-aw/introduction/overview/
-#
-# Benchmark Z3 seq vs nseq string solvers on QF_S test suite from the c3 branch and post results as a GitHub discussion
-#
-# Secrets used:
-# - COPILOT_GITHUB_TOKEN
-# - GH_AW_GITHUB_MCP_SERVER_TOKEN
-# - GH_AW_GITHUB_TOKEN
-# - GITHUB_TOKEN
-#
-# Custom actions used:
-# - actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
-# - actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
-# - actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
-# - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
-# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9)
-# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
-# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-# - github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6
-#
-# Container images used:
-# - ghcr.io/github/gh-aw-firewall/agent:0.27.11@sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7
-# - ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11@sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d
-# - ghcr.io/github/gh-aw-firewall/squid:0.27.11@sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d
-# - ghcr.io/github/gh-aw-mcpg:v0.3.30@sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be
-# - ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b
-# - ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036
-
-name: "QF_S String Solver Benchmark"
-on:
- schedule:
- - cron: "0 0,12 * * *"
- workflow_dispatch:
- inputs:
- aw_context:
- default: ""
- description: "Agent caller context (used internally by Agentic Workflows)."
- required: false
- type: string
-
-permissions: {}
-
-concurrency:
- group: "gh-aw-${{ github.workflow }}"
-
-run-name: "QF_S String Solver Benchmark"
-
-jobs:
- activation:
- runs-on: ubuntu-slim
- permissions:
- actions: read
- contents: read
- env:
- GH_AW_MAX_DAILY_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_DAILY_AI_CREDITS || '5000' }}
- GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
- outputs:
- comment_id: ""
- comment_repo: ""
- daily_ai_credits_exceeded: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_exceeded == 'true' }}
- daily_ai_credits_threshold: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_threshold || '' }}
- daily_ai_credits_total_effective_tokens: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_total_effective_tokens || '' }}
- engine_id: ${{ steps.generate_aw_info.outputs.engine_id }}
- lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }}
- model: ${{ steps.generate_aw_info.outputs.model }}
- secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }}
- setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }}
- setup-span-id: ${{ steps.setup.outputs.span-id }}
- setup-trace-id: ${{ steps.setup.outputs.trace-id }}
- stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }}
- steps:
- - name: Setup Scripts
- id: setup
- uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6
- with:
- destination: ${{ runner.temp }}/gh-aw/actions
- job-name: ${{ github.job }}
- safe-output-artifact-client: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }}
- env:
- GH_AW_SETUP_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/qf-s-benchmark.lock.yml@${{ github.ref }}
- GH_AW_INFO_VERSION: "1.0.65"
- GH_AW_INFO_AWF_VERSION: "v0.27.11"
- GH_AW_INFO_ENGINE_ID: "copilot"
- - name: Generate agentic run info
- id: generate_aw_info
- env:
- GH_AW_INFO_ENGINE_ID: "copilot"
- GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI"
- GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }}
- GH_AW_INFO_VERSION: "1.0.65"
- GH_AW_INFO_AGENT_VERSION: "1.0.65"
- GH_AW_INFO_CLI_VERSION: "v0.81.6"
- GH_AW_INFO_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_INFO_EXPERIMENTAL: "false"
- GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true"
- GH_AW_INFO_STAGED: "false"
- GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]'
- GH_AW_INFO_FIREWALL_ENABLED: "true"
- GH_AW_INFO_AWF_VERSION: "v0.27.11"
- GH_AW_INFO_AWMG_VERSION: ""
- GH_AW_INFO_FIREWALL_TYPE: "squid"
- GH_AW_COMPILED_STRICT: "true"
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs');
- await main(core, context);
- - name: Restore daily AIC usage cache
- id: restore-daily-aic-cache
- if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }}
- continue-on-error: true
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- key: agentic-workflow-usage-qfsbenchmark-${{ github.run_id }}
- restore-keys: agentic-workflow-usage-qfsbenchmark-
- path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl
- - name: Restore daily AIC usage cache (artifact fallback)
- id: restore-daily-aic-cache-fallback
- if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }}
- continue-on-error: true
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_RESTORE_DAILY_AIC_CACHE_HIT: ${{ steps.restore-daily-aic-cache.outputs.cache-hit }}
- GH_AW_RESTORE_DAILY_AIC_CACHE_MATCHED_KEY: ${{ steps.restore-daily-aic-cache.outputs.cache-matched-key }}
- with:
- github-token: ${{ secrets.GITHUB_TOKEN }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/restore_aic_usage_cache_fallback.cjs');
- await main();
- - name: Check daily workflow token guardrail
- id: daily-effective-workflow-guardrail
- if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }}
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_WORKFLOW_ID: "qf-s-benchmark"
- GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- GH_AW_WORKFLOW_DISPATCH_AW_CONTEXT: ${{ github.event.inputs.aw_context || '' }}
- GH_AW_HAS_SLASH_COMMAND: "false"
- GH_AW_HAS_LABEL_COMMAND: "false"
- GH_AW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- GH_AW_MAX_DAILY_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_DAILY_AI_CREDITS || '5000' }}
- with:
- github-token: ${{ secrets.GITHUB_TOKEN }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/check_daily_aic_workflow_guardrail.cjs');
- await main();
- - name: Validate COPILOT_GITHUB_TOKEN secret
- id: validate-secret
- run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default
- env:
- COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
- - name: Checkout .github and .agents folders
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- persist-credentials: false
- sparse-checkout: |
- .github
- .agents
- .antigravity
- .claude
- .codex
- .crush
- .gemini
- .opencode
- .pi
- sparse-checkout-cone-mode: true
- fetch-depth: 1
- - name: Save agent config folders for base branch restoration
- env:
- GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi"
- GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc"
- # poutine:ignore untrusted_checkout_exec
- run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh"
- - name: Check workflow lock file
- id: check-lock-file
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_WORKFLOW_FILE: "qf-s-benchmark.lock.yml"
- GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}"
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs');
- await main();
- - name: Check compile-agentic version
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_COMPILED_VERSION: "v0.81.6"
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs');
- await main();
- - name: Log runtime features
- if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }}
- run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh"
- - name: Create prompt with built-in context
- env:
- GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
- GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl
- GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
- GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
- GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
- GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }}
- GH_AW_GITHUB_ACTOR: ${{ github.actor }}
- GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
- GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
- GH_AW_GITHUB_WORKFLOW: ${{ github.workflow }}
- GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }}
- # poutine:ignore untrusted_checkout_exec
- run: |
- bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh"
- {
- cat << 'GH_AW_PROMPT_5a6a0f70a2d06d17_EOF'
-
- GH_AW_PROMPT_5a6a0f70a2d06d17_EOF
- cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md"
- cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md"
- cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md"
- cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md"
- cat << 'GH_AW_PROMPT_5a6a0f70a2d06d17_EOF'
-
- Tools: create_discussion, missing_tool, missing_data, noop
-
- GH_AW_PROMPT_5a6a0f70a2d06d17_EOF
- cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md"
- cat << 'GH_AW_PROMPT_5a6a0f70a2d06d17_EOF'
-
- The following GitHub context information is available for this workflow:
- {{#if github.actor}}
- - **actor**: __GH_AW_GITHUB_ACTOR__
- {{/if}}
- {{#if github.repository}}
- - **repository**: __GH_AW_GITHUB_REPOSITORY__
- {{/if}}
- {{#if github.workspace}}
- - **workspace**: __GH_AW_GITHUB_WORKSPACE__
- {{/if}}
- {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}}
- - **issue-number**: #__GH_AW_EXPR_802A9F6A__
- {{/if}}
- {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}}
- - **discussion-number**: #__GH_AW_EXPR_1A3A194A__
- {{/if}}
- {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}}
- - **pull-request-number**: #__GH_AW_EXPR_463A214A__
- {{/if}}
- {{#if github.event.comment.id || github.aw.context.comment_id}}
- - **comment-id**: __GH_AW_EXPR_FF1D34CE__
- {{/if}}
- {{#if github.run_id}}
- - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__
- {{/if}}
-
-
- GH_AW_PROMPT_5a6a0f70a2d06d17_EOF
- cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md"
- cat << 'GH_AW_PROMPT_5a6a0f70a2d06d17_EOF'
-
- {{#runtime-import .github/workflows/qf-s-benchmark.md}}
- GH_AW_PROMPT_5a6a0f70a2d06d17_EOF
- } > "$GH_AW_PROMPT"
- - name: Interpolate variables and render templates
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
- GH_AW_ENGINE_ID: "copilot"
- GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
- GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
- GH_AW_GITHUB_WORKFLOW: ${{ github.workflow }}
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs');
- await main();
- - name: Substitute placeholders
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
- GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
- GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
- GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
- GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }}
- GH_AW_GITHUB_ACTOR: ${{ github.actor }}
- GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
- GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
- GH_AW_GITHUB_WORKFLOW: ${{ github.workflow }}
- GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }}
- GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools'
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
-
- const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs');
-
- // Call the substitution function
- return await substitutePlaceholders({
- file: process.env.GH_AW_PROMPT,
- substitutions: {
- GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A,
- GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A,
- GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A,
- GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE,
- GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR,
- GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY,
- GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID,
- GH_AW_GITHUB_WORKFLOW: process.env.GH_AW_GITHUB_WORKFLOW,
- GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE,
- GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST
- }
- });
- - name: Validate prompt placeholders
- env:
- GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
- # poutine:ignore untrusted_checkout_exec
- run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh"
- - name: Print prompt
- env:
- GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
- # poutine:ignore untrusted_checkout_exec
- run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh"
- - name: Upload activation artifact
- if: success()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: activation
- include-hidden-files: true
- path: |
- /tmp/gh-aw/aw_info.json
- /tmp/gh-aw/models.json
- /tmp/gh-aw/aw-prompts/prompt.txt
- /tmp/gh-aw/aw-prompts/prompt-template.txt
- /tmp/gh-aw/aw-prompts/prompt-import-tree.json
- /tmp/gh-aw/github_rate_limits.jsonl
- /tmp/gh-aw/base
- /tmp/gh-aw/.github/agents
- /tmp/gh-aw/.github/skills
- if-no-files-found: ignore
- retention-days: 1
-
- agent:
- needs: activation
- if: needs.activation.outputs.daily_ai_credits_exceeded != 'true'
- runs-on: ubuntu-latest
- permissions: read-all
- concurrency:
- group: "gh-aw-copilot-${{ github.workflow }}"
- queue: max
- env:
- DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
- GH_AW_ASSETS_ALLOWED_EXTS: ""
- GH_AW_ASSETS_BRANCH: ""
- GH_AW_ASSETS_MAX_SIZE_KB: 0
- GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs
- GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
- GH_AW_WORKFLOW_ID_SANITIZED: qfsbenchmark
- outputs:
- agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }}
- ai_credits_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.ai_credits_rate_limit_error || 'false' }}
- aic: ${{ steps.parse-mcp-gateway.outputs.aic }}
- ambient_context: ${{ steps.parse-mcp-gateway.outputs.ambient_context }}
- checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }}
- effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }}
- has_patch: ${{ steps.collect_output.outputs.has_patch }}
- inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }}
- mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }}
- model: ${{ needs.activation.outputs.model }}
- model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }}
- output: ${{ steps.collect_output.outputs.output }}
- output_types: ${{ steps.collect_output.outputs.output_types }}
- setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }}
- setup-span-id: ${{ steps.setup.outputs.span-id }}
- setup-trace-id: ${{ steps.setup.outputs.trace-id }}
- unknown_model_ai_credits: ${{ steps.parse-mcp-gateway.outputs.unknown_model_ai_credits || 'false' }}
- steps:
- - name: Setup Scripts
- id: setup
- uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6
- with:
- destination: ${{ runner.temp }}/gh-aw/actions
- job-name: ${{ github.job }}
- trace-id: ${{ needs.activation.outputs.setup-trace-id }}
- parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }}
- env:
- GH_AW_SETUP_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/qf-s-benchmark.lock.yml@${{ github.ref }}
- GH_AW_INFO_VERSION: "1.0.65"
- GH_AW_INFO_AWF_VERSION: "v0.27.11"
- GH_AW_INFO_ENGINE_ID: "copilot"
- - name: Set runtime paths
- id: set-runtime-paths
- run: |
- {
- echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl"
- echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json"
- echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json"
- } >> "$GITHUB_OUTPUT"
- - name: Create gh-aw temp directory
- run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh"
- - name: Configure gh CLI for GitHub Enterprise
- run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh"
- env:
- GH_TOKEN: ${{ github.token }}
- - name: Checkout c3 branch
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- with:
- fetch-depth: 1
- persist-credentials: false
- ref: c3
-
- - name: Configure Git credentials
- env:
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_SERVER_URL: ${{ github.server_url }}
- GITHUB_TOKEN: ${{ github.token }}
- run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh"
- - name: Checkout PR branch
- id: checkout-pr
- if: |
- github.event.pull_request || github.event.issue.pull_request || github.event_name == 'workflow_dispatch' && fromJSON(github.event.inputs.aw_context || '{}').item_type == 'pull_request'
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- with:
- github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs');
- await main();
- - name: Install GitHub Copilot CLI
- run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.65
- env:
- GH_HOST: github.com
- - name: Install AWF binary
- run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.11
- - name: Determine automatic lockdown mode for GitHub MCP Server
- id: determine-automatic-lockdown
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9)
- env:
- GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }}
- GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }}
- with:
- script: |
- const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs');
- await determineAutomaticLockdown(github, context, core);
- - name: Download activation artifact
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- name: activation
- path: /tmp/gh-aw
- - name: Restore agent config folders from base branch
- if: steps.checkout-pr.outcome == 'success'
- env:
- GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi"
- GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc"
- run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh"
- - name: Restore inline sub-agents from activation artifact
- env:
- GH_AW_SUB_AGENT_DIR: ".github/agents"
- GH_AW_SUB_AGENT_EXT: ".agent.md"
- run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh"
- - name: Restore inline skills from activation artifact
- env:
- GH_AW_SKILL_DIR: ".github/skills"
- run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh"
- - name: Download container images
- run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.11@sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11@sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d ghcr.io/github/gh-aw-firewall/squid:0.27.11@sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d ghcr.io/github/gh-aw-mcpg:v0.3.30@sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036
- - name: Generate Safe Outputs Config
- run: |
- mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs"
- mkdir -p /tmp/gh-aw/safeoutputs
- mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs
- cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_1bab4c882b24ea44_EOF'
- {"create_discussion":{"category":"agentic workflows","close_older_discussions":true,"expires":168,"fallback_to_issue":true,"max":1,"title_prefix":"[QF_S Benchmark] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{}}
- GH_AW_SAFE_OUTPUTS_CONFIG_1bab4c882b24ea44_EOF
- - name: Generate Safe Outputs Tools
- env:
- GH_AW_TOOLS_META_JSON: |
- {
- "description_suffixes": {
- "create_discussion": " CONSTRAINTS: Maximum 1 discussion(s) can be created. Title will be prefixed with \"[QF_S Benchmark] \". Discussions will be created in category \"agentic workflows\"."
- },
- "repo_params": {},
- "dynamic_tools": []
- }
- GH_AW_VALIDATION_JSON: |
- {
- "create_discussion": {
- "defaultMax": 1,
- "fields": {
- "body": {
- "required": true,
- "type": "string",
- "sanitize": true,
- "maxLength": 65000,
- "minLength": 64
- },
- "category": {
- "type": "string",
- "sanitize": true,
- "maxLength": 128
- },
- "repo": {
- "type": "string",
- "maxLength": 256
- },
- "title": {
- "required": true,
- "type": "string",
- "sanitize": true,
- "maxLength": 128
- }
- }
- },
- "missing_data": {
- "defaultMax": 20,
- "fields": {
- "alternatives": {
- "type": "string",
- "sanitize": true,
- "maxLength": 256
- },
- "context": {
- "type": "string",
- "sanitize": true,
- "maxLength": 256
- },
- "data_type": {
- "type": "string",
- "sanitize": true,
- "maxLength": 128
- },
- "reason": {
- "type": "string",
- "sanitize": true,
- "maxLength": 256
- }
- }
- },
- "missing_tool": {
- "defaultMax": 20,
- "fields": {
- "alternatives": {
- "type": "string",
- "sanitize": true,
- "maxLength": 512
- },
- "reason": {
- "required": true,
- "type": "string",
- "sanitize": true,
- "maxLength": 256
- },
- "tool": {
- "type": "string",
- "sanitize": true,
- "maxLength": 128
- }
- }
- },
- "noop": {
- "defaultMax": 1,
- "fields": {
- "message": {
- "required": true,
- "type": "string",
- "sanitize": true,
- "maxLength": 65000
- }
- }
- },
- "report_incomplete": {
- "defaultMax": 5,
- "fields": {
- "details": {
- "type": "string",
- "sanitize": true,
- "maxLength": 65000
- },
- "reason": {
- "required": true,
- "type": "string",
- "sanitize": true,
- "maxLength": 1024
- }
- }
- }
- }
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs');
- await main();
- - name: Start MCP Gateway
- id: start-mcp-gateway
- env:
- GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST: ${{ vars.GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST || 'true' }}
- GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
- GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }}
- GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }}
- GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }}
- GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }}
- GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- set -eo pipefail
- mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config"
-
- # Export gateway environment variables for MCP config and gateway script
- export MCP_GATEWAY_PORT="8080"
- export MCP_GATEWAY_DOMAIN="host.docker.internal"
- export MCP_GATEWAY_HOST_DOMAIN="localhost"
- MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=')
- echo "::add-mask::${MCP_GATEWAY_API_KEY}"
- export MCP_GATEWAY_API_KEY
- export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads"
- mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}"
- export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288"
- export DEBUG="*"
-
- export GH_AW_ENGINE="copilot"
- MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0')
- MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0')
- case "${DOCKER_HOST:-}" in
- unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;;
- /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;;
- * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;;
- esac
- DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0')
- export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --name awmg-mcpg --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.3.30'
-
- mkdir -p "$HOME/.copilot"
- GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node)
- cat << GH_AW_MCP_CONFIG_f014db59cae17bc3_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs"
- {
- "mcpServers": {
- "github": {
- "type": "stdio",
- "container": "ghcr.io/github/github-mcp-server:v1.4.0",
- "env": {
- "GITHUB_HOST": "${GITHUB_SERVER_URL}",
- "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_MCP_SERVER_TOKEN}",
- "GITHUB_READ_ONLY": "1",
- "GITHUB_TOOLSETS": "context,repos,issues,pull_requests"
- },
- "guard-policies": {
- "allow-only": {
- "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY",
- "repos": "$GITHUB_MCP_GUARD_REPOS"
- }
- }
- },
- "safeoutputs": {
- "type": "stdio",
- "container": "ghcr.io/github/gh-aw-node",
- "mounts": ["\${GITHUB_WORKSPACE}:\${GITHUB_WORKSPACE}:rw", "${RUNNER_TEMP}/gh-aw/safeoutputs:${RUNNER_TEMP}/gh-aw/safeoutputs:rw", "/tmp/gh-aw:/tmp/gh-aw:rw"],
- "args": ["-w", "\${GITHUB_WORKSPACE}"],
- "entrypoint": "sh",
- "entrypointArgs": ["-c", "sh ${RUNNER_TEMP}/gh-aw/safeoutputs/start_safe_outputs_mcp.sh"],
- "env": {
- "DEBUG": "*",
- "DEFAULT_BRANCH": "\${DEFAULT_BRANCH}",
- "GH_AW_ASSETS_ALLOWED_EXTS": "\${GH_AW_ASSETS_ALLOWED_EXTS}",
- "GH_AW_ASSETS_BRANCH": "\${GH_AW_ASSETS_BRANCH}",
- "GH_AW_ASSETS_MAX_SIZE_KB": "\${GH_AW_ASSETS_MAX_SIZE_KB}",
- "GH_AW_MCP_LOG_DIR": "\${GH_AW_MCP_LOG_DIR}",
- "GH_AW_SAFE_OUTPUTS": "\${GH_AW_SAFE_OUTPUTS}",
- "GH_AW_SAFE_OUTPUTS_CONFIG_PATH": "\${GH_AW_SAFE_OUTPUTS_CONFIG_PATH}",
- "GH_AW_SAFE_OUTPUTS_TOOLS_PATH": "\${GH_AW_SAFE_OUTPUTS_TOOLS_PATH}",
- "GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST": "\${GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST}",
- "GITHUB_REPOSITORY": "\${GITHUB_REPOSITORY}",
- "GITHUB_TOKEN": "\${GITHUB_TOKEN}",
- "GITHUB_WORKSPACE": "\${GITHUB_WORKSPACE}",
- "RUNNER_TEMP": "\${RUNNER_TEMP}"
- },
- "guard-policies": {
- "write-sink": {
- "accept": [
- "*"
- ]
- }
- }
- }
- },
- "gateway": {
- "port": $MCP_GATEWAY_PORT,
- "domain": "${MCP_GATEWAY_DOMAIN}",
- "apiKey": "${MCP_GATEWAY_API_KEY}",
- "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}"
- }
- }
- GH_AW_MCP_CONFIG_f014db59cae17bc3_EOF
- - name: Mount MCP servers as CLIs
- id: mount-mcp-clis
- continue-on-error: true
- env:
- MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }}
- MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }}
- MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }}
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs');
- await main();
- - name: Clean credentials
- continue-on-error: true
- run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh"
- - name: Audit pre-agent workspace
- id: pre_agent_audit
- continue-on-error: true
- run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh"
- - name: Execute GitHub Copilot CLI
- id: agentic_execution
- # Copilot CLI tool arguments (sorted):
- timeout-minutes: 120
- run: |
- set -o pipefail
- printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt
- trap 'rm -f "$HOME/.copilot/settings.json"' EXIT
- mkdir -p "$HOME/.copilot"
- printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json"
- export XDG_CONFIG_HOME="$HOME"
- export GH_AW_MCP_CONFIG="$HOME/.copilot/mcp-config.json"
- touch /tmp/gh-aw/agent-step-summary.md
- GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true)
- export GH_AW_NODE_BIN
- export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK"
- (umask 177 && touch /tmp/gh-aw/agent-stdio.log)
- GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-1000}"
- printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.11/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"github.com\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"ppa.launchpad.net\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"telemetry.enterprise.githubcopilot.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\",\"www.googleapis.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.5\",\"gpt-5.4\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.11,squid=sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d,agent=sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7,api-proxy=sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json"
- cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json
- export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json"
- GH_AW_DOCKER_HOST=""
- if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then
- GH_AW_DOCKER_HOST="${DOCKER_HOST}"
- fi
- GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS=""
- if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then
- GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw"
- GH_AW_CHROOT_BINARIES_SOURCE_PATH=/tmp/gh-aw GH_AW_CHROOT_IDENTITY_HOME=/tmp/gh-aw/home node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs"
- fi
- GH_AW_TOOL_CACHE_MOUNT=""
- GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"
- if [ -d "$GH_AW_TOOL_CACHE" ]; then
- if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then
- GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro"
- fi
- fi
- # shellcheck disable=SC1003,SC2086
- sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \
- -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log
- env:
- AWF_REFLECT_ENABLED: 1
- COPILOT_AGENT_RUNNER_TYPE: STANDALONE
- COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode
- COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
- COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }}
- GH_AW_LLM_PROVIDER: github
- GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }}
- GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }}
- GH_AW_PHASE: agent
- GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
- GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
- GH_AW_TIMEOUT_MINUTES: 120
- GH_AW_VERSION: v0.81.6
- GITHUB_API_URL: ${{ github.api_url }}
- GITHUB_AW: true
- GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows
- GITHUB_HEAD_REF: ${{ github.head_ref }}
- GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- GITHUB_REF_NAME: ${{ github.ref_name }}
- GITHUB_SERVER_URL: ${{ github.server_url }}
- GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md
- GITHUB_WORKSPACE: ${{ github.workspace }}
- GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com
- GIT_AUTHOR_NAME: github-actions[bot]
- GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com
- GIT_COMMITTER_NAME: github-actions[bot]
- RUNNER_TEMP: ${{ runner.temp }}
- TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }}
- - name: Detect agent errors
- if: always()
- id: detect-agent-errors
- continue-on-error: true
- run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs"
- - name: Configure Git credentials
- env:
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_SERVER_URL: ${{ github.server_url }}
- GITHUB_TOKEN: ${{ github.token }}
- run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh"
- - name: Copy Copilot session state files to logs
- if: always()
- continue-on-error: true
- run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh"
- - name: Stop MCP Gateway
- if: always()
- continue-on-error: true
- env:
- MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }}
- MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }}
- GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }}
- run: |
- bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID"
- - name: Redact secrets in logs
- if: always()
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs');
- await main();
- env:
- GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN'
- SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
- SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }}
- SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }}
- SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Append agent step summary
- if: always()
- run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh"
- - name: Copy Safe Outputs
- if: always()
- env:
- GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
- run: |
- mkdir -p /tmp/gh-aw
- cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true
- - name: Ingest agent output
- id: collect_output
- if: always()
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
- GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com"
- GITHUB_SERVER_URL: ${{ github.server_url }}
- GITHUB_API_URL: ${{ github.api_url }}
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs');
- await main();
- - name: Parse agent logs for step summary
- if: always()
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs');
- await main();
- - name: Parse MCP Gateway logs for step summary
- if: always()
- id: parse-mcp-gateway
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs');
- await main();
- - name: Print firewall logs
- if: always()
- continue-on-error: true
- env:
- AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs
- run: |
- # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts
- # AWF runs with sudo, creating files owned by root
- sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true
- # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step)
- if command -v awf &> /dev/null; then
- awf logs summary | tee -a "$GITHUB_STEP_SUMMARY"
- else
- echo 'AWF binary not installed, skipping firewall log summary'
- fi
- - name: Parse token usage for step summary
- if: always()
- continue-on-error: true
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs');
- await main();
- - name: Print AWF reflect summary
- if: always()
- continue-on-error: true
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs');
- await main();
- - name: Write agent output placeholder if missing
- if: always()
- run: |
- if [ ! -f /tmp/gh-aw/agent_output.json ]; then
- echo '{"items":[]}' > /tmp/gh-aw/agent_output.json
- fi
- - name: Upload agent artifacts
- if: always()
- continue-on-error: true
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: agent
- path: |
- /tmp/gh-aw/aw-prompts/prompt.txt
- /tmp/gh-aw/sandbox/agent/logs/
- /tmp/gh-aw/redacted-urls.log
- /tmp/gh-aw/mcp-logs/
- /tmp/gh-aw/agent_usage.json
- /tmp/gh-aw/agent-stdio.log
- /tmp/gh-aw/pre-agent-audit.txt
- /tmp/gh-aw/agent/
- /tmp/gh-aw/github_rate_limits.jsonl
- /tmp/gh-aw/safeoutputs.jsonl
- /tmp/gh-aw/agent_output.json
- /tmp/gh-aw/aw-*.patch
- /tmp/gh-aw/aw-*.bundle
- /tmp/gh-aw/awf-config.json
- /tmp/gh-aw/sandbox/firewall/logs/
- /tmp/gh-aw/sandbox/firewall/audit/
- /tmp/gh-aw/sandbox/firewall/awf-reflect.json
- if-no-files-found: ignore
-
- conclusion:
- needs:
- - activation
- - agent
- - detection
- - safe_outputs
- if: >
- always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' ||
- needs.activation.outputs.stale_lock_file_failed == 'true' || needs.activation.outputs.daily_ai_credits_exceeded == 'true')
- runs-on: ubuntu-slim
- permissions:
- contents: read
- discussions: write
- issues: write
- concurrency:
- group: "gh-aw-conclusion-qf-s-benchmark"
- cancel-in-progress: false
- queue: max
- env:
- GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
- outputs:
- incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }}
- noop_message: ${{ steps.noop.outputs.noop_message }}
- tools_reported: ${{ steps.missing_tool.outputs.tools_reported }}
- total_count: ${{ steps.missing_tool.outputs.total_count }}
- steps:
- - name: Setup Scripts
- id: setup
- uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6
- with:
- destination: ${{ runner.temp }}/gh-aw/actions
- job-name: ${{ github.job }}
- trace-id: ${{ needs.activation.outputs.setup-trace-id }}
- parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }}
- env:
- GH_AW_SETUP_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/qf-s-benchmark.lock.yml@${{ github.ref }}
- GH_AW_INFO_VERSION: "1.0.65"
- GH_AW_INFO_AWF_VERSION: "v0.27.11"
- GH_AW_INFO_ENGINE_ID: "copilot"
- - name: Download agent output artifact
- id: download-agent-output
- continue-on-error: true
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- name: agent
- path: /tmp/gh-aw/
- - name: Setup agent output environment variable
- id: setup-agent-output-env
- if: steps.download-agent-output.outcome == 'success'
- run: |
- mkdir -p /tmp/gh-aw/
- find "/tmp/gh-aw/" -type f -print
- echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT"
- - name: Collect usage artifact files
- if: always()
- continue-on-error: true
- run: |
- mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection
- echo "Usage artifact source file status:"
- for file in /tmp/gh-aw/aw_info.json /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.json /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do
- [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file"
- done
- [ -f /tmp/gh-aw/aw_info.json ] && cp /tmp/gh-aw/aw_info.json /tmp/gh-aw/usage/aw_info.json || true
- [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true
- [ -f /tmp/gh-aw/agent_usage.json ] && cp /tmp/gh-aw/agent_usage.json /tmp/gh-aw/usage/agent_usage.json || true
- [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true
- [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true
- [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true
- [ -s /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true
- [ -s /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true
- [ -s /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true
- [ -s /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true
- [ -s /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true
- [ -s /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true
- [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl
- [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl
- mkdir -p /tmp/gh-aw/usage/activity
- node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs
- find /tmp/gh-aw/usage -type f -print | sort
- - name: Upload usage artifact
- if: always()
- continue-on-error: true
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: usage
- path: |
- /tmp/gh-aw/usage/aw_info.json
- /tmp/gh-aw/usage/aw-info.jsonl
- /tmp/gh-aw/usage/agent_usage.json
- /tmp/gh-aw/usage/agent_usage.jsonl
- /tmp/gh-aw/usage/detection_usage.jsonl
- /tmp/gh-aw/usage/github_rate_limits.jsonl
- /tmp/gh-aw/usage/agent/token_usage.jsonl
- /tmp/gh-aw/usage/detection/token_usage.jsonl
- /tmp/gh-aw/usage/activity/summary.json
- if-no-files-found: ignore
- - name: Restore daily AIC usage cache
- id: restore-daily-aic-cache-conclusion
- if: always()
- continue-on-error: true
- uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- key: agentic-workflow-usage-qfsbenchmark-${{ github.run_id }}
- restore-keys: agentic-workflow-usage-qfsbenchmark-
- path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl
- - name: Write daily AIC usage cache entry
- id: write-daily-aic-cache
- if: always()
- continue-on-error: true
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- with:
- github-token: ${{ github.token }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/write_daily_aic_usage_cache.cjs');
- await main();
- - name: Save daily AIC usage cache
- id: save-daily-aic-cache
- if: always()
- continue-on-error: true
- uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
- with:
- key: agentic-workflow-usage-qfsbenchmark-${{ github.run_id }}
- path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl
- - name: Upload daily AIC usage cache artifact
- id: upload-daily-aic-cache
- if: always()
- continue-on-error: true
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: aic-usage-cache
- path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl
- if-no-files-found: ignore
- retention-days: 7
- - name: Process no-op messages
- id: noop
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
- GH_AW_NOOP_MAX: "1"
- GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/qf-s-benchmark.md"
- GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }}
- GH_AW_NOOP_REPORT_AS_ISSUE: "false"
- GH_AW_AIC: ${{ needs.agent.outputs.aic }}
- GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }}
- GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }}
- GH_AW_WORKFLOW_ID: "qf-s-benchmark"
- with:
- github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs');
- await main();
- - name: Log detection run
- id: detection_runs
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
- GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/qf-s-benchmark.md"
- GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }}
- GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }}
- with:
- github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs');
- await main();
- - name: Record missing tool
- id: missing_tool
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
- GH_AW_MISSING_TOOL_CREATE_ISSUE: "true"
- GH_AW_MISSING_TOOL_TITLE_PREFIX: "[missing tool]"
- GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/qf-s-benchmark.md"
- with:
- github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs');
- await main();
- - name: Record incomplete
- id: report_incomplete
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
- GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true"
- GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/qf-s-benchmark.md"
- with:
- github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs');
- await main();
- - name: Handle agent failure
- id: handle_agent_failure
- if: always()
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
- GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/qf-s-benchmark.md"
- GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }}
- GH_AW_WORKFLOW_ID: "qf-s-benchmark"
- GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168"
- GH_AW_ENGINE_ID: "copilot"
- GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }}
- GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }}
- GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }}
- GH_AW_AI_CREDITS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.ai_credits_rate_limit_error || 'false' }}
- GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }}
- GH_AW_AIC: ${{ needs.agent.outputs.aic }}
- GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }}
- GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }}
- GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }}
- GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }}
- GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }}
- GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }}
- GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com"
- GH_AW_CREATE_DISCUSSION_ERRORS: ${{ needs.safe_outputs.outputs.create_discussion_errors }}
- GH_AW_CREATE_DISCUSSION_ERROR_COUNT: ${{ needs.safe_outputs.outputs.create_discussion_error_count }}
- GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }}
- GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }}
- GH_AW_DAILY_AI_CREDITS_EXCEEDED: ${{ needs.activation.outputs.daily_ai_credits_exceeded }}
- GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS: ${{ needs.activation.outputs.daily_ai_credits_total_effective_tokens }}
- GH_AW_DAILY_AI_CREDITS_THRESHOLD: ${{ needs.activation.outputs.daily_ai_credits_threshold }}
- GH_AW_GROUP_REPORTS: "false"
- GH_AW_FAILURE_REPORT_AS_ISSUE: "false"
- GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true"
- GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true"
- GH_AW_TIMEOUT_MINUTES: "120"
- with:
- github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs');
- await main();
-
- detection:
- needs:
- - activation
- - agent
- if: always() && needs.agent.result != 'skipped'
- runs-on: ubuntu-latest
- permissions:
- contents: read
- env:
- GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
- outputs:
- aic: ${{ steps.parse_detection_token_usage.outputs.aic }}
- detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }}
- detection_reason: ${{ steps.detection_conclusion.outputs.reason }}
- detection_success: ${{ steps.detection_conclusion.outputs.success }}
- steps:
- - name: Setup Scripts
- id: setup
- uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6
- with:
- destination: ${{ runner.temp }}/gh-aw/actions
- job-name: ${{ github.job }}
- trace-id: ${{ needs.activation.outputs.setup-trace-id }}
- parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }}
- env:
- GH_AW_SETUP_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/qf-s-benchmark.lock.yml@${{ github.ref }}
- GH_AW_INFO_VERSION: "1.0.65"
- GH_AW_INFO_AWF_VERSION: "v0.27.11"
- GH_AW_INFO_ENGINE_ID: "copilot"
- - name: Download agent output artifact
- id: download-agent-output
- continue-on-error: true
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- name: agent
- path: /tmp/gh-aw/
- - name: Setup agent output environment variable
- id: setup-agent-output-env
- if: steps.download-agent-output.outcome == 'success'
- run: |
- mkdir -p /tmp/gh-aw/
- find "/tmp/gh-aw/" -type f -print
- echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT"
- - name: Checkout repository for patch context
- if: needs.agent.outputs.has_patch == 'true'
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- with:
- persist-credentials: false
- # --- Threat Detection ---
- - name: Clean stale firewall files from agent artifact
- run: |
- rm -rf /tmp/gh-aw/sandbox/firewall/logs
- rm -rf /tmp/gh-aw/sandbox/firewall/audit
- - name: Download container images
- run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.11@sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11@sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d ghcr.io/github/gh-aw-firewall/squid:0.27.11@sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d
- - name: Check if detection needed
- id: detection_guard
- if: always()
- env:
- OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }}
- HAS_PATCH: ${{ needs.agent.outputs.has_patch }}
- run: |
- if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then
- echo "run_detection=true" >> "$GITHUB_OUTPUT"
- echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH"
- else
- echo "run_detection=false" >> "$GITHUB_OUTPUT"
- echo "Detection skipped: no agent outputs or patches to analyze"
- fi
- - name: Clear MCP Config for detection
- if: always() && steps.detection_guard.outputs.run_detection == 'true'
- run: |
- rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json"
- rm -f "$HOME/.copilot/mcp-config.json"
- rm -f "$GITHUB_WORKSPACE/.gemini/settings.json"
- - name: Prepare threat detection files
- if: always() && steps.detection_guard.outputs.run_detection == 'true'
- run: |
- mkdir -p /tmp/gh-aw/threat-detection/aw-prompts
- rm -f /tmp/gh-aw/agent_usage.json
- cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true
- if [ ! -s /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt ]; then
- echo "::warning::ERR_VALIDATION: Missing or empty detection context prompt at /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt. Ensure the agent artifact includes /tmp/gh-aw/aw-prompts/prompt.txt. Detection will continue with fallback workflow context."
- fi
- cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true
- for f in /tmp/gh-aw/aw-*.patch; do
- [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true
- done
- for f in /tmp/gh-aw/aw-*.bundle; do
- [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true
- done
- echo "Prepared threat detection files:"
- ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true
- - name: Setup threat detection
- if: always() && steps.detection_guard.outputs.run_detection == 'true'
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- WORKFLOW_NAME: "QF_S String Solver Benchmark"
- WORKFLOW_DESCRIPTION: "Benchmark Z3 seq vs nseq string solvers on QF_S test suite from the c3 branch and post results as a GitHub discussion"
- HAS_PATCH: ${{ needs.agent.outputs.has_patch }}
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs');
- await main();
- - name: Ensure threat-detection directory and log
- if: always() && steps.detection_guard.outputs.run_detection == 'true'
- run: |
- mkdir -p /tmp/gh-aw/threat-detection
- touch /tmp/gh-aw/threat-detection/detection.log
- - name: Setup Node.js
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
- with:
- node-version: '24'
- package-manager-cache: false
- - name: Install GitHub Copilot CLI
- run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.65
- env:
- GH_HOST: github.com
- - name: Install AWF binary
- run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.11
- - name: Execute GitHub Copilot CLI
- if: always() && steps.detection_guard.outputs.run_detection == 'true'
- continue-on-error: true
- id: detection_agentic_execution
- # Copilot CLI tool arguments (sorted):
- timeout-minutes: 20
- run: |
- set -o pipefail
- printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt
- trap 'rm -f "$HOME/.copilot/settings.json"' EXIT
- mkdir -p "$HOME/.copilot"
- printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json"
- export XDG_CONFIG_HOME="$HOME"
- touch /tmp/gh-aw/agent-step-summary.md
- GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true)
- export GH_AW_NODE_BIN
- export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK"
- (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log)
- GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}"
- printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.11/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5},\"container\":{\"imageTag\":\"0.27.11,squid=sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d,agent=sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7,api-proxy=sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json"
- cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json
- export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json"
- GH_AW_DOCKER_HOST=""
- if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then
- GH_AW_DOCKER_HOST="${DOCKER_HOST}"
- fi
- GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS=""
- if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then
- GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw"
- _GH_AW_CHROOT_JSON=$(jq -c --arg src /tmp/gh-aw --arg user "$(id -un)" --argjson uid "$(id -u)" --argjson gid "$(id -g)" --arg home /tmp/gh-aw/home '.chroot={"binariesSourcePath":$src,"identity":{"user":$user,"uid":$uid,"gid":$gid,"home":$home}}' "${RUNNER_TEMP}/gh-aw/awf-config.json") || { echo "chroot config patch failed" >&2; exit 1; }
- printf '%s\n' "$_GH_AW_CHROOT_JSON" > "${RUNNER_TEMP}/gh-aw/awf-config.json"
- printf '%s\n' "$_GH_AW_CHROOT_JSON" > "/tmp/gh-aw/awf-config.json"
- fi
- GH_AW_TOOL_CACHE_MOUNT=""
- GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"
- if [ -d "$GH_AW_TOOL_CACHE" ]; then
- if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then
- GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro"
- fi
- fi
- # shellcheck disable=SC1003,SC2086
- sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \
- -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log
- env:
- AWF_REFLECT_ENABLED: 1
- COPILOT_AGENT_RUNNER_TYPE: STANDALONE
- COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode
- COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
- COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }}
- GH_AW_LLM_PROVIDER: github
- GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS || '400' }}
- GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }}
- GH_AW_PHASE: detection
- GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
- GH_AW_TIMEOUT_MINUTES: 20
- GH_AW_VERSION: v0.81.6
- GITHUB_API_URL: ${{ github.api_url }}
- GITHUB_AW: true
- GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows
- GITHUB_HEAD_REF: ${{ github.head_ref }}
- GITHUB_REF_NAME: ${{ github.ref_name }}
- GITHUB_SERVER_URL: ${{ github.server_url }}
- GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md
- GITHUB_WORKSPACE: ${{ github.workspace }}
- GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com
- GIT_AUTHOR_NAME: github-actions[bot]
- GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com
- GIT_COMMITTER_NAME: github-actions[bot]
- RUNNER_TEMP: ${{ runner.temp }}
- TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }}
- - name: Parse threat detection token usage for step summary
- id: parse_detection_token_usage
- if: always()
- continue-on-error: true
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage
- with:
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs');
- await main();
- - name: Upload threat detection log
- if: always() && steps.detection_guard.outputs.run_detection == 'true'
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: detection
- path: /tmp/gh-aw/threat-detection/detection.log
- if-no-files-found: ignore
- - name: Parse and conclude threat detection
- id: detection_conclusion
- if: always()
- continue-on-error: true
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }}
- DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }}
- GH_AW_DETECTION_CONTINUE_ON_ERROR: "true"
- with:
- script: |
- try {
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs');
- await main();
- } catch (loadErr) {
- const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false';
- const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure';
- const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr));
- core.error(msg);
- core.setOutput('reason', 'parse_error');
- if (continueOnError && !detectionExecutionFailed) {
- core.warning('\u26A0\uFE0F ' + msg);
- core.setOutput('conclusion', 'warning');
- core.setOutput('success', 'false');
- } else {
- core.setOutput('conclusion', 'failure');
- core.setOutput('success', 'false');
- core.setFailed(msg);
- }
- }
-
- safe_outputs:
- needs:
- - activation
- - agent
- - detection
- if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success'
- runs-on: ubuntu-slim
- permissions:
- contents: read
- discussions: write
- issues: write
- timeout-minutes: 45
- env:
- GH_AW_AGENT_AIC: ${{ needs.agent.outputs.aic }}
- GH_AW_AIC: ${{ needs.agent.outputs.aic }}
- GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }}
- GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/qf-s-benchmark"
- GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }}
- GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }}
- GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }}
- GH_AW_ENGINE_ID: "copilot"
- GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }}
- GH_AW_ENGINE_VERSION: "1.0.65"
- GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
- GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }}
- GH_AW_WORKFLOW_ID: "qf-s-benchmark"
- GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/qf-s-benchmark.md"
- outputs:
- code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }}
- code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }}
- create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }}
- create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }}
- process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }}
- process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }}
- steps:
- - name: Setup Scripts
- id: setup
- uses: github/gh-aw-actions/setup@ba6380cc6e5be5d21677bebe04d52fb48e3abec7 # v0.81.6
- with:
- destination: ${{ runner.temp }}/gh-aw/actions
- job-name: ${{ github.job }}
- trace-id: ${{ needs.activation.outputs.setup-trace-id }}
- parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }}
- env:
- GH_AW_SETUP_WORKFLOW_NAME: "QF_S String Solver Benchmark"
- GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/qf-s-benchmark.lock.yml@${{ github.ref }}
- GH_AW_INFO_VERSION: "1.0.65"
- GH_AW_INFO_AWF_VERSION: "v0.27.11"
- GH_AW_INFO_ENGINE_ID: "copilot"
- - name: Download agent output artifact
- id: download-agent-output
- continue-on-error: true
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- name: agent
- path: /tmp/gh-aw/
- - name: Setup agent output environment variable
- id: setup-agent-output-env
- if: steps.download-agent-output.outcome == 'success'
- run: |
- mkdir -p /tmp/gh-aw/
- find "/tmp/gh-aw/" -type f -print
- echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT"
- - name: Configure GH_HOST for enterprise compatibility
- id: ghes-host-config
- shell: bash
- run: | # zizmor: ignore[github-env] - GITHUB_SERVER_URL is set by GitHub Actions, not user input.
- # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct
- # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op.
- GH_HOST="${GITHUB_SERVER_URL#https://}"
- GH_HOST="${GH_HOST#http://}"
- echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV"
- - name: Process Safe Outputs
- id: process_safe_outputs
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
- GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }}
- GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com"
- GITHUB_SERVER_URL: ${{ github.server_url }}
- GITHUB_API_URL: ${{ github.api_url }}
- GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_discussion\":{\"category\":\"agentic workflows\",\"close_older_discussions\":true,\"expires\":168,\"fallback_to_issue\":true,\"max\":1,\"title_prefix\":\"[QF_S Benchmark] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}"
- with:
- github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
- script: |
- const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
- setupGlobals(core, github, context, exec, io, getOctokit);
- const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs');
- await main();
- - name: Upload Safe Outputs Items
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: safe-outputs-items
- path: |
- /tmp/gh-aw/safe-output-items.jsonl
- /tmp/gh-aw/temporary-id-map.json
- if-no-files-found: ignore
-
diff --git a/.github/workflows/qf-s-benchmark.md b/.github/workflows/qf-s-benchmark.md
deleted file mode 100644
index e57582d264..0000000000
--- a/.github/workflows/qf-s-benchmark.md
+++ /dev/null
@@ -1,405 +0,0 @@
----
-description: Benchmark Z3 seq vs nseq string solvers on QF_S test suite from the c3 branch and post results as a GitHub discussion
-
-on:
- schedule:
- - cron: "0 0,12 * * *"
- workflow_dispatch:
-
-permissions: read-all
-
-network: defaults
-
-tools:
- bash: true
- github:
- toolsets: [default]
-
-safe-outputs:
- report-failure-as-issue: false
- create-discussion:
- title-prefix: "[QF_S Benchmark] "
- category: "Agentic Workflows"
- close-older-discussions: true
- missing-tool:
- create-issue: true
- noop:
- report-as-issue: false
-
-timeout-minutes: 120
-
-steps:
- - name: Checkout c3 branch
- uses: actions/checkout@v6.0.2
- with:
- ref: c3
- fetch-depth: 1
- persist-credentials: false
-
----
-
-# QF_S String Solver Benchmark
-
-## Job Description
-
-Your name is ${{ github.workflow }}. You are an expert performance analyst for the Z3 theorem prover, specializing in the string/sequence theory. Your task is to benchmark the `seq` solver (classical string theory) against the `nseq` solver (ZIPT-based string theory) on the QF_S test suite from the `c3` branch, and post a structured report as a GitHub Discussion.
-
-The workspace already contains the `c3` branch (checked out by the preceding workflow step).
-
-## Phase 1: Set Up the Build Environment
-
-Install required build tools:
-
-```bash
-sudo apt-get update -y
-sudo apt-get install -y cmake ninja-build python3 python3-pip time
-```
-
-Verify tools:
-
-```bash
-cmake --version
-ninja --version
-python3 --version
-```
-
-## Phase 2: Build Z3 in Release Mode
-
-Build Z3 in Release mode for accurate benchmark performance numbers and lower memory usage. Running `ninja` in the background with `&` is not allowed — concurrent C++ compilation and LLM inference can exhaust available RAM and kill the agent process.
-
-```bash
-mkdir -p /tmp/z3-build
-cd /tmp/z3-build
-cmake "$GITHUB_WORKSPACE" \
- -G Ninja \
- -DCMAKE_BUILD_TYPE=Release \
- -DZ3_BUILD_TEST_EXECUTABLES=OFF \
- 2>&1 | tee /tmp/z3-cmake.log
-ninja -j2 z3 2>&1 | tee /tmp/z3-build.log
-```
-
-Verify the binary was built:
-
-```bash
-/tmp/z3-build/z3 --version
-```
-
-If the build fails, report it immediately and stop.
-
-Once the binary is confirmed working, call the `noop` safe-output tool with the message `"Z3 built successfully from the c3 branch. Benchmark starting — results will be posted as a GitHub Discussion once complete."` This keepalive call refreshes the safe-output MCP session before the long benchmark run begins, preventing a session timeout.
-
-## Phase 3: Discover QF_S Benchmark Files
-
-Find all `.smt2` benchmark files in the workspace that belong to the QF_S logic:
-
-```bash
-# Search for explicit QF_S logic declarations
-grep -rl 'QF_S' "$GITHUB_WORKSPACE" --include='*.smt2' 2>/dev/null > /tmp/qf_s_files.txt
-
-# Also look in dedicated benchmark directories
-find "$GITHUB_WORKSPACE" \
- \( -path "*/QF_S/*" -o -path "*/qf_s/*" -o -path "*/benchmarks/*" \) \
- -name '*.smt2' 2>/dev/null >> /tmp/qf_s_files.txt
-
-# Deduplicate
-sort -u /tmp/qf_s_files.txt -o /tmp/qf_s_files.txt
-
-TOTAL=$(wc -l < /tmp/qf_s_files.txt)
-echo "Found $TOTAL QF_S benchmark files"
-head -20 /tmp/qf_s_files.txt
-```
-
-If fewer than 5 files are found, also scan the entire workspace for any `.smt2` file that exercises string constraints:
-
-```bash
-if [ "$TOTAL" -lt 5 ]; then
- grep -rl 'declare.*String\|str\.\|seq\.' "$GITHUB_WORKSPACE" \
- --include='*.smt2' 2>/dev/null >> /tmp/qf_s_files.txt
- sort -u /tmp/qf_s_files.txt -o /tmp/qf_s_files.txt
- TOTAL=$(wc -l < /tmp/qf_s_files.txt)
- echo "After extended search: $TOTAL files"
-fi
-```
-
-Cap the benchmark set to keep total runtime under 60 minutes:
-
-```bash
-# Use at most 300 files; take a random sample if more are available
-if [ "$TOTAL" -gt 300 ]; then
- shuf -n 300 /tmp/qf_s_files.txt > /tmp/qf_s_sample.txt
-else
- cp /tmp/qf_s_files.txt /tmp/qf_s_sample.txt
-fi
-SAMPLE=$(wc -l < /tmp/qf_s_sample.txt)
-echo "Running benchmarks on $SAMPLE files"
-```
-
-## Phase 4: Run Benchmarks — seq vs nseq
-
-Run each benchmark with both solvers. Use a per-file timeout of 5 seconds. Set Z3's internal timeout to 4 seconds so it exits cleanly before the shell timeout fires.
-
-```bash
-Z3=/tmp/z3-build/z3
-TIMEOUT_SEC=5
-Z3_TIMEOUT_SEC=4
-RESULTS=/tmp/benchmark-results.csv
-
-echo "file,seq_result,seq_time_ms,nseq_result,nseq_time_ms" > "$RESULTS"
-
-total=0
-done_count=0
-while IFS= read -r smt_file; do
- total=$((total + 1))
-
- # Run with seq solver; capture both stdout (z3 output) and stderr (time output)
- SEQ_OUT=$({ time timeout "$TIMEOUT_SEC" "$Z3" \
- smt.string_solver=seq \
- -T:"$Z3_TIMEOUT_SEC" \
- "$smt_file" 2>/dev/null; } 2>&1)
- SEQ_RESULT=$(echo "$SEQ_OUT" | grep -E '^(sat|unsat|unknown)' | head -1)
- SEQ_MS=$(echo "$SEQ_OUT" | grep real | awk '{split($2,a,"m"); split(a[2],b,"s"); printf "%d", (a[1]*60+b[1])*1000}')
- [ -z "$SEQ_RESULT" ] && SEQ_RESULT="timeout"
- [ -z "$SEQ_MS" ] && SEQ_MS=$((TIMEOUT_SEC * 1000))
-
- # Run with nseq solver; same structure
- NSEQ_OUT=$({ time timeout "$TIMEOUT_SEC" "$Z3" \
- smt.string_solver=nseq \
- -T:"$Z3_TIMEOUT_SEC" \
- "$smt_file" 2>/dev/null; } 2>&1)
- NSEQ_RESULT=$(echo "$NSEQ_OUT" | grep -E '^(sat|unsat|unknown)' | head -1)
- NSEQ_MS=$(echo "$NSEQ_OUT" | grep real | awk '{split($2,a,"m"); split(a[2],b,"s"); printf "%d", (a[1]*60+b[1])*1000}')
- [ -z "$NSEQ_RESULT" ] && NSEQ_RESULT="timeout"
- [ -z "$NSEQ_MS" ] && NSEQ_MS=$((TIMEOUT_SEC * 1000))
-
- SHORT=$(basename "$smt_file")
- echo "$SHORT,$SEQ_RESULT,$SEQ_MS,$NSEQ_RESULT,$NSEQ_MS" >> "$RESULTS"
-
- done_count=$((done_count + 1))
- if [ $((done_count % 50)) -eq 0 ]; then
- echo "Progress: $done_count / $SAMPLE files completed"
- fi
-done < /tmp/qf_s_sample.txt
-
-echo "Benchmark run complete: $done_count files"
-```
-
-## Phase 5: Collect Seq Traces for Interesting Cases
-
-For benchmarks where `seq` solves in under 2 s but `nseq` times out (seq-fast/nseq-slow cases), collect a brief `seq` trace to understand what algorithm is used:
-
-```bash
-Z3=/tmp/z3-build/z3
-mkdir -p /tmp/traces
-
-# Find seq-fast / nseq-slow files: seq solved (sat/unsat) in <2000ms AND nseq timed out
-awk -F, 'NR>1 && ($2=="sat"||$2=="unsat") && $3<2000 && $4=="timeout" {print $1}' \
- /tmp/benchmark-results.csv > /tmp/seq_fast_nseq_slow.txt
-echo "seq-fast / nseq-slow files: $(wc -l < /tmp/seq_fast_nseq_slow.txt)"
-
-# Collect traces for at most 5 such cases
-head -5 /tmp/seq_fast_nseq_slow.txt | while IFS= read -r short; do
- # Find the full path
- full=$(grep "/$short$" /tmp/qf_s_sample.txt | head -1)
- [ -z "$full" ] && continue
- timeout 5 "$Z3" \
- smt.string_solver=seq \
- -tr:seq \
- -T:5 \
- "$full" > "/tmp/traces/${short%.smt2}.seq.trace" 2>&1 || true
-done
-```
-
-## Phase 6: Analyze Results
-
-Compute summary statistics from the CSV. Save the analysis script to a file and run it:
-
-```bash
-cat > /tmp/analyze_benchmark.py << 'PYEOF'
-import csv, sys
-
-results = []
-with open('/tmp/benchmark-results.csv') as f:
- reader = csv.DictReader(f)
- for row in reader:
- results.append(row)
-
-total = len(results)
-if total == 0:
- print("No results found.")
- sys.exit(0)
-
-def is_correct(r, solver):
- prefix = 'seq' if solver == 'seq' else 'nseq'
- return r[f'{prefix}_result'] in ('sat', 'unsat')
-
-def timed_out(r, solver):
- prefix = 'seq' if solver == 'seq' else 'nseq'
- return r[f'{prefix}_result'] == 'timeout'
-
-seq_solved = sum(1 for r in results if is_correct(r, 'seq'))
-nseq_solved = sum(1 for r in results if is_correct(r, 'nseq'))
-seq_to = sum(1 for r in results if timed_out(r, 'seq'))
-nseq_to = sum(1 for r in results if timed_out(r, 'nseq'))
-
-seq_times = [int(r['seq_time_ms']) for r in results if is_correct(r, 'seq')]
-nseq_times = [int(r['nseq_time_ms']) for r in results if is_correct(r, 'nseq')]
-
-def median(lst):
- s = sorted(lst)
- n = len(s)
- return s[n//2] if n else 0
-
-def mean(lst):
- return sum(lst)//len(lst) if lst else 0
-
-# Disagreements (sat vs unsat or vice-versa)
-disagreements = [
- r for r in results
- if r['seq_result'] in ('sat','unsat')
- and r['nseq_result'] in ('sat','unsat')
- and r['seq_result'] != r['nseq_result']
-]
-
-# seq-fast / nseq-slow: seq solved in <2s, nseq timed out
-seq_fast_nseq_slow = [
- r for r in results
- if is_correct(r, 'seq') and int(r['seq_time_ms']) < 2000 and timed_out(r, 'nseq')
-]
-# nseq-fast / seq-slow: nseq solved in <2s, seq timed out
-nseq_fast_seq_slow = [
- r for r in results
- if is_correct(r, 'nseq') and int(r['nseq_time_ms']) < 2000 and timed_out(r, 'seq')
-]
-
-print(f"TOTAL={total}")
-print(f"SEQ_SOLVED={seq_solved}")
-print(f"NSEQ_SOLVED={nseq_solved}")
-print(f"SEQ_TIMEOUTS={seq_to}")
-print(f"NSEQ_TIMEOUTS={nseq_to}")
-print(f"SEQ_MEDIAN_MS={median(seq_times)}")
-print(f"NSEQ_MEDIAN_MS={median(nseq_times)}")
-print(f"SEQ_MEAN_MS={mean(seq_times)}")
-print(f"NSEQ_MEAN_MS={mean(nseq_times)}")
-print(f"DISAGREEMENTS={len(disagreements)}")
-print(f"SEQ_FAST_NSEQ_SLOW={len(seq_fast_nseq_slow)}")
-print(f"NSEQ_FAST_SEQ_SLOW={len(nseq_fast_seq_slow)}")
-
-# Print top-10 slowest for nseq that seq handles fast
-print("\nTOP_SEQ_FAST_NSEQ_SLOW:")
-for r in sorted(seq_fast_nseq_slow, key=lambda x: -int(x['nseq_time_ms']))[:10]:
- print(f" {r['file']} seq={r['seq_time_ms']}ms nseq={r['nseq_time_ms']}ms seq_result={r['seq_result']} nseq_result={r['nseq_result']}")
-
-print("\nTOP_NSEQ_FAST_SEQ_SLOW:")
-for r in sorted(nseq_fast_seq_slow, key=lambda x: -int(x['seq_time_ms']))[:10]:
- print(f" {r['file']} seq={r['seq_time_ms']}ms nseq={r['nseq_time_ms']}ms seq_result={r['seq_result']} nseq_result={r['nseq_result']}")
-
-if disagreements:
- print(f"\nDISAGREEMENTS ({len(disagreements)}):")
- for r in disagreements[:10]:
- print(f" {r['file']} seq={r['seq_result']} nseq={r['nseq_result']}")
-PYEOF
-
-python3 /tmp/analyze_benchmark.py
-```
-
-## Phase 7: Create GitHub Discussion
-
-Use the `create_discussion` safe-output tool to post a structured benchmark report.
-
-The discussion body should be formatted as follows (fill in real numbers from Phase 6):
-
-```markdown
-# QF_S Benchmark: seq vs nseq
-
-**Date**: YYYY-MM-DD
-**Branch**: c3
-**Commit**: ``
-**Workflow Run**: [#](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
-**Files benchmarked**: N (capped at 300, timeout 5 s per file)
-
----
-
-## Summary
-
-| Metric | seq | nseq |
-|--------|-----|------|
-| Files solved (sat/unsat) | SEQ_SOLVED | NSEQ_SOLVED |
-| Timeouts | SEQ_TO | NSEQ_TO |
-| Median solve time (solved files) | X ms | Y ms |
-| Mean solve time (solved files) | X ms | Y ms |
-| **Disagreements (sat≠unsat)** | — | N |
-
----
-
-## Performance Comparison
-
-### seq-fast / nseq-slow (seq < 2 s, nseq timed out)
-
-These are benchmarks where the classical `seq` solver is significantly faster. These represent regression risk for `nseq`.
-
-| File | seq (ms) | nseq (ms) | seq result | nseq result |
-|------|----------|-----------|------------|-------------|
-[TOP 10 ENTRIES]
-
-### nseq-fast / seq-slow (nseq < 2 s, seq timed out)
-
-These are benchmarks where `nseq` shows a performance advantage.
-
-| File | seq (ms) | nseq (ms) | seq result | nseq result |
-|------|----------|-----------|------------|-------------|
-[TOP 10 ENTRIES]
-
----
-
-## Correctness
-
-**Disagreements** (files where seq says `sat` but nseq says `unsat` or vice versa): N
-
-[If disagreements exist, list all of them here with file paths and both results]
-
----
-
-## seq Trace Analysis (seq-fast / nseq-slow cases)
-
-
-Click to expand trace snippets for top seq-fast/nseq-slow cases
-
-[Insert trace snippet for each traced file, or "No traces collected" if section was skipped]
-
-
-
----
-
-## Raw Data
-
-
-Full results CSV (click to expand)
-
-```csv
-[PASTE FIRST 200 LINES OF /tmp/benchmark-results.csv]
-```
-
-
-
----
-
-*Generated by the QF_S Benchmark workflow. To reproduce: build Z3 from the `c3` branch and run `z3 smt.string_solver=seq|nseq -T:10 `.*
-```
-
-## Edge Cases
-
-- If the build fails, call `missing_data` explaining the build error and stop.
-- If no benchmark files are found at all, call `missing_data` explaining that no QF_S `.smt2` files were found in the `c3` branch.
-- If Z3 crashes (segfault) on a file with either solver, record the result as `crash` and continue.
-- If the total benchmark set is very small (< 5 files), note this prominently in the discussion and suggest adding more QF_S benchmarks to the `c3` branch.
-- If zero disagreements and both solvers time out on the same files, note that the solvers are in agreement.
-- If `create_discussion` fails (e.g., MCP session error), call `report_incomplete` with the reason and include the top-line statistics (files solved, timeouts, disagreement count) in the `details` field.
-
-## Important Notes
-
-- **DO NOT** modify any source files or create pull requests.
-- **DO NOT** run `ninja` or any build command in the background with `&` — concurrent C++ compilation and LLM inference can exhaust available RAM and kill the agent process. Always wait for build commands to complete before proceeding.
-- **DO NOT** run benchmarks for longer than 100 minutes total (leave buffer for posting).
-- **DO** always report the commit SHA so results can be correlated with specific code versions.
-- **DO** close older QF_S Benchmark discussions automatically (configured via `close-older-discussions: true`).
-- **DO** highlight disagreements prominently — these are potential correctness bugs.
diff --git a/README.md b/README.md
index 35ab011b72..5bb3c5d3a3 100644
--- a/README.md
+++ b/README.md
@@ -46,9 +46,9 @@ See the [release notes](RELEASE_NOTES.md) for notes on various stable releases o
| -------------|-----------------|---------------|---------------------|-------------------|
| [](https://github.com/Z3Prover/z3/actions/workflows/api-coherence-checker.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/code-simplifier.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/release-notes-updater.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/workflow-suggestion-agent.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/academic-citation-tracker.lock.yml) |
-| Issue Backlog | Memory Safety Report | QF-S Benchmark | Specbot Crash Analyzer | SMTLIB Benchmark Finder |
-| --------------|----------------------|----------------|------------------------|-------------------------|
-| [](https://github.com/Z3Prover/z3/actions/workflows/issue-backlog-processor.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/memory-safety-report.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/qf-s-benchmark.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/specbot-crash-analyzer.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/smtlib-benchmark-finder.lock.yml) |
+| Issue Backlog | Memory Safety Report | Specbot Crash Analyzer | SMTLIB Benchmark Finder |
+| --------------|----------------------|------------------------|-------------------------|
+| [](https://github.com/Z3Prover/z3/actions/workflows/issue-backlog-processor.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/memory-safety-report.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/specbot-crash-analyzer.lock.yml) | [](https://github.com/Z3Prover/z3/actions/workflows/smtlib-benchmark-finder.lock.yml) |
| TPTP Benchmark |
|----------------|
diff --git a/agentics/qf-s-benchmark.md b/agentics/qf-s-benchmark.md
deleted file mode 100644
index 3ceb8f58e7..0000000000
--- a/agentics/qf-s-benchmark.md
+++ /dev/null
@@ -1,364 +0,0 @@
-# QF_S String Solver Benchmark
-
-## Job Description
-
-Your name is ${{ github.workflow }}. You are an expert performance analyst for the Z3 theorem prover, specializing in the string/sequence theory. Your task is to benchmark the `seq` solver (classical string theory) against the `nseq` solver (ZIPT-based string theory) on the QF_S test suite from the `c3` branch, and post a structured report as a GitHub Discussion.
-
-The workspace already contains the `c3` branch (checked out by the preceding workflow step).
-
-## Phase 1: Set Up the Build Environment
-
-Install required build tools:
-
-```bash
-sudo apt-get update -y
-sudo apt-get install -y cmake ninja-build python3 python3-pip time
-```
-
-Verify tools:
-
-```bash
-cmake --version
-ninja --version
-python3 --version
-```
-
-## Phase 2: Build Z3 in Debug Mode with Seq Tracing
-
-Build Z3 with debug symbols so that tracing and timing data are meaningful.
-
-```bash
-mkdir -p /tmp/z3-build
-cd /tmp/z3-build
-cmake "$GITHUB_WORKSPACE" \
- -G Ninja \
- -DCMAKE_BUILD_TYPE=Debug \
- -DZ3_BUILD_TEST_EXECUTABLES=OFF \
- 2>&1 | tee /tmp/z3-cmake.log
-ninja z3 2>&1 | tee /tmp/z3-build.log
-```
-
-Verify the binary was built:
-
-```bash
-/tmp/z3-build/z3 --version
-```
-
-If the build fails, report it immediately and stop.
-
-## Phase 3: Discover QF_S Benchmark Files
-
-Find all `.smt2` benchmark files in the workspace that belong to the QF_S logic:
-
-```bash
-# Search for explicit QF_S logic declarations
-grep -rl 'QF_S' "$GITHUB_WORKSPACE" --include='*.smt2' 2>/dev/null > /tmp/qf_s_files.txt
-
-# Also look in dedicated benchmark directories
-find "$GITHUB_WORKSPACE" \
- \( -path "*/QF_S/*" -o -path "*/qf_s/*" -o -path "*/benchmarks/*" \) \
- -name '*.smt2' 2>/dev/null >> /tmp/qf_s_files.txt
-
-# Deduplicate
-sort -u /tmp/qf_s_files.txt -o /tmp/qf_s_files.txt
-
-TOTAL=$(wc -l < /tmp/qf_s_files.txt)
-echo "Found $TOTAL QF_S benchmark files"
-head -20 /tmp/qf_s_files.txt
-```
-
-If fewer than 5 files are found, also scan the entire workspace for any `.smt2` file that exercises string constraints:
-
-```bash
-if [ "$TOTAL" -lt 5 ]; then
- grep -rl 'declare.*String\|str\.\|seq\.' "$GITHUB_WORKSPACE" \
- --include='*.smt2' 2>/dev/null >> /tmp/qf_s_files.txt
- sort -u /tmp/qf_s_files.txt -o /tmp/qf_s_files.txt
- TOTAL=$(wc -l < /tmp/qf_s_files.txt)
- echo "After extended search: $TOTAL files"
-fi
-```
-
-Cap the benchmark set to keep total runtime under 60 minutes:
-
-```bash
-# Use at most 500 files; take a random sample if more are available
-if [ "$TOTAL" -gt 500 ]; then
- shuf -n 500 /tmp/qf_s_files.txt > /tmp/qf_s_sample.txt
-else
- cp /tmp/qf_s_files.txt /tmp/qf_s_sample.txt
-fi
-SAMPLE=$(wc -l < /tmp/qf_s_sample.txt)
-echo "Running benchmarks on $SAMPLE files"
-```
-
-## Phase 4: Run Benchmarks — seq vs nseq
-
-Run each benchmark with both solvers. Use a per-file timeout of 10 seconds. Set Z3's internal timeout to 9 seconds so it exits cleanly before the shell timeout fires.
-
-```bash
-Z3=/tmp/z3-build/z3
-TIMEOUT_SEC=10
-Z3_TIMEOUT_SEC=9
-RESULTS=/tmp/benchmark-results.csv
-
-echo "file,seq_result,seq_time_ms,nseq_result,nseq_time_ms" > "$RESULTS"
-
-total=0
-done_count=0
-while IFS= read -r smt_file; do
- total=$((total + 1))
-
- # Run with seq solver; capture both stdout (z3 output) and stderr (time output)
- SEQ_OUT=$({ time timeout "$TIMEOUT_SEC" "$Z3" \
- smt.string_solver=seq \
- -T:"$Z3_TIMEOUT_SEC" \
- "$smt_file" 2>/dev/null; } 2>&1)
- SEQ_RESULT=$(echo "$SEQ_OUT" | grep -E '^(sat|unsat|unknown)' | head -1)
- SEQ_MS=$(echo "$SEQ_OUT" | grep real | awk '{split($2,a,"m"); split(a[2],b,"s"); printf "%d", (a[1]*60+b[1])*1000}')
- [ -z "$SEQ_RESULT" ] && SEQ_RESULT="timeout"
- [ -z "$SEQ_MS" ] && SEQ_MS=$((TIMEOUT_SEC * 1000))
-
- # Run with nseq solver; same structure
- NSEQ_OUT=$({ time timeout "$TIMEOUT_SEC" "$Z3" \
- smt.string_solver=nseq \
- -T:"$Z3_TIMEOUT_SEC" \
- "$smt_file" 2>/dev/null; } 2>&1)
- NSEQ_RESULT=$(echo "$NSEQ_OUT" | grep -E '^(sat|unsat|unknown)' | head -1)
- NSEQ_MS=$(echo "$NSEQ_OUT" | grep real | awk '{split($2,a,"m"); split(a[2],b,"s"); printf "%d", (a[1]*60+b[1])*1000}')
- [ -z "$NSEQ_RESULT" ] && NSEQ_RESULT="timeout"
- [ -z "$NSEQ_MS" ] && NSEQ_MS=$((TIMEOUT_SEC * 1000))
-
- SHORT=$(basename "$smt_file")
- echo "$SHORT,$SEQ_RESULT,$SEQ_MS,$NSEQ_RESULT,$NSEQ_MS" >> "$RESULTS"
-
- done_count=$((done_count + 1))
- if [ $((done_count % 50)) -eq 0 ]; then
- echo "Progress: $done_count / $SAMPLE files completed"
- fi
-done < /tmp/qf_s_sample.txt
-
-echo "Benchmark run complete: $done_count files"
-```
-
-## Phase 5: Collect Seq Traces for Interesting Cases
-
-For benchmarks where `seq` solves in under 2 s but `nseq` times out (seq-fast/nseq-slow cases), collect a brief `seq` trace to understand what algorithm is used:
-
-```bash
-Z3=/tmp/z3-build/z3
-mkdir -p /tmp/traces
-
-# Find seq-fast / nseq-slow files: seq solved (sat/unsat) in <2000ms AND nseq timed out
-awk -F, 'NR>1 && ($2=="sat"||$2=="unsat") && $3<2000 && $4=="timeout" {print $1}' \
- /tmp/benchmark-results.csv > /tmp/seq_fast_nseq_slow.txt
-echo "seq-fast / nseq-slow files: $(wc -l < /tmp/seq_fast_nseq_slow.txt)"
-
-# Collect traces for at most 5 such cases
-head -5 /tmp/seq_fast_nseq_slow.txt | while IFS= read -r short; do
- # Find the full path
- full=$(grep "/$short$" /tmp/qf_s_sample.txt | head -1)
- [ -z "$full" ] && continue
- timeout 5 "$Z3" \
- smt.string_solver=seq \
- -tr:seq \
- -T:5 \
- "$full" > "/tmp/traces/${short%.smt2}.seq.trace" 2>&1 || true
-done
-```
-
-## Phase 6: Analyze Results
-
-Compute summary statistics from the CSV:
-
-```bash
-Save the analysis script to a file and run it:
-
-```bash
-cat > /tmp/analyze_benchmark.py << 'PYEOF'
-import csv, sys
-
-results = []
-with open('/tmp/benchmark-results.csv') as f:
- reader = csv.DictReader(f)
- for row in reader:
- results.append(row)
-
-total = len(results)
-if total == 0:
- print("No results found.")
- sys.exit(0)
-
-def is_correct(r, solver):
- prefix = 'seq' if solver == 'seq' else 'nseq'
- return r[f'{prefix}_result'] in ('sat', 'unsat')
-
-def timed_out(r, solver):
- prefix = 'seq' if solver == 'seq' else 'nseq'
- return r[f'{prefix}_result'] == 'timeout'
-
-seq_solved = sum(1 for r in results if is_correct(r, 'seq'))
-nseq_solved = sum(1 for r in results if is_correct(r, 'nseq'))
-seq_to = sum(1 for r in results if timed_out(r, 'seq'))
-nseq_to = sum(1 for r in results if timed_out(r, 'nseq'))
-
-seq_times = [int(r['seq_time_ms']) for r in results if is_correct(r, 'seq')]
-nseq_times = [int(r['nseq_time_ms']) for r in results if is_correct(r, 'nseq')]
-
-def median(lst):
- s = sorted(lst)
- n = len(s)
- return s[n//2] if n else 0
-
-def mean(lst):
- return sum(lst)//len(lst) if lst else 0
-
-# Disagreements (sat vs unsat or vice-versa)
-disagreements = [
- r for r in results
- if r['seq_result'] in ('sat','unsat')
- and r['nseq_result'] in ('sat','unsat')
- and r['seq_result'] != r['nseq_result']
-]
-
-# seq-fast / nseq-slow: seq solved in <2s, nseq timed out
-seq_fast_nseq_slow = [
- r for r in results
- if is_correct(r, 'seq') and int(r['seq_time_ms']) < 2000 and timed_out(r, 'nseq')
-]
-# nseq-fast / seq-slow: nseq solved in <2s, seq timed out
-nseq_fast_seq_slow = [
- r for r in results
- if is_correct(r, 'nseq') and int(r['nseq_time_ms']) < 2000 and timed_out(r, 'seq')
-]
-
-print(f"TOTAL={total}")
-print(f"SEQ_SOLVED={seq_solved}")
-print(f"NSEQ_SOLVED={nseq_solved}")
-print(f"SEQ_TIMEOUTS={seq_to}")
-print(f"NSEQ_TIMEOUTS={nseq_to}")
-print(f"SEQ_MEDIAN_MS={median(seq_times)}")
-print(f"NSEQ_MEDIAN_MS={median(nseq_times)}")
-print(f"SEQ_MEAN_MS={mean(seq_times)}")
-print(f"NSEQ_MEAN_MS={mean(nseq_times)}")
-print(f"DISAGREEMENTS={len(disagreements)}")
-print(f"SEQ_FAST_NSEQ_SLOW={len(seq_fast_nseq_slow)}")
-print(f"NSEQ_FAST_SEQ_SLOW={len(nseq_fast_seq_slow)}")
-
-# Print top-10 slowest for nseq that seq handles fast
-print("\nTOP_SEQ_FAST_NSEQ_SLOW:")
-for r in sorted(seq_fast_nseq_slow, key=lambda x: -int(x['nseq_time_ms']))[:10]:
- print(f" {r['file']} seq={r['seq_time_ms']}ms nseq={r['nseq_time_ms']}ms seq_result={r['seq_result']} nseq_result={r['nseq_result']}")
-
-print("\nTOP_NSEQ_FAST_SEQ_SLOW:")
-for r in sorted(nseq_fast_seq_slow, key=lambda x: -int(x['seq_time_ms']))[:10]:
- print(f" {r['file']} seq={r['seq_time_ms']}ms nseq={r['nseq_time_ms']}ms seq_result={r['seq_result']} nseq_result={r['nseq_result']}")
-
-if disagreements:
- print(f"\nDISAGREEMENTS ({len(disagreements)}):")
- for r in disagreements[:10]:
- print(f" {r['file']} seq={r['seq_result']} nseq={r['nseq_result']}")
-PYEOF
-
-python3 /tmp/analyze_benchmark.py
-```
-
-## Phase 7: Create GitHub Discussion
-
-Use the `create_discussion` safe-output tool to post a structured benchmark report.
-
-The discussion body should be formatted as follows (fill in real numbers from Phase 6):
-
-```markdown
-# QF_S Benchmark: seq vs nseq
-
-**Date**: YYYY-MM-DD
-**Branch**: c3
-**Commit**: ``
-**Workflow Run**: [#](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
-**Files benchmarked**: N (capped at 500, timeout 10 s per file)
-
----
-
-## Summary
-
-| Metric | seq | nseq |
-|--------|-----|------|
-| Files solved (sat/unsat) | SEQ_SOLVED | NSEQ_SOLVED |
-| Timeouts | SEQ_TO | NSEQ_TO |
-| Median solve time (solved files) | X ms | Y ms |
-| Mean solve time (solved files) | X ms | Y ms |
-| **Disagreements (sat≠unsat)** | — | N |
-
----
-
-## Performance Comparison
-
-### seq-fast / nseq-slow (seq < 2 s, nseq timed out)
-
-These are benchmarks where the classical `seq` solver is significantly faster. These represent regression risk for `nseq`.
-
-| File | seq (ms) | nseq (ms) | seq result | nseq result |
-|------|----------|-----------|------------|-------------|
-[TOP 10 ENTRIES]
-
-### nseq-fast / seq-slow (nseq < 2 s, seq timed out)
-
-These are benchmarks where `nseq` shows a performance advantage.
-
-| File | seq (ms) | nseq (ms) | seq result | nseq result |
-|------|----------|-----------|------------|-------------|
-[TOP 10 ENTRIES]
-
----
-
-## Correctness
-
-**Disagreements** (files where seq says `sat` but nseq says `unsat` or vice versa): N
-
-[If disagreements exist, list all of them here with file paths and both results]
-
----
-
-## seq Trace Analysis (seq-fast / nseq-slow cases)
-
-
-Click to expand trace snippets for top seq-fast/nseq-slow cases
-
-[Insert trace snippet for each traced file, or "No traces collected" if section was skipped]
-
-
-
----
-
-## Raw Data
-
-
-Full results CSV (click to expand)
-
-```csv
-[PASTE FIRST 200 LINES OF /tmp/benchmark-results.csv]
-```
-
-
-
----
-
-*Generated by the QF_S Benchmark workflow. To reproduce: build Z3 from the `c3` branch and run `z3 smt.string_solver=seq|nseq -T:10 `.*
-```
-
-## Edge Cases
-
-- If the build fails, call `missing_data` explaining the build error and stop.
-- If no benchmark files are found at all, call `missing_data` explaining that no QF_S `.smt2` files were found in the `c3` branch.
-- If Z3 crashes (segfault) on a file with either solver, record the result as `crash` and continue.
-- If the total benchmark set is very small (< 5 files), note this prominently in the discussion and suggest adding more QF_S benchmarks to the `c3` branch.
-- If zero disagreements and both solvers time out on the same files, note that the solvers are in agreement.
-
-## Important Notes
-
-- **DO NOT** modify any source files or create pull requests.
-- **DO NOT** run benchmarks for longer than 80 minutes total (leave buffer for posting).
-- **DO** always report the commit SHA so results can be correlated with specific code versions.
-- **DO** close older ZIPT Benchmark discussions automatically (configured via `close-older-discussions: true`).
-- **DO** highlight disagreements prominently — these are potential correctness bugs.