mirrors/z3
3
0
Fork 0
mirror of https://github.com/Z3Prover/z3 synced 2026-04-25 13:23:32 +00:00
Code Activity

Apply qf-s-benchmark fix: replace ZIPT/dotnet workflow with seq vs nseq only (#9266)

Browse source 
* Apply qf-s-benchmark fix from agentics/qf-s-benchmark.md: remove ZIPT/dotnet dependency

Agent-Logs-Url: https://github.com/Z3Prover/z3/sessions/c36bada5-c222-4b97-99c4-08392955b32d

Co-authored-by: NikolajBjorner <3085284+NikolajBjorner@users.noreply.github.com>

* Update qf-s-benchmark title prefix and note to QF_S Benchmark

Agent-Logs-Url: https://github.com/Z3Prover/z3/sessions/c36bada5-c222-4b97-99c4-08392955b32d

Co-authored-by: NikolajBjorner <3085284+NikolajBjorner@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: NikolajBjorner <3085284+NikolajBjorner@users.noreply.github.com>
This commit is contained in:
Copilot 2026-04-10 14:30:44 -07:00 • committed by GitHub
parent 58ad1f0918
commit 9c81571eb8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 570 additions and 570 deletions
Download patch file Download diff file Expand all files Collapse all files

464
.github/workflows/qf-s-benchmark.lock.yml generated vendored
View file

@ -1,3 +1,5 @@
# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"e5e5c332eb206c2bde9e7e5cb0bb1babe7b1c50e0437a00b4093ddb8b5ab80cf","compiler_version":"v0.67.4","strict":true,"agent_id":"copilot"}
# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"ed597411d8f924073f98dfc5c65a23a2325f34cd","version":"v8"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"v0.67.4","version":"v0.67.4"}]}
# ___ _ _
# / _ \ | | (_)
# | |_| | __ _ ___ _ __ | |_ _ ___
@ -12,7 +14,7 @@
# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \
# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/
#
# This file was automatically generated by gh-aw (v0.65.4). DO NOT EDIT.
# This file was automatically generated by gh-aw (v0.67.4). DO NOT EDIT.
#
# To update this file, edit the corresponding .md file and run:
# gh aw compile
@ -20,11 +22,22 @@
#
# For more information: https://github.github.com/gh-aw/introduction/overview/
#
# Run Z3 string solver benchmarks (seq vs nseq) on QF_S test suite from the c3 branch and post results as a GitHub discussion
# Benchmark Z3 seq vs nseq string solvers on QF_S test suite from the c3 branch and post results as a GitHub discussion
#
# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"d7c341a4c4224962ddf5d76ae2e39b3fc7965a5d9a7899d0674877de090be242","compiler_version":"v0.65.4","strict":true,"agent_id":"copilot"}
# Secrets used:
# - COPILOT_GITHUB_TOKEN
# - GH_AW_GITHUB_MCP_SERVER_TOKEN
# - GH_AW_GITHUB_TOKEN
# - GITHUB_TOKEN
#
# Custom actions used:
# - actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
# - actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
# - actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
# - github/gh-aw-actions/setup@v0.67.4
name: "ZIPT String Solver Benchmark"
name: "QF_S String Solver Benchmark"
"on":
schedule:
- cron: "0 0,12 * * *"
@ -41,12 +54,13 @@ permissions: {}
concurrency:
group: "gh-aw-${{ github.workflow }}"
run-name: "ZIPT String Solver Benchmark"
run-name: "QF_S String Solver Benchmark"
jobs:
activation:
runs-on: ubuntu-slim
permissions:
actions: read
contents: read
outputs:
comment_id: ""
@ -54,27 +68,30 @@ jobs:
lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }}
model: ${{ steps.generate_aw_info.outputs.model }}
secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }}
setup-trace-id: ${{ steps.setup.outputs.trace-id }}
steps:
- name: Setup Scripts
uses: github/gh-aw-actions/setup@934698b44320d87a7a9196339f90293f10bd2247 # v0.65.4
id: setup
uses: github/gh-aw-actions/setup@v0.67.4
with:
destination: ${{ runner.temp }}/gh-aw/actions
job-name: ${{ github.job }}
- name: Generate agentic run info
id: generate_aw_info
env:
GH_AW_INFO_ENGINE_ID: "copilot"
GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI"
GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'auto' }}
GH_AW_INFO_VERSION: "latest"
GH_AW_INFO_AGENT_VERSION: "latest"
GH_AW_INFO_CLI_VERSION: "v0.65.4"
GH_AW_INFO_WORKFLOW_NAME: "ZIPT String Solver Benchmark"
GH_AW_INFO_VERSION: "1.0.20"
GH_AW_INFO_AGENT_VERSION: "1.0.20"
GH_AW_INFO_CLI_VERSION: "v0.67.4"
GH_AW_INFO_WORKFLOW_NAME: "QF_S String Solver Benchmark"
GH_AW_INFO_EXPERIMENTAL: "false"
GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true"
GH_AW_INFO_STAGED: "false"
GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]'
GH_AW_INFO_FIREWALL_ENABLED: "true"
GH_AW_INFO_AWF_VERSION: "v0.25.6"
GH_AW_INFO_AWF_VERSION: "v0.25.18"
GH_AW_INFO_AWMG_VERSION: ""
GH_AW_INFO_FIREWALL_TYPE: "squid"
GH_AW_COMPILED_STRICT: "true"
@ -87,7 +104,7 @@ jobs:
await main(core, context);
- name: Validate COPILOT_GITHUB_TOKEN secret
id: validate-secret
run: ${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default
run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default
env:
COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
- name: Checkout .github and .agents folders
@ -99,10 +116,11 @@ jobs:
.agents
sparse-checkout-cone-mode: true
fetch-depth: 1
- name: Check workflow file timestamps
- name: Check workflow lock file
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
GH_AW_WORKFLOW_FILE: "qf-s-benchmark.lock.yml"
GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}"
with:
script: |
const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
@ -112,7 +130,7 @@ jobs:
- name: Check compile-agentic version
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
GH_AW_COMPILED_VERSION: "v0.65.4"
GH_AW_COMPILED_VERSION: "v0.67.4"
with:
script: |
const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
@ -130,19 +148,20 @@ jobs:
GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
GH_AW_GITHUB_WORKFLOW: ${{ github.workflow }}
GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }}
# poutine:ignore untrusted_checkout_exec
run: |
bash ${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh
bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh"
{
cat << 'GH_AW_PROMPT_c81e7aa1da1942d6_EOF'
cat << 'GH_AW_PROMPT_c25676ba2ab40d85_EOF'
<system>
GH_AW_PROMPT_c81e7aa1da1942d6_EOF
GH_AW_PROMPT_c25676ba2ab40d85_EOF
cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md"
cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md"
cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md"
cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md"
cat << 'GH_AW_PROMPT_c81e7aa1da1942d6_EOF'
cat << 'GH_AW_PROMPT_c25676ba2ab40d85_EOF'
<safe-output-tools>
Tools: create_discussion, missing_tool, missing_data, noop
</safe-output-tools>
@ -174,19 +193,20 @@ jobs:
{{/if}}
</github-context>
GH_AW_PROMPT_c81e7aa1da1942d6_EOF
GH_AW_PROMPT_c25676ba2ab40d85_EOF
cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md"
cat << 'GH_AW_PROMPT_c81e7aa1da1942d6_EOF'
cat << 'GH_AW_PROMPT_c25676ba2ab40d85_EOF'
</system>
{{#runtime-import .github/workflows/qf-s-benchmark.md}}
GH_AW_PROMPT_c81e7aa1da1942d6_EOF
GH_AW_PROMPT_c25676ba2ab40d85_EOF
} > "$GH_AW_PROMPT"
- name: Interpolate variables and render templates
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }}
GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
GH_AW_GITHUB_WORKFLOW: ${{ github.workflow }}
with:
script: |
const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
@ -204,6 +224,7 @@ jobs:
GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
GH_AW_GITHUB_WORKFLOW: ${{ github.workflow }}
GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }}
with:
script: |
@ -223,6 +244,7 @@ jobs:
GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER,
GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY,
GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID,
GH_AW_GITHUB_WORKFLOW: process.env.GH_AW_GITHUB_WORKFLOW,
GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE
}
});
@ -230,12 +252,12 @@ jobs:
env:
GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
# poutine:ignore untrusted_checkout_exec
run: bash ${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh
run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh"
- name: Print prompt
env:
GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
# poutine:ignore untrusted_checkout_exec
run: bash ${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh
run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh"
- name: Upload activation artifact
if: success()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
@ -244,6 +266,8 @@ jobs:
path: |
/tmp/gh-aw/aw_info.json
/tmp/gh-aw/aw-prompts/prompt.txt
/tmp/gh-aw/github_rate_limits.jsonl
if-no-files-found: ignore
retention-days: 1
agent:
@ -261,16 +285,21 @@ jobs:
GH_AW_WORKFLOW_ID_SANITIZED: qfsbenchmark
outputs:
checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }}
effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }}
has_patch: ${{ steps.collect_output.outputs.has_patch }}
inference_access_error: ${{ steps.detect-inference-error.outputs.inference_access_error || 'false' }}
model: ${{ needs.activation.outputs.model }}
output: ${{ steps.collect_output.outputs.output }}
output_types: ${{ steps.collect_output.outputs.output_types }}
setup-trace-id: ${{ steps.setup.outputs.trace-id }}
steps:
- name: Setup Scripts
uses: github/gh-aw-actions/setup@934698b44320d87a7a9196339f90293f10bd2247 # v0.65.4
id: setup
uses: github/gh-aw-actions/setup@v0.67.4
with:
destination: ${{ runner.temp }}/gh-aw/actions
job-name: ${{ github.job }}
trace-id: ${{ needs.activation.outputs.setup-trace-id }}
- name: Set runtime paths
id: set-runtime-paths
run: |
@ -278,9 +307,9 @@ jobs:
echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" >> "$GITHUB_OUTPUT"
echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" >> "$GITHUB_OUTPUT"
- name: Create gh-aw temp directory
run: bash ${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh
run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh"
- name: Configure gh CLI for GitHub Enterprise
run: bash ${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh
run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh"
env:
GH_TOKEN: ${{ github.token }}
- name: Checkout c3 branch
@ -294,13 +323,14 @@ jobs:
env:
REPO_NAME: ${{ github.repository }}
SERVER_URL: ${{ github.server_url }}
GITHUB_TOKEN: ${{ github.token }}
run: |
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
git config --global am.keepcr true
# Re-authenticate git with GitHub token
SERVER_URL_STRIPPED="${SERVER_URL#https://}"
git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git"
git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git"
echo "Git configured with standard GitHub Actions identity"
- name: Checkout PR branch
id: checkout-pr
@ -317,9 +347,11 @@ jobs:
const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs');
await main();
- name: Install GitHub Copilot CLI
run: ${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh latest
run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.20
env:
GH_HOST: github.com
- name: Install AWF binary
run: bash ${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh v0.25.6
run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.18
- name: Determine automatic lockdown mode for GitHub MCP Server
id: determine-automatic-lockdown
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
@ -331,114 +363,134 @@ jobs:
const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs');
await determineAutomaticLockdown(github, context, core);
- name: Download container images
run: bash ${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.25.6 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.6 ghcr.io/github/gh-aw-firewall/squid:0.25.6 ghcr.io/github/gh-aw-mcpg:v0.2.11 ghcr.io/github/github-mcp-server:v0.32.0 node:lts-alpine
run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.18 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.18 ghcr.io/github/gh-aw-firewall/squid:0.25.18 ghcr.io/github/gh-aw-mcpg:v0.2.17 ghcr.io/github/github-mcp-server:v0.32.0 node:lts-alpine
- name: Write Safe Outputs Config
run: |
mkdir -p ${RUNNER_TEMP}/gh-aw/safeoutputs
mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs"
mkdir -p /tmp/gh-aw/safeoutputs
mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs
cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_2df51426bf24e365_EOF'
{"create_discussion":{"category":"agentic workflows","close_older_discussions":true,"expires":168,"fallback_to_issue":true,"max":1,"title_prefix":"[ZIPT Benchmark] "},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"}}
GH_AW_SAFE_OUTPUTS_CONFIG_2df51426bf24e365_EOF
cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_ef274886e4650c6c_EOF'
{"create_discussion":{"category":"agentic workflows","close_older_discussions":true,"expires":168,"fallback_to_issue":true,"max":1,"title_prefix":"[QF_S Benchmark] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{}}
GH_AW_SAFE_OUTPUTS_CONFIG_ef274886e4650c6c_EOF
- name: Write Safe Outputs Tools
run: |
cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/tools_meta.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_META_919533c9cc87fea1_EOF'
{
"description_suffixes": {
"create_discussion": " CONSTRAINTS: Maximum 1 discussion(s) can be created. Title will be prefixed with \"[ZIPT Benchmark] \". Discussions will be created in category \"agentic workflows\"."
},
"repo_params": {},
"dynamic_tools": []
}
GH_AW_SAFE_OUTPUTS_TOOLS_META_919533c9cc87fea1_EOF
cat > ${RUNNER_TEMP}/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_906347b49a85a96b_EOF'
{
"create_discussion": {
"defaultMax": 1,
"fields": {
"body": {
"required": true,
"type": "string",
"sanitize": true,
"maxLength": 65000
},
"category": {
"type": "string",
"sanitize": true,
"maxLength": 128
},
"repo": {
"type": "string",
"maxLength": 256
},
"title": {
"required": true,
"type": "string",
"sanitize": true,
"maxLength": 128
env:
GH_AW_TOOLS_META_JSON: |
{
"description_suffixes": {
"create_discussion": " CONSTRAINTS: Maximum 1 discussion(s) can be created. Title will be prefixed with \"[QF_S Benchmark] \". Discussions will be created in category \"agentic workflows\"."
},
"repo_params": {},
"dynamic_tools": []
}
GH_AW_VALIDATION_JSON: |
{
"create_discussion": {
"defaultMax": 1,
"fields": {
"body": {
"required": true,
"type": "string",
"sanitize": true,
"maxLength": 65000
},
"category": {
"type": "string",
"sanitize": true,
"maxLength": 128
},
"repo": {
"type": "string",
"maxLength": 256
},
"title": {
"required": true,
"type": "string",
"sanitize": true,
"maxLength": 128
}
}
}
},
"missing_data": {
"defaultMax": 20,
"fields": {
"alternatives": {
"type": "string",
"sanitize": true,
"maxLength": 256
},
"context": {
"type": "string",
"sanitize": true,
"maxLength": 256
},
"data_type": {
"type": "string",
"sanitize": true,
"maxLength": 128
},
"reason": {
"type": "string",
"sanitize": true,
"maxLength": 256
},
"missing_data": {
"defaultMax": 20,
"fields": {
"alternatives": {
"type": "string",
"sanitize": true,
"maxLength": 256
},
"context": {
"type": "string",
"sanitize": true,
"maxLength": 256
},
"data_type": {
"type": "string",
"sanitize": true,
"maxLength": 128
},
"reason": {
"type": "string",
"sanitize": true,
"maxLength": 256
}
}
}
},
"missing_tool": {
"defaultMax": 20,
"fields": {
"alternatives": {
"type": "string",
"sanitize": true,
"maxLength": 512
},
"reason": {
"required": true,
"type": "string",
"sanitize": true,
"maxLength": 256
},
"tool": {
"type": "string",
"sanitize": true,
"maxLength": 128
},
"missing_tool": {
"defaultMax": 20,
"fields": {
"alternatives": {
"type": "string",
"sanitize": true,
"maxLength": 512
},
"reason": {
"required": true,
"type": "string",
"sanitize": true,
"maxLength": 256
},
"tool": {
"type": "string",
"sanitize": true,
"maxLength": 128
}
}
}
},
"noop": {
"defaultMax": 1,
"fields": {
"message": {
"required": true,
"type": "string",
"sanitize": true,
"maxLength": 65000
},
"noop": {
"defaultMax": 1,
"fields": {
"message": {
"required": true,
"type": "string",
"sanitize": true,
"maxLength": 65000
}
}
},
"report_incomplete": {
"defaultMax": 5,
"fields": {
"details": {
"type": "string",
"sanitize": true,
"maxLength": 65000
},
"reason": {
"required": true,
"type": "string",
"sanitize": true,
"maxLength": 1024
}
}
}
}
}
GH_AW_SAFE_OUTPUTS_VALIDATION_906347b49a85a96b_EOF
node ${RUNNER_TEMP}/gh-aw/actions/generate_safe_outputs_tools.cjs
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
script: |
const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
setupGlobals(core, github, context, exec, io);
const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs');
await main();
- name: Generate Safe Outputs MCP Server Config
id: safe-outputs-config
run: |
@ -461,6 +513,7 @@ jobs:
id: safe-outputs-start
env:
DEBUG: '*'
GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }}
GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }}
GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ runner.temp }}/gh-aw/safeoutputs/tools.json
@ -469,13 +522,14 @@ jobs:
run: |
# Environment variables are set above to prevent template injection
export DEBUG
export GH_AW_SAFE_OUTPUTS
export GH_AW_SAFE_OUTPUTS_PORT
export GH_AW_SAFE_OUTPUTS_API_KEY
export GH_AW_SAFE_OUTPUTS_TOOLS_PATH
export GH_AW_SAFE_OUTPUTS_CONFIG_PATH
export GH_AW_MCP_LOG_DIR
bash ${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh
bash "${RUNNER_TEMP}/gh-aw/actions/start_safe_outputs_server.sh"
- name: Start MCP Gateway
id: start-mcp-gateway
@ -502,10 +556,10 @@ jobs:
export DEBUG="*"
export GH_AW_ENGINE="copilot"
export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.11'
export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17'
mkdir -p /home/runner/.copilot
cat << GH_AW_MCP_CONFIG_515b057f9cf50300_EOF | bash ${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh
cat << GH_AW_MCP_CONFIG_c990404fbee0ddc5_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh"
{
"mcpServers": {
"github": {
@ -546,7 +600,7 @@ jobs:
"payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}"
}
}
GH_AW_MCP_CONFIG_515b057f9cf50300_EOF
GH_AW_MCP_CONFIG_c990404fbee0ddc5_EOF
- name: Download activation artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
@ -554,7 +608,7 @@ jobs:
path: /tmp/gh-aw
- name: Clean git credentials
continue-on-error: true
run: bash ${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh
run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh"
- name: Execute GitHub Copilot CLI
id: agentic_execution
# Copilot CLI tool arguments (sorted):
@ -563,8 +617,8 @@ jobs:
set -o pipefail
touch /tmp/gh-aw/agent-step-summary.md
# shellcheck disable=SC1003
sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --image-tag 0.25.6 --skip-pull --enable-api-proxy \
-- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log
sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --image-tag 0.25.18 --skip-pull --enable-api-proxy \
-- /bin/bash -c 'node ${RUNNER_TEMP}/gh-aw/actions/copilot_driver.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log
env:
COPILOT_AGENT_RUNNER_TYPE: STANDALONE
COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
@ -573,9 +627,10 @@ jobs:
GH_AW_PHASE: agent
GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
GH_AW_VERSION: v0.65.4
GH_AW_VERSION: v0.67.4
GITHUB_API_URL: ${{ github.api_url }}
GITHUB_AW: true
GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows
GITHUB_HEAD_REF: ${{ github.head_ref }}
GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
GITHUB_REF_NAME: ${{ github.ref_name }}
@ -591,36 +646,24 @@ jobs:
id: detect-inference-error
if: always()
continue-on-error: true
run: bash ${RUNNER_TEMP}/gh-aw/actions/detect_inference_access_error.sh
run: bash "${RUNNER_TEMP}/gh-aw/actions/detect_inference_access_error.sh"
- name: Configure Git credentials
env:
REPO_NAME: ${{ github.repository }}
SERVER_URL: ${{ github.server_url }}
GITHUB_TOKEN: ${{ github.token }}
run: |
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
git config --global am.keepcr true
# Re-authenticate git with GitHub token
SERVER_URL_STRIPPED="${SERVER_URL#https://}"
git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git"
git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git"
echo "Git configured with standard GitHub Actions identity"
- name: Copy Copilot session state files to logs
if: always()
continue-on-error: true
run: |
# Copy Copilot session state files to logs folder for artifact collection
# This ensures they are in /tmp/gh-aw/ where secret redaction can scan them
SESSION_STATE_DIR="$HOME/.copilot/session-state"
LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs"
if [ -d "$SESSION_STATE_DIR" ]; then
echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR"
mkdir -p "$LOGS_DIR"
cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true
echo "Session state files copied successfully"
else
echo "No session-state directory found at $SESSION_STATE_DIR"
fi
run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh"
- name: Stop MCP Gateway
if: always()
continue-on-error: true
@ -629,7 +672,7 @@ jobs:
MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }}
GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }}
run: |
bash ${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID"
bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID"
- name: Redact secrets in logs
if: always()
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
@ -647,7 +690,7 @@ jobs:
SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Append agent step summary
if: always()
run: bash ${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh
run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh"
- name: Copy Safe Outputs
if: always()
env:
@ -683,6 +726,7 @@ jobs:
await main();
- name: Parse MCP Gateway logs for step summary
if: always()
id: parse-mcp-gateway
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
script: |
@ -705,6 +749,16 @@ jobs:
else
echo 'AWF binary not installed, skipping firewall log summary'
fi
- name: Parse token usage for step summary
if: always()
continue-on-error: true
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
script: |
const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
setupGlobals(core, github, context, exec, io);
const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs');
await main();
- name: Write agent output placeholder if missing
if: always()
run: |
@ -722,8 +776,10 @@ jobs:
/tmp/gh-aw/sandbox/agent/logs/
/tmp/gh-aw/redacted-urls.log
/tmp/gh-aw/mcp-logs/
/tmp/gh-aw/agent_usage.json
/tmp/gh-aw/agent-stdio.log
/tmp/gh-aw/agent/
/tmp/gh-aw/github_rate_limits.jsonl
/tmp/gh-aw/safeoutputs.jsonl
/tmp/gh-aw/agent_output.json
/tmp/gh-aw/aw-*.patch
@ -756,14 +812,18 @@ jobs:
group: "gh-aw-conclusion-qf-s-benchmark"
cancel-in-progress: false
outputs:
incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }}
noop_message: ${{ steps.noop.outputs.noop_message }}
tools_reported: ${{ steps.missing_tool.outputs.tools_reported }}
total_count: ${{ steps.missing_tool.outputs.total_count }}
steps:
- name: Setup Scripts
uses: github/gh-aw-actions/setup@934698b44320d87a7a9196339f90293f10bd2247 # v0.65.4
id: setup
uses: github/gh-aw-actions/setup@v0.67.4
with:
destination: ${{ runner.temp }}/gh-aw/actions
job-name: ${{ github.job }}
trace-id: ${{ needs.activation.outputs.setup-trace-id }}
- name: Download agent output artifact
id: download-agent-output
continue-on-error: true
@ -784,22 +844,25 @@ jobs:
env:
GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
GH_AW_NOOP_MAX: "1"
GH_AW_WORKFLOW_NAME: "ZIPT String Solver Benchmark"
GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }}
GH_AW_NOOP_REPORT_AS_ISSUE: "false"
with:
github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
script: |
const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
setupGlobals(core, github, context, exec, io);
const { main } = require('${{ runner.temp }}/gh-aw/actions/noop.cjs');
const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs');
await main();
- name: Record Missing Tool
- name: Record missing tool
id: missing_tool
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
GH_AW_MISSING_TOOL_CREATE_ISSUE: "true"
GH_AW_MISSING_TOOL_TITLE_PREFIX: "[missing tool]"
GH_AW_WORKFLOW_NAME: "ZIPT String Solver Benchmark"
GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
with:
github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
script: |
@ -807,13 +870,27 @@ jobs:
setupGlobals(core, github, context, exec, io);
const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs');
await main();
- name: Handle Agent Failure
- name: Record incomplete
id: report_incomplete
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true"
GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
with:
github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
script: |
const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
setupGlobals(core, github, context, exec, io);
const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs');
await main();
- name: Handle agent failure
id: handle_agent_failure
if: always()
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
GH_AW_WORKFLOW_NAME: "ZIPT String Solver Benchmark"
GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }}
GH_AW_WORKFLOW_ID: "qf-s-benchmark"
@ -834,37 +911,27 @@ jobs:
setupGlobals(core, github, context, exec, io);
const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs');
await main();
- name: Handle No-Op Message
id: handle_noop_message
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
GH_AW_WORKFLOW_NAME: "ZIPT String Solver Benchmark"
GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }}
GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }}
GH_AW_NOOP_REPORT_AS_ISSUE: "false"
with:
github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
script: |
const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
setupGlobals(core, github, context, exec, io);
const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs');
await main();
detection:
needs: agent
needs:
- activation
- agent
if: >
always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true')
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }}
detection_success: ${{ steps.detection_conclusion.outputs.success }}
steps:
- name: Setup Scripts
uses: github/gh-aw-actions/setup@934698b44320d87a7a9196339f90293f10bd2247 # v0.65.4
id: setup
uses: github/gh-aw-actions/setup@v0.67.4
with:
destination: ${{ runner.temp }}/gh-aw/actions
job-name: ${{ github.job }}
trace-id: ${{ needs.activation.outputs.setup-trace-id }}
- name: Download agent output artifact
id: download-agent-output
continue-on-error: true
@ -879,9 +946,14 @@ jobs:
mkdir -p /tmp/gh-aw/
find "/tmp/gh-aw/" -type f -print
echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT"
- name: Checkout repository for patch context
if: needs.agent.outputs.has_patch == 'true'
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
# --- Threat Detection ---
- name: Download container images
run: bash ${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.25.6 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.6 ghcr.io/github/gh-aw-firewall/squid:0.25.6
run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.25.18 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.18 ghcr.io/github/gh-aw-firewall/squid:0.25.18
- name: Check if detection needed
id: detection_guard
if: always()
@ -920,8 +992,8 @@ jobs:
if: always() && steps.detection_guard.outputs.run_detection == 'true'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
WORKFLOW_NAME: "ZIPT String Solver Benchmark"
WORKFLOW_DESCRIPTION: "Run Z3 string solver benchmarks (seq vs nseq) on QF_S test suite from the c3 branch and post results as a GitHub discussion"
WORKFLOW_NAME: "QF_S String Solver Benchmark"
WORKFLOW_DESCRIPTION: "Benchmark Z3 seq vs nseq string solvers on QF_S test suite from the c3 branch and post results as a GitHub discussion"
HAS_PATCH: ${{ needs.agent.outputs.has_patch }}
with:
script: |
@ -935,9 +1007,11 @@ jobs:
mkdir -p /tmp/gh-aw/threat-detection
touch /tmp/gh-aw/threat-detection/detection.log
- name: Install GitHub Copilot CLI
run: ${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh latest
run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.20
env:
GH_HOST: github.com
- name: Install AWF binary
run: bash ${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh v0.25.6
run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.25.18
- name: Execute GitHub Copilot CLI
if: always() && steps.detection_guard.outputs.run_detection == 'true'
id: detection_agentic_execution
@ -947,17 +1021,18 @@ jobs:
set -o pipefail
touch /tmp/gh-aw/agent-step-summary.md
# shellcheck disable=SC1003
sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,github.com,host.docker.internal,telemetry.enterprise.githubcopilot.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --image-tag 0.25.6 --skip-pull --enable-api-proxy \
-- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log
sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --allow-domains api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,github.com,host.docker.internal,telemetry.enterprise.githubcopilot.com --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --image-tag 0.25.18 --skip-pull --enable-api-proxy \
-- /bin/bash -c 'node ${RUNNER_TEMP}/gh-aw/actions/copilot_driver.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log
env:
COPILOT_AGENT_RUNNER_TYPE: STANDALONE
COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }}
GH_AW_PHASE: detection
GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
GH_AW_VERSION: v0.65.4
GH_AW_VERSION: v0.67.4
GITHUB_API_URL: ${{ github.api_url }}
GITHUB_AW: true
GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows
GITHUB_HEAD_REF: ${{ github.head_ref }}
GITHUB_REF_NAME: ${{ github.ref_name }}
GITHUB_SERVER_URL: ${{ github.server_url }}
@ -990,6 +1065,7 @@ jobs:
safe_outputs:
needs:
- activation
- agent
- detection
if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success'
@ -1001,10 +1077,11 @@ jobs:
timeout-minutes: 15
env:
GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/qf-s-benchmark"
GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }}
GH_AW_ENGINE_ID: "copilot"
GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }}
GH_AW_WORKFLOW_ID: "qf-s-benchmark"
GH_AW_WORKFLOW_NAME: "ZIPT String Solver Benchmark"
GH_AW_WORKFLOW_NAME: "QF_S String Solver Benchmark"
outputs:
code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }}
code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }}
@ -1014,9 +1091,12 @@ jobs:
process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }}
steps:
- name: Setup Scripts
uses: github/gh-aw-actions/setup@934698b44320d87a7a9196339f90293f10bd2247 # v0.65.4
id: setup
uses: github/gh-aw-actions/setup@v0.67.4
with:
destination: ${{ runner.temp }}/gh-aw/actions
job-name: ${{ github.job }}
trace-id: ${{ needs.activation.outputs.setup-trace-id }}
- name: Download agent output artifact
id: download-agent-output
continue-on-error: true
@ -1048,7 +1128,7 @@ jobs:
GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com"
GITHUB_SERVER_URL: ${{ github.server_url }}
GITHUB_API_URL: ${{ github.api_url }}
GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_discussion\":{\"category\":\"agentic workflows\",\"close_older_discussions\":true,\"expires\":168,\"fallback_to_issue\":true,\"max\":1,\"title_prefix\":\"[ZIPT Benchmark] \"},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"}}"
GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_discussion\":{\"category\":\"agentic workflows\",\"close_older_discussions\":true,\"expires\":168,\"fallback_to_issue\":true,\"max\":1,\"title_prefix\":\"[QF_S Benchmark] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}"
with:
github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
script: |
@ -1056,11 +1136,11 @@ jobs:
setupGlobals(core, github, context, exec, io);
const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs');
await main();
- name: Upload Safe Output Items
- name: Upload Safe Outputs Items
if: always()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
with:
name: safe-output-items
name: safe-outputs-items
path: /tmp/gh-aw/safe-output-items.jsonl
if-no-files-found: ignore