340 lines
9.3 KiB
Bash
Executable File
340 lines
9.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
|
|
die() {
|
|
echo "FAIL: $1" >&2
|
|
exit 1
|
|
}
|
|
|
|
make_fake_codex() {
|
|
local dir="$1"
|
|
local exit_code="$2"
|
|
local message="$3"
|
|
|
|
cat > "$dir/codex" <<SH
|
|
#!/bin/sh
|
|
printf '%s\n' "\$*" > "\${CODEX_FAKE_ARGS:-/dev/null}"
|
|
echo "$message"
|
|
exit $exit_code
|
|
SH
|
|
chmod +x "$dir/codex"
|
|
}
|
|
|
|
make_fake_sleeping_codex() {
|
|
local dir="$1"
|
|
|
|
cat > "$dir/codex" <<'SH'
|
|
#!/bin/sh
|
|
trap 'exit 143' TERM
|
|
while true; do
|
|
sleep 1
|
|
done
|
|
SH
|
|
chmod +x "$dir/codex"
|
|
}
|
|
|
|
make_fake_signaled_codex() {
|
|
local dir="$1"
|
|
|
|
cat > "$dir/codex" <<'SH'
|
|
#!/bin/sh
|
|
kill -TERM $$
|
|
SH
|
|
chmod +x "$dir/codex"
|
|
}
|
|
|
|
make_fake_large_output_codex() {
|
|
local dir="$1"
|
|
|
|
cat > "$dir/codex" <<'SH'
|
|
#!/bin/sh
|
|
printf '0123456789'
|
|
SH
|
|
chmod +x "$dir/codex"
|
|
}
|
|
|
|
make_fake_transcript_codex() {
|
|
local dir="$1"
|
|
|
|
cat > "$dir/codex" <<'SH'
|
|
#!/bin/sh
|
|
last_message_file=""
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = "--output-last-message" ]; then
|
|
shift
|
|
last_message_file="$1"
|
|
fi
|
|
shift
|
|
done
|
|
|
|
printf '%s\n' 'codex' 'final answer' 'tokens used' '123' 'final answer'
|
|
printf '%s\n' 'final answer' > "$last_message_file"
|
|
SH
|
|
chmod +x "$dir/codex"
|
|
}
|
|
|
|
encoded_json() {
|
|
printf '%s' "$1" | base64 | tr -d '\n'
|
|
}
|
|
|
|
run_entrypoint() {
|
|
local tmpdir="$1"
|
|
shift
|
|
|
|
env \
|
|
PATH="$tmpdir:$PATH" \
|
|
CODEX_HOME="$tmpdir/codex-home" \
|
|
GITHUB_OUTPUT="$tmpdir/github-output" \
|
|
"$@" \
|
|
bash "$ROOT_DIR/entrypoint.sh" > "$tmpdir/stdout" 2> "$tmpdir/stderr"
|
|
}
|
|
|
|
assert_status() {
|
|
local actual="$1"
|
|
local expected="$2"
|
|
|
|
[[ "$actual" -eq "$expected" ]] || die "expected status $expected, got $actual"
|
|
}
|
|
|
|
test_missing_oauth() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_codex "$tmpdir" 0 "unused"
|
|
|
|
set +e
|
|
run_entrypoint "$tmpdir" MODEL="gpt-test" PROMPT="hello"
|
|
local status="$?"
|
|
set -e
|
|
|
|
assert_status "$status" 1
|
|
grep -q "OAUTH is required" "$tmpdir/stderr" || die "missing OAUTH error"
|
|
grep -q "status=failed" "$tmpdir/github-output" || die "missing OAUTH failed status output"
|
|
grep -q "OAUTH is required" "$tmpdir/github-output" || die "missing OAUTH output"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_missing_model() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_codex "$tmpdir" 0 "unused"
|
|
|
|
set +e
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '{}')" PROMPT="hello"
|
|
local status="$?"
|
|
set -e
|
|
|
|
assert_status "$status" 1
|
|
grep -q "MODEL is required" "$tmpdir/stderr" || die "missing MODEL error"
|
|
grep -q "status=failed" "$tmpdir/github-output" || die "missing MODEL failed status output"
|
|
grep -q "MODEL is required" "$tmpdir/github-output" || die "missing MODEL output"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_invalid_base64() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_codex "$tmpdir" 0 "unused"
|
|
|
|
set +e
|
|
run_entrypoint "$tmpdir" OAUTH="not-base64" MODEL="gpt-test" PROMPT="hello"
|
|
local status="$?"
|
|
set -e
|
|
|
|
assert_status "$status" 1
|
|
grep -q "valid base64" "$tmpdir/stderr" || die "invalid base64 error"
|
|
grep -q "status=failed" "$tmpdir/github-output" || die "invalid base64 failed status output"
|
|
grep -q "valid base64" "$tmpdir/github-output" || die "invalid base64 output"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_non_object_json() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_codex "$tmpdir" 0 "unused"
|
|
|
|
set +e
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '[]')" MODEL="gpt-test" PROMPT="hello"
|
|
local status="$?"
|
|
set -e
|
|
|
|
assert_status "$status" 1
|
|
grep -q "JSON object" "$tmpdir/stderr" || die "non-object JSON error"
|
|
grep -q "status=failed" "$tmpdir/github-output" || die "non-object JSON failed status output"
|
|
grep -q "JSON object" "$tmpdir/github-output" || die "non-object JSON output"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_success_output() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_codex "$tmpdir" 0 "codex ok"
|
|
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '{}')" MODEL="gpt-test" PROMPT="hello"
|
|
|
|
grep -q "status=completed" "$tmpdir/github-output" || die "completed status output"
|
|
grep -q "codex ok" "$tmpdir/github-output" || die "codex output"
|
|
[[ ! -e "$tmpdir/codex-home/auth.json" ]] || die "auth.json was not cleaned up"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_empty_prompt() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_codex "$tmpdir" 0 "codex ok"
|
|
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '{}')" MODEL="gpt-test" PROMPT=""
|
|
|
|
grep -q "codex ok" "$tmpdir/github-output" || die "empty prompt output"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_prompt_with_shell_characters() {
|
|
local tmpdir
|
|
local prompt
|
|
tmpdir="$(mktemp -d)"
|
|
prompt='hello; rm -rf / $(echo bad) "quoted"'
|
|
make_fake_codex "$tmpdir" 0 "codex ok"
|
|
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '{}')" MODEL="gpt-test" PROMPT="$prompt" CODEX_FAKE_ARGS="$tmpdir/codex-args"
|
|
|
|
grep -qF "$prompt" "$tmpdir/codex-args" || die "prompt was not passed as one argument"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_failure_output() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_codex "$tmpdir" 7 "codex failed"
|
|
|
|
set +e
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '{}')" MODEL="gpt-test" PROMPT="hello"
|
|
local status="$?"
|
|
set -e
|
|
|
|
assert_status "$status" 7
|
|
grep -q "status=failed" "$tmpdir/github-output" || die "failed status output"
|
|
grep -q "codex failed" "$tmpdir/github-output" || die "failed codex output"
|
|
[[ ! -e "$tmpdir/codex-home/auth.json" ]] || die "auth.json was not cleaned up after failure"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_missing_codex_command_output() {
|
|
local tmpdir
|
|
local bindir
|
|
tmpdir="$(mktemp -d)"
|
|
bindir="$tmpdir/bin"
|
|
mkdir -p "$bindir"
|
|
ln -s "$(command -v bash)" "$bindir/bash"
|
|
ln -s "$(command -v dirname)" "$bindir/dirname"
|
|
ln -s "$(command -v node)" "$bindir/node"
|
|
|
|
set +e
|
|
run_entrypoint "$tmpdir" PATH="$bindir" OAUTH="$(encoded_json '{}')" MODEL="gpt-test" PROMPT="hello"
|
|
local status="$?"
|
|
set -e
|
|
|
|
assert_status "$status" 1
|
|
grep -q "status=failed" "$tmpdir/github-output" || die "missing codex command failed status output"
|
|
grep -q "Unable to find codex command" "$tmpdir/github-output" || die "missing codex command error output"
|
|
[[ ! -e "$tmpdir/codex-home/auth.json" ]] || die "auth.json was not cleaned up after missing codex"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_codex_timeout_output() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_sleeping_codex "$tmpdir"
|
|
|
|
set +e
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '{}')" MODEL="gpt-test" PROMPT="hello" CODEX_TIMEOUT_MS="100"
|
|
local status="$?"
|
|
set -e
|
|
|
|
assert_status "$status" 143
|
|
grep -q "status=failed" "$tmpdir/github-output" || die "timeout failed status output"
|
|
grep -q "Codex execution timed out after 100 ms" "$tmpdir/github-output" || die "timeout message output"
|
|
[[ ! -e "$tmpdir/codex-home/auth.json" ]] || die "auth.json was not cleaned up after timeout"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_codex_signal_output() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_signaled_codex "$tmpdir"
|
|
|
|
set +e
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '{}')" MODEL="gpt-test" PROMPT="hello"
|
|
local status="$?"
|
|
set -e
|
|
|
|
assert_status "$status" 1
|
|
grep -q "status=failed" "$tmpdir/github-output" || die "signal failed status output"
|
|
grep -q "Codex process terminated by signal SIGTERM" "$tmpdir/github-output" || die "signal message output"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_codex_output_truncation() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_large_output_codex "$tmpdir"
|
|
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '{}')" MODEL="gpt-test" PROMPT="hello" CODEX_OUTPUT_LIMIT_BYTES="5"
|
|
|
|
grep -q "status=completed" "$tmpdir/github-output" || die "truncated output completed status"
|
|
grep -q "01234" "$tmpdir/github-output" || die "truncated output prefix"
|
|
grep -q "\\[Output truncated\\]" "$tmpdir/github-output" || die "truncation message output"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_last_message_output_without_transcript_duplicate() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_transcript_codex "$tmpdir"
|
|
|
|
run_entrypoint "$tmpdir" OAUTH="$(encoded_json '{}')" MODEL="gpt-test" PROMPT="hello"
|
|
|
|
[[ "$(grep -c "final answer" "$tmpdir/stdout")" -eq 1 ]] || die "stdout printed final answer more than once"
|
|
grep -q "status=completed" "$tmpdir/github-output" || die "last message completed status"
|
|
[[ "$(grep -c "final answer" "$tmpdir/github-output")" -eq 1 ]] || die "github output included transcript duplicate"
|
|
! grep -q "tokens used" "$tmpdir/github-output" || die "github output included codex transcript"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_invalid_execution_config_uses_defaults() {
|
|
local tmpdir
|
|
tmpdir="$(mktemp -d)"
|
|
make_fake_large_output_codex "$tmpdir"
|
|
|
|
run_entrypoint "$tmpdir" \
|
|
OAUTH="$(encoded_json '{}')" \
|
|
MODEL="gpt-test" \
|
|
PROMPT="hello" \
|
|
CODEX_TIMEOUT_MS="invalid" \
|
|
CODEX_OUTPUT_LIMIT_BYTES="invalid"
|
|
|
|
grep -q "status=completed" "$tmpdir/github-output" || die "invalid config completed status"
|
|
grep -q "0123456789" "$tmpdir/github-output" || die "invalid output limit did not use default"
|
|
! grep -q "\\[Output truncated\\]" "$tmpdir/github-output" || die "invalid output limit unexpectedly truncated"
|
|
rm -rf "$tmpdir"
|
|
}
|
|
|
|
test_missing_oauth
|
|
test_missing_model
|
|
test_invalid_base64
|
|
test_non_object_json
|
|
test_success_output
|
|
test_empty_prompt
|
|
test_prompt_with_shell_characters
|
|
test_failure_output
|
|
test_missing_codex_command_output
|
|
test_codex_timeout_output
|
|
test_codex_signal_output
|
|
test_codex_output_truncation
|
|
test_last_message_output_without_transcript_duplicate
|
|
test_invalid_execution_config_uses_defaults
|
|
|
|
echo "entrypoint tests passed"
|