Fix/many problems identified by ai

.github/workflows/ci.yml

@@ -11,10 +11,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # Static analysis: shellcheck every shell script. The `error` severity gate
-  # is a hard failure (the repo is clean at this level today). Warnings are
-  # reported separately and do not fail the build, so they can be cleaned up
-  # incrementally without blocking merges.
+  # Static analysis: shellcheck every shell script. The repo is clean at both
+  # `error` and `warning` severity, so both are hard gates -- this ratchets the
+  # warning count at zero, preventing warnings from silently accumulating.
+  # Genuinely-unavoidable findings are silenced locally with a `# shellcheck
+  # disable=SCxxxx` directive (with a justification) rather than by lowering the
+  # gate. The test jobs `needs: shellcheck`, so a lint failure short-circuits the
+  # (more expensive) matrix instead of running it in parallel regardless.
   shellcheck:
     name: ShellCheck
     runs-on: ubuntu-latest
@@ -24,25 +27,19 @@ jobs:
       - name: Install shellcheck
         run: sudo apt-get update && sudo apt-get install -y shellcheck
 
-      - name: Collect shell scripts
-        id: collect
+      - name: ShellCheck (warning severity, blocking)
         run: |
           # ell and the launcher have no .sh extension; include them explicitly.
-          files="$(git ls-files '*.sh' 'ell' | tr '\n' ' ')"
-          echo "files=${files}" >> "${GITHUB_OUTPUT}"
-          echo "Scripts to check: ${files}"
-
-      - name: ShellCheck (errors, blocking)
-        run: shellcheck -S error ${{ steps.collect.outputs.files }}
-
-      - name: ShellCheck (warnings, non-blocking)
-        continue-on-error: true
-        run: shellcheck -S warning ${{ steps.collect.outputs.files }}
+          # Use NUL-delimited paths piped to xargs -0 so filenames containing
+          # spaces or newlines are handled correctly (rather than word-splitting
+          # an unquoted, space-joined list).
+          git ls-files -z '*.sh' 'ell' | xargs -0 shellcheck -S warning
 
   # Run the full test suite across the oldest and current supported bash
   # versions using the project's own containerised entry point.
   tests:
     name: Tests (bash ${{ matrix.bash }})
+    needs: shellcheck
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -57,3 +54,132 @@ jobs:
       # curl and a global launcher, so no extra setup is needed here.
       - name: Run test suite
         run: bash tests/entry.sh
+
+  # Run the suite with mawk as `awk`. Debian/Ubuntu default to mawk (not the
+  # gawk/busybox awk the container jobs use), and render_to_text.awk is written
+  # to be portable across awk implementations -- this actually exercises that.
+  tests-mawk:
+    name: Tests (mawk)
+    needs: shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install mawk and make it the default awk
+        run: |
+          sudo apt-get update && sudo apt-get install -y mawk
+          # Point /usr/bin/awk at mawk for this run.
+          sudo update-alternatives --set awk /usr/bin/mawk
+          awk -W version 2>&1 | head -1 || awk --version 2>&1 | head -1 || true
+
+      - name: Run test suite
+        run: bash tests/entry.sh
+
+  # Run the suite on macOS to exercise the BSD toolchain (BSD awk/sed/stat/
+  # mktemp, macOS curl) the code is written to support. macOS ships bash 3.2,
+  # which ell rejects, so install a supported bash; GNU `timeout` is absent on
+  # BSD, so provide it from coreutils (as gtimeout) for the test harness.
+  tests-macos:
+    name: Tests (macOS / BSD tools)
+    needs: shellcheck
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install bash and coreutils (for timeout)
+        run: |
+          brew install bash coreutils
+          # Expose gtimeout as `timeout` for the test harness, on PATH ahead of
+          # anything else. The suite otherwise uses the system BSD tools.
+          mkdir -p "${HOME}/bin"
+          ln -sf "$(brew --prefix coreutils)/libexec/gnubin/timeout" "${HOME}/bin/timeout"
+          echo "${HOME}/bin" >> "${GITHUB_PATH}"
+          # Put the newer bash ahead of the system 3.2.
+          echo "$(brew --prefix)/bin" >> "${GITHUB_PATH}"
+
+      - name: Show tool versions
+        run: |
+          bash --version | head -1
+          command -v timeout && timeout --version | head -1
+          awk --version 2>/dev/null | head -1 || echo "awk: BSD (no --version)"
+
+      - name: Run test suite
+        run: bash tests/entry.sh
+
+  # Run the suite on Windows under a real MSYS2 environment (not Git Bash).
+  # MSYS2 provides a fuller POSIX runtime than Git Bash -- consistent msys-2.0
+  # runtime, ACL-backed permissions, real symlinks, and util-linux (script) --
+  # so more of the suite is expected to work here than under Git Bash. Tools are
+  # installed via pacman. This is currently informational (continue-on-error)
+  # until it is proven stable, then it can become a hard gate.
+  tests-msys2:
+    name: Tests (Windows / MSYS2)
+    needs: shellcheck
+    runs-on: windows-latest
+    continue-on-error: true
+    defaults:
+      run:
+        shell: msys2 {0}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up MSYS2
+        uses: msys2/setup-msys2@v2
+        with:
+          msystem: MSYS
+          update: true
+          # bash comes with MSYS2; add the tools the suite uses. coreutils
+          # provides `timeout`; util-linux provides `script` for record tests;
+          # python is used by the real-PTY record test. 
+          install: >-
+            bash
+            coreutils
+            curl
+            gawk
+            sed
+            grep
+            util-linux
+            python
+
+      - name: Show tool versions
+        run: |
+          bash --version | head -1
+          command -v timeout && timeout --version | head -1
+          command -v script && echo "script: present"
+          awk --version 2>/dev/null | head -1 || true
+
+      - name: Run test suite
+        run: bash tests/entry.sh
+
+  # Run the suite on Windows under Git Bash (the default `shell: bash` on the
+  # windows runner). Git Bash is a deliberately limited environment: over NTFS
+  # it cannot create genuinely group/world-writable files or (by default) real
+  # symlinks, and it ships no `script`. Tests that depend on those capabilities
+  # detect the limitation at runtime and SKIP (see docs/Configuration.md,
+  # "Windows"); for a fuller POSIX environment use the tests-msys2 job above.
+  # This job stays informational: it reports what works rather than blocking.
+  tests-windows:
+    name: Tests (Windows / Git Bash, informational)
+    needs: shellcheck
+    runs-on: windows-latest
+    continue-on-error: true
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Show environment
+        run: |
+          bash --version | head -1
+          uname -a || true
+          for t in curl awk sed stat mktemp script stty tty date chmod; do
+            if command -v "${t}" >/dev/null 2>&1; then
+              printf '%-8s %s\n' "${t}" "$(command -v "${t}")"
+            else
+              printf '%-8s MISSING\n' "${t}"
+            fi
+          done
+
+      - name: Run test suite
+        run: bash tests/entry.sh

CHANGELOG.md

@@ -0,0 +1,101 @@
+# Changelog
+
+All notable changes to this project are documented here. The format is based on
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
+adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+This is a large hardening pass focused on security, robustness, and testing.
+
+### Security
+
+- Templates are no longer rendered through the shell. Previously template
+  contents were run through `eval`, so a template (or a prompt / recorded
+  context interpolated into one) could execute arbitrary commands. Rendering is
+  now a strict allowlist substitution with JSON escaping and no shell
+  evaluation.
+- `-O KEY=VALUE` no longer runs through `eval`. It is assigned directly and the
+  key is validated, closing a command-injection hole (`-O 'X=$(cmd)'`).
+  Note: as a consequence of the safe, allowlist-only template renderer, variables
+  set via `-O` are exported into the environment but are **no longer substituted
+  into templates** (the old `eval` renderer did expand them). Only the fixed
+  placeholder allowlist is rendered.
+- Config files are only sourced when trusted: owned by the current user (or
+  root) and not group-/world-writable. This blocks the "hostile `.ellrc` in the
+  current directory" arbitrary-code-execution vector. Refusals are now reported
+  at a visible log level.
+- `ELL_API_STYLE` and template names are validated as single path segments, so
+  they cannot be used to source/load arbitrary files by path traversal.
+- The recorded terminal context (`SHELL_CONTEXT`) now passes through the
+  `post_input` hooks, so redaction applies to it too, not just the prompt.
+- The API key is kept out of the process table: it is passed to curl via a
+  `--config` file (mode 0600, removed after use) instead of a `--header`
+  argument, and it is no longer logged at debug level.
+- ell refuses to send credentials to a plaintext `http://` URL (loopback
+  excepted); override with `ELL_ALLOW_INSECURE_URL=true`.
+- The bundled redaction plugin now ships **disabled by default** (its hook has a
+  `.disabled` suffix); enable it by removing the suffix.
+
+### Fixed
+
+- Streaming completions that finish abnormally (e.g. `length` / `MAX_TOKENS`)
+  are now detected and reported as failures instead of silent success
+  (openai read the finish reason from the wrong path).
+- A value-taking option given with no argument (e.g. `ell -m`) no longer causes
+  an infinite loop; it exits with a usage error.
+- Interactive mode exits cleanly on EOF (Ctrl-D) instead of looping forever, and
+  a final unterminated line is still processed. The exit hint now names Ctrl-D.
+- `-o`/`--output` is accepted as documented (previously only `--output-file`),
+  and output is no longer redirected to a file in record/interactive mode (an
+  always-true comparison was fixed).
+- The `--record` "already enabled" guard now works, and `ELL_RECORD` is
+  exported like the other flags.
+- Record mode re-execs ell by an absolute, shell-quoted path, so it works when
+  ell is run in place or is not on `PATH`.
+- Template resolution rejects path-traversal names and accepts an
+  `ELL_TEMPLATE_PATH` with or without a trailing slash; plugin-hook discovery no
+  longer breaks on paths containing spaces/newlines.
+- Missing `ELL_API_URL` now fails early with a clear, actionable message
+  (EX_CONFIG) instead of an opaque curl error; a missing `ELL_API_KEY` warns.
+- curl failures are reported with a human-readable explanation (DNS, connection
+  refused, timeout, TLS, …) rather than a bare exit code.
+- Functions are exported with `export -f` rather than as empty variables.
+- Bash 4.1 compatibility is restored (the version gate, docs and CI now
+  consistently target 4.1).
+
+### Changed
+
+- HTTP requests go through a shared `ell_curl` wrapper with connection/overall
+  timeouts (`ELL_CONNECT_TIMEOUT`, `ELL_MAX_TIME`) and HTTP error handling
+  (`--fail-with-body`), so a stalled server no longer hangs and a 4xx/5xx no
+  longer surfaces as a vague "Unexpected format".
+- The openai and gemini backends share their non-streaming path, end-of-stream
+  reporting and PIPESTATUS handling via `helpers/backend_common.sh`; each keeps
+  only its provider-specific streaming parser.
+- `helpers/http.sh` moved out of `llm_backends/` for consistency with the other
+  helpers.
+- The default log level is unified across entry points.
+
+### Performance
+
+- The pure-bash JSON parser copies runs of ordinary string characters in one
+  operation and no longer forks a subshell per key lookup.
+- The gemini streaming parser tracks JSON object boundaries instead of
+  re-parsing the whole accumulated buffer per line (was O(n²)).
+- Streaming loops use bash builtins instead of a `grep`/`cut`/`tr` subprocess
+  per line, and logging no longer forks `date`/`basename` per line.
+
+### Added
+
+- Documentation: `docs/Architecture.md`, `docs/Backends.md`, expanded
+  `docs/Templates.md` and `docs/Risk_Consideration.md`, and this changelog.
+- Extensive test coverage: unit tests for argument parsing, config loading, path
+  resolution, the HTTP helper, template rendering, the JSON parser, logging, the
+  syntax-highlight and paginator plugins; and end-to-end tests for the request
+  pipeline, hook stages, interactive mode, output redirection, error paths, the
+  launcher, record mode, the bash version gate and terminal-size fallback.
+
+## [0.1.1]
+
+- Baseline release prior to the hardening pass above.