commit bb68b6b9f2159d0b0a9a6976ad977049eecc6086 Author: Raghav <1858838+quantumrag@users.noreply.github.com> Date: Thu Apr 16 09:04:22 2026 +0530 Initial commit Co-Authored-By: Oz diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..1478aff --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# Host + network +GITEA_DOMAIN=git.bhatfamily.in +GITEA_ROOT_URL=https://git.bhatfamily.in/ +GITEA_HTTP_PORT=3000 +GITEA_SSH_PORT=2222 +TLS_EMAIL=admin@bhatfamily.in + +# Storage (host path) +GITEA_BASE_PATH=/media/rbhat/DATA/gitea + +# Timezone + permissions +TZ=Asia/Kolkata +PUID=1000 +PGID=1000 + +# Database +POSTGRES_USER=gitea +POSTGRES_PASSWORD=change-me-strong-password +POSTGRES_DB=gitea + +# Gitea secrets (set strong values before internet exposure) +GITEA_SECRET_KEY=change-me-gitea-secret-key +GITEA_INTERNAL_TOKEN=change-me-gitea-internal-token diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2334d82 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.env +*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..e4446e0 --- /dev/null +++ b/README.md @@ -0,0 +1,91 @@ +# Gitea Homelab Automation (`git.bhatfamily.in`) +Automated Docker-based setup for a self-hosted Gitea server with PostgreSQL, persistent storage at `/media/rbhat/DATA/gitea`, and lifecycle scripts for install, test, and uninstall. +## What this repository provides +- `docker-compose.yml` for: + - `gitea/gitea:1.24.2` + - `postgres:16-alpine` + - optional TLS reverse proxy (`caddy:2.10-alpine`, profile: `tls`) +- Idempotent lifecycle scripts: + - `scripts/install.sh` + - `scripts/test.sh` + - `scripts/uninstall.sh` +- Environment template: `.env.example` +- Troubleshooting and network/DNS notes in `docs/` +## Layout +- Host storage root: `/media/rbhat/DATA/gitea` +- Gitea data volume: `/media/rbhat/DATA/gitea/gitea-data` +- Repository root (host): `/media/rbhat/DATA/gitea/gitea-data/git/repositories` +- PostgreSQL data: `/media/rbhat/DATA/gitea/postgres` +- Caddy data/config: `/media/rbhat/DATA/gitea/caddy-data`, `/media/rbhat/DATA/gitea/caddy-config` +## Prerequisites +- Docker + Docker Compose plugin installed +- `curl` installed +- `ufw` optional (if active, scripts add/remove rules for Gitea ports) +- Sudo access to manage firewall rules +## Quick start (baseline, no TLS profile) +1. Copy and edit environment values: + - `cp .env.example .env` + - Change at least: + - `POSTGRES_PASSWORD` + - `GITEA_SECRET_KEY` + - `GITEA_INTERNAL_TOKEN` +2. Install/start stack: + - `./scripts/install.sh` +3. Validate setup: + - `./scripts/test.sh` +4. Open Gitea UI: + - `http://localhost:3000` (or your configured HTTP port) +## Quick start (TLS reverse proxy profile) +1. Ensure `.env` has correct values: + - `GITEA_DOMAIN=git.bhatfamily.in` + - `GITEA_ROOT_URL=https://git.bhatfamily.in/` + - `TLS_EMAIL=` (used by Caddy for ACME account contact) +2. Ensure DNS + router/NAT are configured first (see `docs/cloudflare-networking.md`). +3. Install with TLS profile: + - `./scripts/install.sh --with-tls --open-public-web` +4. Test TLS profile (strict): + - `./scripts/test.sh --with-tls` +5. If DNS/cert is still propagating, run non-blocking external check: + - `./scripts/test.sh --with-tls --allow-pending-external` +6. Access: + - `https://git.bhatfamily.in` +## Uninstall +- Stop and remove containers, keep data: + - `./scripts/uninstall.sh` +- Stop and remove containers including TLS profile: + - `./scripts/uninstall.sh --with-tls` +- Remove added 80/443 firewall rules too (if added with install flag): + - `./scripts/uninstall.sh --with-tls --close-public-web` +- Stop and remove containers and delete persistent data: + - `./scripts/uninstall.sh --with-tls --purge-data` +- Non-interactive full teardown: + - `./scripts/uninstall.sh --with-tls --purge-data --purge-images --close-public-web --yes` +## Port defaults +- Host HTTP: `3000` -> container `3000` +- Host SSH: `2222` -> container `22` +- TLS profile ports: `80`, `443` -> Caddy +## Firewall behavior +When UFW is active: +- install always adds: + - `allow /tcp` (comment: `Gitea HTTP`) + - `allow /tcp` (comment: `Gitea SSH`) +- install with `--open-public-web` also adds: + - `allow 80/tcp` (comment: `Gitea TLS HTTP-01`) + - `allow 443/tcp` (comment: `Gitea TLS HTTPS`) +- uninstall always removes Gitea HTTP/SSH rules +- uninstall with `--close-public-web` removes 80/443 rules +## Cloudflare and home network changes +See `docs/cloudflare-networking.md` for complete instructions. +## Troubleshooting +See `docs/troubleshooting.md` for diagnostics and common fixes. +## Backup basics +- Backup application data: + - `/media/rbhat/DATA/gitea/gitea-data` +- Backup PostgreSQL data: + - `/media/rbhat/DATA/gitea/postgres` +- If TLS profile used, backup Caddy state too: + - `/media/rbhat/DATA/gitea/caddy-data` + - `/media/rbhat/DATA/gitea/caddy-config` +For consistent backups, stop containers first: +- `docker compose --env-file .env -f docker-compose.yml down` +Then archive directories and restart with `./scripts/install.sh` (or with `--with-tls`). diff --git a/caddy/Caddyfile b/caddy/Caddyfile new file mode 100644 index 0000000..0d345c9 --- /dev/null +++ b/caddy/Caddyfile @@ -0,0 +1,19 @@ +{ + email {$TLS_EMAIL} +} + +{$GITEA_DOMAIN} { + encode zstd gzip + reverse_proxy gitea:3000 + + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + X-Content-Type-Options "nosniff" + X-Frame-Options "SAMEORIGIN" + Referrer-Policy "strict-origin-when-cross-origin" + } +} + +:80 { + redir https://{$GITEA_DOMAIN}{uri} permanent +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..44b63c1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,74 @@ +services: + gitea: + image: gitea/gitea:1.24.2 + container_name: gitea + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + env_file: + - .env + environment: + USER_UID: ${PUID} + USER_GID: ${PGID} + GITEA__server__DOMAIN: ${GITEA_DOMAIN} + GITEA__server__ROOT_URL: ${GITEA_ROOT_URL} + GITEA__server__SSH_DOMAIN: ${GITEA_DOMAIN} + GITEA__server__SSH_PORT: ${GITEA_SSH_PORT} + GITEA__server__START_SSH_SERVER: "false" + GITEA__database__DB_TYPE: postgres + GITEA__database__HOST: postgres:5432 + GITEA__database__NAME: ${POSTGRES_DB} + GITEA__database__USER: ${POSTGRES_USER} + GITEA__database__PASSWD: ${POSTGRES_PASSWORD} + GITEA__security__INSTALL_LOCK: "true" + GITEA__security__SECRET_KEY: ${GITEA_SECRET_KEY} + GITEA__security__INTERNAL_TOKEN: ${GITEA_INTERNAL_TOKEN} + ports: + - "${GITEA_HTTP_PORT}:3000" + - "${GITEA_SSH_PORT}:22" + volumes: + - ${GITEA_BASE_PATH}/gitea-data:/data + healthcheck: + test: ["CMD", "wget", "--spider", "--quiet", "http://localhost:3000/api/healthz"] + interval: 20s + timeout: 5s + retries: 10 + start_period: 30s + + postgres: + image: postgres:16-alpine + container_name: gitea-postgres + restart: unless-stopped + env_file: + - .env + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - ${GITEA_BASE_PATH}/postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 20s + + caddy: + image: caddy:2.10-alpine + container_name: gitea-caddy + restart: unless-stopped + profiles: ["tls"] + env_file: + - .env + depends_on: + gitea: + condition: service_healthy + ports: + - "80:80" + - "443:443" + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro + - ${GITEA_BASE_PATH}/caddy-data:/data + - ${GITEA_BASE_PATH}/caddy-config:/config diff --git a/docs/cloudflare-networking.md b/docs/cloudflare-networking.md new file mode 100644 index 0000000..f970a61 --- /dev/null +++ b/docs/cloudflare-networking.md @@ -0,0 +1,52 @@ +# Cloudflare DNS + Home Network Setup for `git.bhatfamily.in` +## Goal +Expose local Gitea securely from home network while preserving Git-over-SSH support. +## Recommended Cloudflare DNS records +Create DNS records in Cloudflare for zone `bhatfamily.in`: +1. `A` record + - Name: `git` + - Content: your home public IPv4 + - Proxy status: **DNS only** (gray cloud) +2. Optional `AAAA` record + - Name: `git` + - Content: your home public IPv6 + - Proxy status: **DNS only** +## Why DNS-only is recommended +This setup uses direct inbound routing for both HTTPS and custom SSH (`2222` by default). DNS-only avoids Cloudflare proxy protocol limitations around direct SSH forwarding. +## Router/NAT rules +Create forwards to this server's LAN IP: +- TCP 80 -> `:80` (ACME challenge + redirect) +- TCP 443 -> `:443` (HTTPS via Caddy) +- TCP 2222 -> `:2222` (Git SSH) +## Firewall alignment +If UFW is active, use install flag to open public web ports: +- `./scripts/install.sh --with-tls --open-public-web` +To close those later: +- `./scripts/uninstall.sh --with-tls --close-public-web` +## ISP constraints check +Some ISPs block inbound ports. Validate from outside your network: +- `curl -I https://git.bhatfamily.in` +- `nc -vz git.bhatfamily.in 2222` +If blocked, use alternate routing (VPN/tunnel) or ISP-compatible ports. +## Recommended hardening +- Keep strong secrets in `.env` +- Restrict SSH source ranges if practical +- Keep containers patched (`docker compose pull` and recreate) +- Add off-host backups for gitea/postgres/caddy data directories + +## Post-cutover verification checklist +Run these after DNS/NAT/firewall updates to confirm end-to-end readiness: +1. DNS resolution + - `dig +short git.bhatfamily.in A` +2. HTTPS response and redirect chain + - `curl -I https://git.bhatfamily.in` + - `curl -I http://git.bhatfamily.in` +3. Certificate validity/issuer + - `openssl s_client -connect git.bhatfamily.in:443 -servername git.bhatfamily.in /dev/null | openssl x509 -noout -subject -issuer -dates` +4. Git SSH port reachability + - `nc -vz git.bhatfamily.in 2222` +5. Stack self-check + - `./scripts/test.sh --with-tls` +If any check fails during first-time propagation, run: +- `./scripts/test.sh --with-tls --allow-pending-external` +Then re-run strict checks once DNS/certificate propagation completes. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..2642260 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,60 @@ +# Troubleshooting +## 1) Containers fail to start +- Check compose status: + - `docker compose --env-file .env -f docker-compose.yml ps` +- If TLS profile enabled: + - `docker compose --env-file .env -f docker-compose.yml --profile tls ps` +- Inspect logs: + - `docker compose --env-file .env -f docker-compose.yml logs gitea` + - `docker compose --env-file .env -f docker-compose.yml logs postgres` + - `docker compose --env-file .env -f docker-compose.yml --profile tls logs caddy` +## 2) Gitea health endpoint fails +- Local probe: + - `curl -v http://localhost:3000/api/healthz` +- If port changed, use your `GITEA_HTTP_PORT`. +- Confirm mapping: + - `docker compose --env-file .env -f docker-compose.yml ps` +## 3) Database connection errors +- Confirm PostgreSQL health: + - `docker compose --env-file .env -f docker-compose.yml ps postgres` +- Re-check `.env` values: + - `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` +- Look for auth failures in postgres logs. +## 4) Permission issues under `/media/rbhat/DATA/gitea` +- Ensure current user can read/write this path. +- If needed: + - `sudo chown -R rbhat:rbhat /media/rbhat/DATA/gitea` + - `sudo chmod -R u+rwX /media/rbhat/DATA/gitea` +## 5) Firewall blocks access +- Check active rules: + - `sudo ufw status` +- Expected allows: + - `/tcp` + - `/tcp` + - `80/tcp` and `443/tcp` if TLS profile is internet-exposed +## 6) TLS cert not issuing +- Ensure `git.bhatfamily.in` resolves publicly to your home WAN IP. +- Ensure inbound TCP 80 and 443 are forwarded to this host. +- Ensure `TLS_EMAIL` is set in `.env`. +- Verify Caddy logs for ACME failures: + - `docker compose --env-file .env -f docker-compose.yml --profile tls logs caddy` +## 7) Port conflicts on 80/443 +- Check listeners: + - `ss -tulpen | grep -E '(:80\s|:443\s)'` +- Stop conflicting services or disable TLS profile until resolved. +## 8) DNS resolves but service unreachable +- Verify router forwarding targets correct LAN IP. +- Verify host listening: + - `ss -tulpen | grep -E '3000|2222|80|443'` +- Test from external network (mobile hotspot) to avoid NAT loopback confusion. +## 9) TLS tests fail during propagation +- Strict mode (default) fails until DNS/routing/cert trust is ready: + - `./scripts/test.sh --with-tls` +- Temporary non-blocking mode: + - `./scripts/test.sh --with-tls --allow-pending-external` +## 10) Reset stack while keeping data +- `./scripts/uninstall.sh --with-tls` +- `./scripts/install.sh --with-tls --open-public-web` +## 11) Full clean rebuild +- `./scripts/uninstall.sh --with-tls --purge-data --purge-images --close-public-web --yes` +- Re-run install diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..363ba45 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" + +WITH_TLS=false +OPEN_PUBLIC_WEB=false + +for arg in "$@"; do + case "$arg" in + --with-tls) WITH_TLS=true ;; + --open-public-web) OPEN_PUBLIC_WEB=true ;; + *) die "Unknown argument: $arg" ;; + esac +done + +log "Starting Gitea install" +require_cmd docker +require_cmd curl + +load_env +if [[ "${WITH_TLS}" == "true" ]]; then + : "${TLS_EMAIL:?TLS_EMAIL is required in .env when using --with-tls}" +fi + +mkdir -p "${GITEA_BASE_PATH}" "${GITEA_BASE_PATH}/gitea-data" "${GITEA_BASE_PATH}/postgres" "${GITEA_BASE_PATH}/backups" "${GITEA_BASE_PATH}/caddy-data" "${GITEA_BASE_PATH}/caddy-config" + +log "Pulling images" +if [[ "${WITH_TLS}" == "true" ]]; then + compose --profile tls pull +else + compose pull +fi + +log "Starting containers" +if [[ "${WITH_TLS}" == "true" ]]; then + compose --profile tls up -d +else + compose up -d +fi + +wait_for_gitea_health +apply_firewall_rules "${OPEN_PUBLIC_WEB}" + +cat <&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Required command not found: $1" +} + +create_env_if_missing() { + if [[ ! -f "${ENV_FILE}" ]]; then + log "No .env found; creating .env from template" + cp "${ENV_EXAMPLE_FILE}" "${ENV_FILE}" + fi +} + +load_env() { + [[ -f "${ENV_EXAMPLE_FILE}" ]] || die "Missing template: ${ENV_EXAMPLE_FILE}" + create_env_if_missing + + set -a + source "${ENV_FILE}" + set +a + + : "${GITEA_BASE_PATH:?GITEA_BASE_PATH is required in .env}" + : "${GITEA_HTTP_PORT:?GITEA_HTTP_PORT is required in .env}" + : "${GITEA_SSH_PORT:?GITEA_SSH_PORT is required in .env}" + : "${GITEA_DOMAIN:?GITEA_DOMAIN is required in .env}" +} + +compose() { + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" "$@" +} + +wait_for_gitea_health() { + local retries=45 + local delay=4 + local url="http://localhost:${GITEA_HTTP_PORT}/api/healthz" + + log "Waiting for Gitea health endpoint: ${url}" + for ((i=1; i<=retries; i++)); do + if curl -fsS "${url}" >/dev/null 2>&1; then + log "Gitea health check passed" + return 0 + fi + sleep "${delay}" + done + + die "Gitea did not become healthy in time" +} + +ufw_active() { + command -v ufw >/dev/null 2>&1 || return 1 + sudo ufw status | grep -q '^Status: active' +} + +apply_firewall_rules() { + local open_public_web="${1:-false}" + + if ufw_active; then + log "Applying UFW rules for Gitea ports" + sudo ufw allow "${GITEA_HTTP_PORT}/tcp" comment 'Gitea HTTP' + sudo ufw allow "${GITEA_SSH_PORT}/tcp" comment 'Gitea SSH' + + if [[ "${open_public_web}" == "true" ]]; then + log "Applying UFW rules for public reverse proxy ports" + sudo ufw allow 80/tcp comment 'Gitea TLS HTTP-01' + sudo ufw allow 443/tcp comment 'Gitea TLS HTTPS' + fi + else + log "UFW not active; skipping firewall rule setup" + fi +} + +remove_firewall_rules() { + local close_public_web="${1:-false}" + + if ufw_active; then + log "Removing UFW rules for Gitea ports" + sudo ufw --force delete allow "${GITEA_HTTP_PORT}/tcp" || true + sudo ufw --force delete allow "${GITEA_SSH_PORT}/tcp" || true + + if [[ "${close_public_web}" == "true" ]]; then + log "Removing UFW rules for public reverse proxy ports" + sudo ufw --force delete allow 80/tcp || true + sudo ufw --force delete allow 443/tcp || true + fi + else + log "UFW not active; skipping firewall rule removal" + fi +} diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..a439de4 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" + +WITH_TLS=false +ALLOW_PENDING_EXTERNAL=false + +for arg in "$@"; do + case "$arg" in + --with-tls) WITH_TLS=true ;; + --allow-pending-external) ALLOW_PENDING_EXTERNAL=true ;; + *) die "Unknown argument: $arg" ;; + esac +done + +load_env +require_cmd curl + +log "Checking docker compose status" +if [[ "${WITH_TLS}" == "true" ]]; then + compose --profile tls ps +else + compose ps +fi + +log "Checking local HTTP health" +curl -fsS "http://localhost:${GITEA_HTTP_PORT}/api/healthz" && printf '\n' + +log "Checking local web root" +curl -I -sS "http://localhost:${GITEA_HTTP_PORT}" | grep -E 'HTTP/[0-9.]+ 200|HTTP/[0-9.]+ 302' + +if command -v getent >/dev/null 2>&1; then + log "Checking DNS resolution for ${GITEA_DOMAIN}" + getent hosts "${GITEA_DOMAIN}" || log "DNS resolution not yet set for ${GITEA_DOMAIN}" +fi + +if command -v nc >/dev/null 2>&1; then + log "Checking SSH TCP port on localhost" + nc -z localhost "${GITEA_SSH_PORT}" +fi + +if [[ "${WITH_TLS}" == "true" ]]; then + log "Checking Caddy container status" + compose --profile tls ps caddy | grep -E 'gitea-caddy|caddy' + + if command -v nc >/dev/null 2>&1; then + log "Checking reverse proxy listener ports" + nc -z localhost 80 + nc -z localhost 443 + fi + + if getent hosts "${GITEA_DOMAIN}" >/dev/null 2>&1; then + log "Checking HTTPS response by domain" + if [[ "${ALLOW_PENDING_EXTERNAL}" == "true" ]]; then + if ! curl -k -I -sS "https://${GITEA_DOMAIN}" | grep -E 'HTTP/[0-9.]+ 200|HTTP/[0-9.]+ 302|HTTP/[0-9.]+ 308'; then + log "HTTPS domain check did not pass yet (pending DNS/routing/certificate propagation)" + fi + else + curl -I -sS "https://${GITEA_DOMAIN}" | grep -E 'HTTP/[0-9.]+ 200|HTTP/[0-9.]+ 302|HTTP/[0-9.]+ 308' + fi + else + if [[ "${ALLOW_PENDING_EXTERNAL}" == "true" ]]; then + log "Skipping strict HTTPS domain check until DNS is configured" + else + die "DNS not configured for ${GITEA_DOMAIN}; rerun with --allow-pending-external if still propagating" + fi + fi +fi + +log "All tests passed" diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh new file mode 100755 index 0000000..037570b --- /dev/null +++ b/scripts/uninstall.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" + +PURGE_DATA=false +PURGE_IMAGES=false +ASSUME_YES=false +WITH_TLS=false +CLOSE_PUBLIC_WEB=false + +for arg in "$@"; do + case "$arg" in + --purge-data) PURGE_DATA=true ;; + --purge-images) PURGE_IMAGES=true ;; + --yes) ASSUME_YES=true ;; + --with-tls) WITH_TLS=true ;; + --close-public-web) CLOSE_PUBLIC_WEB=true ;; + *) die "Unknown argument: $arg" ;; + esac +done + +load_env + +if [[ "${PURGE_DATA}" == "true" && "${ASSUME_YES}" != "true" ]]; then + read -r -p "This will delete ${GITEA_BASE_PATH}/gitea-data and ${GITEA_BASE_PATH}/postgres. Continue? [y/N]: " answer + [[ "${answer}" == "y" || "${answer}" == "Y" ]] || die "Aborted by user" +fi + +log "Stopping and removing containers" +if [[ "${WITH_TLS}" == "true" ]]; then + compose --profile tls down --remove-orphans +else + compose down --remove-orphans +fi + +if [[ "${PURGE_IMAGES}" == "true" ]]; then + log "Removing docker images" + docker image rm gitea/gitea:1.24.2 postgres:16-alpine caddy:2.10-alpine || true +fi + +if [[ "${PURGE_DATA}" == "true" ]]; then + log "Purging persistent data under ${GITEA_BASE_PATH}" + rm -rf "${GITEA_BASE_PATH}/gitea-data" "${GITEA_BASE_PATH}/postgres" "${GITEA_BASE_PATH}/caddy-data" "${GITEA_BASE_PATH}/caddy-config" +fi + +remove_firewall_rules "${CLOSE_PUBLIC_WEB}" + +log "Uninstall complete"