diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bfef85..46ce8f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,19 @@ jobs: uv run fintracker openapi /tmp/openapi.json diff -u ../openapi/openapi.json /tmp/openapi.json + backend-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + # build only (no push): a broken Dockerfile should fail here, not on the VPS + - uses: docker/build-push-action@v6 + with: + context: backend + push: false + cache-from: type=gha + cache-to: type=gha,mode=max + app: runs-on: ubuntu-latest defaults: diff --git a/backend/Dockerfile b/backend/Dockerfile index 9d815f2..6a7e60d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -21,7 +21,6 @@ COPY --from=builder /app/src /app/src COPY --from=builder /app/alembic.ini /app/alembic.ini COPY --from=builder /app/alembic /app/alembic COPY certs /app/certs -RUN mkdir -p /data/uploads && chown -R app /data USER app EXPOSE 8000 CMD ["fintracker", "serve", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 9b27ef9..5afafea 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -74,7 +74,7 @@ include = ["src", "tests"] extraPaths = ["src"] pythonVersion = "3.12" typeCheckingMode = "standard" -reportMissingImports = "warning" +reportMissingImports = "error" [tool.pytest.ini_options] pythonpath = ["src"] diff --git a/backend/src/fintracker/api/routers/health.py b/backend/src/fintracker/api/routers/health.py index 5471bb3..02214f4 100644 --- a/backend/src/fintracker/api/routers/health.py +++ b/backend/src/fintracker/api/routers/health.py @@ -1,6 +1,6 @@ from __future__ import annotations -from fastapi import APIRouter +from fastapi import APIRouter, Response, status from pydantic import BaseModel from sqlalchemy import text @@ -17,10 +17,12 @@ class Health(BaseModel): @router.get("/health", name="check") -async def check(session: SessionDep) -> Health: +async def check(session: SessionDep, response: Response) -> Health: + """503 when the database cannot be reached, so a container healthcheck can trust the code.""" try: await session.execute(text("SELECT 1")) db = "ok" except Exception as exc: db = f"error: {type(exc).__name__}" + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE return Health(status="ok" if db == "ok" else "degraded", version=__version__, database=db) diff --git a/backend/src/fintracker/api/routers/imports.py b/backend/src/fintracker/api/routers/imports.py index dfd1762..c166ef3 100644 --- a/backend/src/fintracker/api/routers/imports.py +++ b/backend/src/fintracker/api/routers/imports.py @@ -20,7 +20,7 @@ from typing import Annotated from fastapi import APIRouter, File, Form, Query, UploadFile from sqlalchemy import select -from fintracker.api.deps import CurrentUser, SessionDep +from fintracker.api.deps import CurrentUser, SessionDep, SettingsDep from fintracker.api.errors import Problem from fintracker.api.schemas.imports import ( AccountSuggestion, @@ -64,13 +64,19 @@ def _problem(exc: ImportProblem) -> Problem: @router.post("", name="create") async def create_import( session: SessionDep, + settings: SettingsDep, _: CurrentUser, file: Annotated[UploadFile, File(description="the report itself")], account_id: Annotated[int | None, Form(description="target account, if known")] = None, parser: Annotated[str | None, Form(description="force a parser from registry.names()")] = None, ) -> ImportPreview: """Upload a report, parse it and show what committing it would do. Writes no event.""" - data = await file.read() + limit = settings.max_upload_bytes + data = await file.read(limit + 1) + if len(data) > limit: + raise Problem( + 413, "Payload Too Large", f"Report is larger than {limit // (1024 * 1024)} MiB" + ) try: outcome = await report_import.upload( session, diff --git a/backend/tests/api/test_imports_api.py b/backend/tests/api/test_imports_api.py index b2cdd00..4cda872 100644 --- a/backend/tests/api/test_imports_api.py +++ b/backend/tests/api/test_imports_api.py @@ -459,3 +459,16 @@ async def test_pending_rows_are_created_by_the_preview_alone(app, http, auth_hea async with get_sessionmaker()() as session: assert await session.scalar(select(func.count()).select_from(PendingInstrument)) == 1 assert await session.scalar(select(func.count()).select_from(Event)) == 0 + + +async def test_oversized_report_is_413(app, http, auth_headers, parser, monkeypatch): + from fintracker.config import get_settings + + monkeypatch.setenv("MAX_UPLOAD_BYTES", "1024") + get_settings.cache_clear() + try: + response = await upload(http, auth_headers, b"x" * 2048) + finally: + monkeypatch.undo() + get_settings.cache_clear() + assert response.status_code == 413, response.text diff --git a/backend/tests/test_health.py b/backend/tests/test_health.py index 1f7fa42..9625894 100644 --- a/backend/tests/test_health.py +++ b/backend/tests/test_health.py @@ -4,3 +4,20 @@ async def test_health_reports_db(client): body = r.json() assert body["status"] == "ok" assert body["database"] == "ok" + + +async def test_health_is_503_when_the_database_is_down(app, client): + from fintracker.api.deps import get_session + + class _Broken: + async def execute(self, *_a, **_k): + raise ConnectionRefusedError + + async def broken(): + yield _Broken() + + app.dependency_overrides[get_session] = broken + r = await client.get("/api/v1/health") + assert r.status_code == 503 + assert r.json()["status"] == "degraded" + assert r.json()["database"].startswith("error") diff --git a/backend/tests/test_migrations.py b/backend/tests/test_migrations.py new file mode 100644 index 0000000..844d89b --- /dev/null +++ b/backend/tests/test_migrations.py @@ -0,0 +1,20 @@ +"""Models and Alembic history must describe the same schema — the `alembic check` of this +project. `migrated` upgrades a throwaway database to head, so a model change committed +without its revision shows up here as a non-empty diff.""" + +from __future__ import annotations + +from alembic.autogenerate import compare_metadata +from alembic.migration import MigrationContext + + +async def test_migrations_match_the_models(migrated: str): + import fintracker.models # noqa: F401 — registers every table on Base.metadata + from fintracker.db import get_engine + from fintracker.db.base import Base + + async with get_engine().connect() as conn: + diff = await conn.run_sync( + lambda c: compare_metadata(MigrationContext.configure(c), Base.metadata) + ) + assert diff == [], f"models changed without a migration (run `just revision`): {diff}" diff --git a/deploy/Caddyfile b/deploy/Caddyfile index ca5e292..0304625 100644 --- a/deploy/Caddyfile +++ b/deploy/Caddyfile @@ -2,6 +2,10 @@ encode zstd gzip handle /api/* { + # outer guard; the API itself answers 413 for a report over MAX_UPLOAD_BYTES (25 MiB) + request_body { + max_size 30MB + } reverse_proxy api:8000 } diff --git a/docker-compose.yml b/docker-compose.yml index 386fc98..12c3c76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,10 @@ services: db: image: docker.io/library/postgres:17-alpine restart: unless-stopped - env_file: .env environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_INITDB_ARGS: "--encoding=UTF8 --no-locale" volumes: - pgdata:/var/lib/postgresql/data @@ -13,16 +15,39 @@ services: timeout: 3s retries: 20 + # one-shot: api and worker start only after the schema is at head, so neither races the other + migrate: + build: ./backend + environment: + DATABASE_URL: ${DATABASE_URL} + command: ["alembic", "upgrade", "head"] + depends_on: + db: + condition: service_healthy + api: build: ./backend restart: unless-stopped env_file: .env - command: ["sh", "-c", "alembic upgrade head && exec fintracker serve --host 0.0.0.0 --port 8000"] + command: ["fintracker", "serve", "--host", "0.0.0.0", "--port", "8000"] + # not curl: the slim image has none. An empty ProxyHandler because a host proxy baked into + # the image at build time (see docs/ai/ops.md) would otherwise swallow 127.0.0.1. + healthcheck: + test: + - CMD + - python + - -c + - >- + import sys, urllib.request as u; + o = u.build_opener(u.ProxyHandler({})); + sys.exit(0 if o.open("http://127.0.0.1:8000/api/v1/health", timeout=4).status == 200 else 1) + interval: 30s + timeout: 6s + retries: 3 + start_period: 30s depends_on: - db: - condition: service_healthy - volumes: - - uploads:/data/uploads + migrate: + condition: service_completed_successfully expose: - "8000" @@ -31,16 +56,28 @@ services: restart: unless-stopped env_file: .env command: ["fintracker", "worker"] + # the worker has no port; it touches a file every 10 s (`worker/scheduler.py: _heartbeat`) + healthcheck: + test: + - CMD + - python + - -c + - >- + import os, sys, time; + sys.exit(0 if time.time() - os.path.getmtime("/tmp/fintracker-worker-heartbeat") < 60 else 1) + interval: 30s + timeout: 5s + retries: 3 + start_period: 60s depends_on: - db: - condition: service_healthy - volumes: - - uploads:/data/uploads + migrate: + condition: service_completed_successfully caddy: image: docker.io/library/caddy:2-alpine restart: unless-stopped - env_file: .env + environment: + DOMAIN: ${DOMAIN:?set DOMAIN in .env} ports: - "80:80" - "443:443" @@ -55,7 +92,6 @@ services: pg-backup: image: docker.io/library/postgres:17-alpine restart: unless-stopped - env_file: .env entrypoint: ["sh", "-c"] # nightly logical dump, keep 14 days; copy ./backups off-site with rclone/cron on the host command: @@ -67,6 +103,8 @@ services: sleep 86400 done environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} PGPASSWORD: ${POSTGRES_PASSWORD} volumes: - ./backups:/backups @@ -76,6 +114,5 @@ services: volumes: pgdata: - uploads: caddy_data: caddy_config: diff --git a/docs/ai/ops.md b/docs/ai/ops.md index 19e9a57..ea8bc70 100644 --- a/docs/ai/ops.md +++ b/docs/ai/ops.md @@ -4,13 +4,19 @@ ```bash cp .env.example .env # заполнить POSTGRES_PASSWORD, JWT_SECRET (openssl rand -hex 32), DOMAIN, токены -just up # docker compose up -d --build: db, api (миграции при старте), worker, caddy, pg-backup +just up # docker compose up -d --build: db, migrate (one-shot `alembic upgrade head`), api, worker, caddy, pg-backup docker compose exec api fintracker create-user you@example.com ``` - Caddy: авто-TLS на `DOMAIN`, `/api/*` → api:8000, остальное — Flutter web из `app/build/web`. - Worker — единственный экземпляр планировщика; ручной синк через `POST /api/v1/sync/{source}` ставит задачу в `sync_job`, worker забирает её раз в 5 с. +- Расписание — `worker/jobs.default_schedule()`: zenmoney каждые 30 мин, cbr 13:45 и 18:00 МСК, + tinvest каждые 3 ч с 8:10, moex в 10:20/14:20/19:20/23:20, tinvest_events и moex_payouts раз в + сутки утром. tinvest* без `TINVEST_TOKEN` в расписание не берутся (в логе worker'а — warning). +- Healthcheck: `api` ходит на `/api/v1/health` (503, если БД недоступна), `worker` пишет файл + `/tmp/fintracker-worker-heartbeat` каждые 10 с, проверка смотрит на его возраст (< 60 с). + Compose сам не перезапускает unhealthy-контейнер — статус виден в `docker compose ps`. - `pg-backup` делает `pg_dump -Fc` раз в сутки в `./backups`, хранит 14 дней; off-site копию настраивает хост (rclone/cron), см. открытый вопрос в плане. @@ -26,6 +32,7 @@ docker compose up -d ## Секреты Только `.env` на сервере. В чат и в git не попадают. T-Invest токен — read-only. +В compose секреты раздаются по потребности: `api`/`worker` читают весь `.env`, `migrate` получает только `DATABASE_URL`, `db` и `pg-backup` — `POSTGRES_*`, `caddy` — `DOMAIN`. ## Проверено с podman (2026-09-17) @@ -33,4 +40,4 @@ docker compose up -d - Имена образов в `docker-compose.yml` и `Dockerfile` полностью квалифицированы (`docker.io/library/...`) — podman без `unqualified-search-registries` короткие имена не резолвит. - **Прокси хоста впекается в образ** при `podman build`/`compose --build` (`HTTP_PROXY` попадает в `Config.Env`), после чего запросы к `localhost` внутри контейнера идут в недостижимый прокси. Поэтому `just up` собирает с `env -u …proxy…`. На VPS без прокси это ни на что не влияет. - `env_file: .env` в compose — буквальный путь; `--env-file` меняет только подстановку `${…}` в самом compose-файле. -- Команда `api` использует `exec`, иначе `sh -c` не передаёт SIGTERM uvicorn'у и compose добивает контейнер по таймауту. +- Миграции идут в one-shot сервисе `migrate`; `api` и `worker` ждут его `service_completed_successfully`. Для podman-compose это условие не проверялось.