diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..5de4a18 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,81 @@ +name: Build / Test / Push + +on: + push: + branches: + - "**" + workflow_call: + workflow_dispatch: + +env: + BUILD_SUFFIX: -build-${{ github.run_id }}_${{ github.run_attempt }} + +jobs: + docker-build: + uses: BerkeleyLibrary/.github/.github/workflows/docker-build.yml@3.1.0 + with: + image: ghcr.io/${{ github.repository }} + secrets: inherit + + test: + runs-on: ubuntu-24.04 + needs: docker-build + env: + COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml + DOCKER_APP_IMAGE: ${{ needs.docker-build.outputs.image }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Docker Compose + uses: docker/setup-compose-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Basic setup + run: | + ARTIFACTS_DIR="${RUNNER_TEMP}/artifacts" + mkdir -p "$ARTIFACTS_DIR" + echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> $GITHUB_ENV + echo "TEST_START=$(date +%s)" >> $GITHUB_ENV + - name: Setup the stack + run: | + docker compose run app bin/dbinit + docker compose up --wait + - name: Install testing and linting dependencies + run: | + docker compose exec app pip install --no-cache-dir -e .[test,lint] + - name: Run pytest + run: | + docker compose exec app pytest + + - name: Copy out artifacts + if: ${{ always() }} + run: | + docker compose cp app:/app/artifacts "${ARTIFACTS_DIR}" || true + docker compose logs > "${ARTIFACTS_DIR}/docker-compose-services.log" + docker compose config > "${ARTIFACTS_DIR}/docker-compose.merged.yml" + docker events --json --since $TEST_START --until `date +%s` | tee "${ARTIFACTS_DIR}/docker-events.json" + + - name: Upload the test report + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: quiabo Build Report (${{ github.run_id }}_${{ github.run_attempt }}) + path: ${{ env.ARTIFACTS_DIR }} + if-no-files-found: error + + push: + needs: + - docker-build + - test + uses: BerkeleyLibrary/.github/.github/workflows/docker-push.yml@3.1.0 + with: + image: ghcr.io/${{ github.repository }} + build-image-arm64: ${{ needs.docker-build.outputs.image-arm64 }} + build-image-x64: ${{ needs.docker-build.outputs.image-x64 }} + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8a60fa7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,13 @@ +name: Release + +on: + push: + tags: + - '**' + workflow_dispatch: + +jobs: + release: + uses: BerkeleyLibrary/.github/.github/workflows/docker-release.yml@main + with: + image: ghcr.io/${{ github.repository }} diff --git a/.gitignore b/.gitignore index 83972fa..2abc6e4 100644 --- a/.gitignore +++ b/.gitignore @@ -216,3 +216,7 @@ __marimo__/ # Streamlit .streamlit/secrets.toml + +# other stuff +artifacts/* +uv.lock diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d0bd838 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +FROM python:3.14-slim AS reqs + +ENV APP_USER=quiabo +ENV APP_UID=40098 +ENV VIRTUAL_ENV=/venv + +RUN apt-get update -y && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ + gcc \ + libpq-dev \ + libxml2-dev \ + python3-dev \ + postgresql-client \ + && rm -rf /var/lib/apt/lists/ + +RUN groupadd --system --gid $APP_UID $APP_USER \ + && useradd --home-dir /app --system --uid $APP_UID --gid $APP_USER $APP_USER + +RUN mkdir -p /app && mkdir -p /venv + +RUN chown -R $APP_USER:$APP_USER /app /venv + +USER $APP_USER + +RUN python -m venv /venv +ENV PATH=/venv/bin:$PATH + +RUN python -m pip install -U setuptools + +WORKDIR /app + +COPY pyproject.toml . + +FROM reqs AS app + +WORKDIR /app +USER $APP_USER + +COPY quiabo quiabo +COPY bin bin +COPY README.md README.md +COPY test test +RUN pip install --no-cache-dir -e . + +EXPOSE 8000 + +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0", "quiabo:app"] diff --git a/README.md b/README.md index dbccc8a..2de99d5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,59 @@ -# quiabo -Backend web service for running OCR processes +# ![quiabo logo](quiabo/static/quiabo.png) quiabo + +`quiabo` is a backend web service for running OCR jobs implemented as a Flask and Celery application. + +## Dependencies + +Python dependencies are declared in `pyproject.toml`. + +## Development + +Spin up the application using Docker Compose. There are number of dependencies (Postgres and Redis) as well as Flask/Celery app components (`app`, `worker`, and optionially `flower`). Redis serves as the Celery broker (source of jobs) and Postgres is the Celery results backend. + +```bash +# Build the Docker image for app, worker, and flower +docker compose build + +# Create the postgres database; only needed the first time +docker compose run --rm app bin/dbinit + +# Start the Flask app, which will be running on http://localhost:8000/ +docker compose up --detach + +# Optionally start Flower, which is a dashboard for the Celery queue and +# will be running on http://localhost:5555/ +docker compose up --profile flower --detach +``` + +## Testing + +Once the stack is started, execute the tests by running `pytest` in one +of the running containers. Note that testing and linting dependencies are +not installed by default, so you'll need to do that too. + +```bash +# Install the testing and linting dependencies + +docker compose exec app pip install --no-cache-dir -e .[test,lint] + +# Run all the tests +docker compose exec app pytest + +# Run tests with a specific marker +# Example: only run the unit tests +docker compose exec app pytest -m unit +``` + +Test results/reports are written to `./artifacts/pytest`. + +## Configuration + +`quiabo`'s configuration is handled by environment variables using Flask's +[`from_prefixed_env()`](https://flask.palletsprojects.com/en/stable/config/#configuring-from-environment-variables) method, using `QUIABO` as the +prefix. Celery configuration is set using the same method. At a minimum, +you will need to set the following: + +| Environment variable | Purpose | Example | +| -------------------- | ------- | ------- | +| `QUIABO_CELERY__broker_url` | Connection URL for the Celery broker (e.g. Redis) | `redis://redis:6379` | +| `QUIABO_CELERY__result_backend` | SQLAlchemy connection URL for the Celery result backend (e.g. Postgres) | `db+postgresql://postgres:postgres@db:5432/quiabo` | diff --git a/artifacts/.keep b/artifacts/.keep new file mode 100644 index 0000000..e69de29 diff --git a/bin/dbinit b/bin/dbinit new file mode 100755 index 0000000..327ddb4 --- /dev/null +++ b/bin/dbinit @@ -0,0 +1,20 @@ +#!/bin/sh -e + +# Initialise the configured database environment for use with quiabo +# +# Copyright © 2026 The Regents of the University of California. MIT license. + +# Use the credentials from the app's environment. +export PGHOST=${POSTGRES_HOST:-db} +export PGPORT=${POSTGRES_PORT:-5432} +export PGUSER=${POSTGRES_USER} +export PGPASSWORD=${POSTGRES_PASSWORD} +export PGDATABASE=${POSTGRES_DB} + +# Determine if the database needs to be created or not. +if [ "$(psql -d template1 -t -A -c "SELECT COUNT(*) FROM pg_database WHERE datname='${POSTGRES_DB}';")" = '0' ]; then + echo Creating database ${POSTGRES_DB}... + createdb +else + echo Database ${POSTGRES_DB} already exists. Bye bye! +fi diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..47df5c4 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,22 @@ +services: + db: + volumes: !reset + + redis: + volumes: !reset + + app: + build: !reset + depends_on: !reset + ports: !reset + image: ${DOCKER_APP_IMAGE} + env_file: !override env.example + volumes: !reset + + worker: + build: !reset + image: ${DOCKER_APP_IMAGE} + depends_on: !reset + ports: !reset + env_file: !override env.example + volumes: !reset diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1d40c3f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,107 @@ +x-dbconfig: + environment: &dbconfig + POSTGRES_USER: ${POSTGRES_USER:-quiabo} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-quiabo} + POSTGRES_HOST: ${POSTGRES_HOST:-db} + POSTGRES_PORT: ${POSTGRES_PORT:-5432} + POSTGRES_DB: ${POSTGRES_DB:-quiabo} + +x-quiabo-common: + environment: &quiabo-common-environment + QUIABO_CELERY__broker_url: ${QUIABO__CELERY__broker_url:-redis://redis:6379} + QUIABO_CELERY__result_backend: ${QUIABO__CELERY__result_backend:-db+postgresql://${POSTGRES_USER:-quiabo}:${POSTGRES_PASSWORD:-quiabo}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-quiabo}} + QUIABO_CELERY__task_default_queue: ${QUIABO_CELERY__task_default_queue:-quiabo} + QUIABO_CELERY__task_ignore_result: ${QUIABO_CELERY__task_ignore_result:-false} + + +services: + db: + environment: + <<: *dbconfig + image: postgres:16 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER}"] + interval: 10s + retries: 5 + start_period: 5s + ports: + - 5432:5432 + restart: always + volumes: + - postgres-db-volume:/var/lib/postgresql/data + + app: + build: + context: . + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + environment: + <<: + - *quiabo-common-environment + - *dbconfig + init: true + restart: always + ports: + - 8000:8000 + volumes: + - ./quiabo:/app/quiabo:rw + + worker: + build: + context: . + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + environment: + <<: + - *quiabo-common-environment + - *dbconfig + init: true + restart: always + command: celery -A quiabo.celery_app worker --loglevel INFO + volumes: + - ./quiabo:/app/quiabo:rw + + flower: + build: + context: . + profiles: + - flower + depends_on: + redis: + condition: service_healthy + environment: + <<: *quiabo-common-environment + init: true + restart: always + command: celery -A quiabo.celery_app flower + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:5555/"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + ports: + - 127.0.0.1:5555:5555 + volumes: + - ./quiabo:/app/quiabo:rw + + redis: + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 30s + retries: 50 + start_period: 30s + image: redis:8 + ports: + - 6379:6379 + restart: always + +volumes: + postgres-db-volume: diff --git a/env.example b/env.example new file mode 100644 index 0000000..5e7f0ad --- /dev/null +++ b/env.example @@ -0,0 +1,3 @@ +POSTGRES_USER=root +POSTGRES_PASSWORD=root +POSTGRES_DB=quiabo diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dabba13 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,68 @@ +[build-system] +requires = ["setuptools >= 77.0.3"] +build-backend = "setuptools.build_meta" + +[project] +name = "quiabo" +description = "Backend web service for running OCR processes" +version = "0.0.1" +dependencies = [ + "celery", + "flask", + "flower", + "gunicorn", + "psycopg2", + "redis", + "sqlalchemy" +] +requires-python = ">= 3.13, < 3.15" +authors = [ + {name = "maría a. matienzo"}, + {name = "Jason Raitz"}, + {name = "Steve Sullivan"}, +] +readme = "README.md" +license = "MIT" + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-cov", +] +lint = [ + "mypy ~= 1.17.1", + "pydoclint ~= 0.6.10", + "pylint ~= 3.3", +] + +[project.urls] +Repository = "https://github.com/BerkeleyLibrary/quiabo" +Issues = "https://github.com/BerkeleyLibrary/quiabo" + +[tool.mypy] +python_version = "3.13" +warn_unused_configs = true +warn_redundant_casts = true +warn_return_any = true + +[tool.pydoclint] +allow-init-docstring = true +skip-checking-raises = true +style = "sphinx" + +[tool.pytest] +minversion = "9.0" +addopts = [ + "-v", + "-s", + "--junit-xml=artifacts/pytest.xml", + "--cov-report=term", + "--cov-report=html:artifacts/coverage", + "--cov=quiabo", +] +markers = [ + "unit: fast isolated unit tests with no external dependencies", +] + +[tool.setuptools] +py-modules = ["quiabo"] diff --git a/quiabo/__init__.py b/quiabo/__init__.py new file mode 100644 index 0000000..fb46a17 --- /dev/null +++ b/quiabo/__init__.py @@ -0,0 +1,23 @@ +"""Flask application intialization functions.""" + +from flask import Flask + +from quiabo import health, root +from quiabo.celery import celery_init_app + +def create_app() -> Flask: + """ + Creates the Flask application. + + :rtype: flask.Flask + """ + flask_app = Flask(__name__) + flask_app.config.from_prefixed_env(prefix="QUIABO") + + flask_app.register_blueprint(root.bp) + flask_app.register_blueprint(health.bp) + + return flask_app + +app = create_app() +celery_app = celery_init_app(app) diff --git a/quiabo/celery.py b/quiabo/celery.py new file mode 100644 index 0000000..6d016b3 --- /dev/null +++ b/quiabo/celery.py @@ -0,0 +1,36 @@ +"""Code to initialize the Celery application based on an existing Flask app.""" + +from celery import Celery, Task +from flask import Flask + +def celery_init_app(app: Flask) -> Celery: + """ + Given a properly configured Flask application, return a configured + Celery app. + """ + class FlaskTask(Task): + """ + Class used to provide access to Celery decoratorss. + + :see: https://flask.palletsprojects.com/en/stable/patterns/celery/ + """ + def __call__(self, *args: object, **kwargs: object) -> object: + """ + Create a callable instance since Celery otherwise does not + have direct access to the Flask application context. + + :param args: positional arguments to get passed in the call + :type args: object + :param kwargs: keyword arguments to get passed in the call + :type kwargs: object + :return: The output of the task to be run. + :rtype: Object + """ + with app.app_context(): + return self.run(*args, **kwargs) + + celery_app = Celery(app.name, task_cls=FlaskTask) + celery_app.config_from_object(app.config["CELERY"]) + celery_app.set_default() + app.extensions["celery"] = celery_app + return celery_app diff --git a/quiabo/health.py b/quiabo/health.py new file mode 100644 index 0000000..5b206f5 --- /dev/null +++ b/quiabo/health.py @@ -0,0 +1,25 @@ +"""Route/controller for a healthcheck endpoint. Requires significant expansion.""" + +from flask import Blueprint + +bp = Blueprint("health", __name__, url_prefix="/health") + +@bp.route("") +def default() -> dict[str, dict[str, str|bool]]: + """ + Default healthcheck endpoint. Because of Flask magic, this gets returned + to the client as a JSON object. + + Healthchecks are expected to contain a key for the healthcheck, the + message, and a bool for whether the check is successful. + + :return: The output of the healthcheck. Currently only confirms that + the application is running and can serve the route. + :rtype: dict[str, dict[str, str|bool]] + """ + return { + "default": { + "message": "Application is running", + "success": True + } + } diff --git a/quiabo/root.py b/quiabo/root.py new file mode 100644 index 0000000..d963d96 --- /dev/null +++ b/quiabo/root.py @@ -0,0 +1,12 @@ +""" +Route/controller for the application root. +""" + +from flask import Blueprint, render_template + +bp = Blueprint("root", __name__, url_prefix="") + +@bp.route("/") +def index() -> str: + """Default root endpoint.""" + return render_template('root/index.html') diff --git a/quiabo/static/quiabo.png b/quiabo/static/quiabo.png new file mode 100644 index 0000000..bb420bd Binary files /dev/null and b/quiabo/static/quiabo.png differ diff --git a/quiabo/templates/root/index.html b/quiabo/templates/root/index.html new file mode 100644 index 0000000..08ac543 --- /dev/null +++ b/quiabo/templates/root/index.html @@ -0,0 +1,8 @@ + + + quiabo + + quiabo logo +

Goodbye Doggy!

+ + diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..2614a47 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,41 @@ +# pylint: disable=W0621 + +"""pytest setup and fixtures for quiabo.""" + +import pytest +from flask import Flask +from quiabo import app as flask_app + +@pytest.fixture() +def app(): + """Create a configured instance of the Flask application for testing. + + :return: The Flask application. + :rtype: flask.Flask + """ + app = flask_app + app.config.update({ + "TESTING": True, + }) + + yield app + + # clean up / reset resources here + + +@pytest.fixture() +def client(app: Flask): + """ + Given a configured application, return a test HTTP client for checking + routes/controllers. + """ + return app.test_client() + + +@pytest.fixture() +def runner(app: Flask): + """ + Given a configured application, return a test runner for running CLI + commands. + """ + return app.test_cli_runner() diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit/test_celery.py b/test/unit/test_celery.py new file mode 100644 index 0000000..4dddcf7 --- /dev/null +++ b/test/unit/test_celery.py @@ -0,0 +1,10 @@ +"""Test Celery app initialization.""" + +from celery import Celery as CeleryApp +from quiabo import celery + +def test_celery_init_app(app): + """Ensure a test Celery application is instantiated.""" + with app.app_context(): + celery_app = celery.celery_init_app(app) + assert isinstance(celery_app, CeleryApp) diff --git a/test/unit/test_health.py b/test/unit/test_health.py new file mode 100644 index 0000000..b38beed --- /dev/null +++ b/test/unit/test_health.py @@ -0,0 +1,6 @@ +"""Test route/controller for healthchecks.""" + +def test_health_default_route(client): + """Test default healthcheck route.""" + response = client.get("/health") + assert response.json["default"]["success"] is True diff --git a/test/unit/test_root.py b/test/unit/test_root.py new file mode 100644 index 0000000..e94fca3 --- /dev/null +++ b/test/unit/test_root.py @@ -0,0 +1,6 @@ +"""Test application root route/controller.""" + +def test_root_route(client): + """Ensure the expected message gets returned from the application root.""" + response = client.get("/") + assert b"Goodbye Doggy!" in response.data