Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions .make/release.mk
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,7 @@ release-dirs:
DOCKER_LDFLAGS += -X "$(OC_REPO)/pkg/config/defaults.BaseDataPathType=path" -X "$(OC_REPO)/pkg/config/defaults.BaseDataPathValue=/var/lib/opencloud"
DOCKER_LDFLAGS += -X "$(OC_REPO)/pkg/config/defaults.BaseConfigPathType=path" -X "$(OC_REPO)/pkg/config/defaults.BaseConfigPathValue=/etc/opencloud"

# We can't link statically when vips is enabled but we still
# prefer static linking where possible
ifndef ENABLE_VIPS
DOCKER_LDFLAGS += -extldflags "-static"
endif
DOCKER_LDFLAGS += -extldflags "-static"

release-linux-docker-amd64: release-dirs
GOOS=linux \
Expand Down
3 changes: 0 additions & 3 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
"type": "go",
"request": "launch",
"mode": "debug",
"buildFlags": [
// "-tags", "enable_vips"
],
"program": "${workspaceFolder}/opencloud/cmd/opencloud",
"args": ["server"],
"env": {
Expand Down
16 changes: 1 addition & 15 deletions .woodpecker.star
Original file line number Diff line number Diff line change
Expand Up @@ -806,20 +806,6 @@ def testOpencloud(ctx):
"mv coverage.out unit.coverage.out",
],
},
{
# the only run that covers the resize code the official docker
# images ship
# FIXME: run it with the libvips those images ship, see issue 3746
"name": "test-thumbnail-parity-vips",
"image": OC_CI_GOLANG,
"environment": {
"THUMBNAILS_PARITY_IMAGOR_URL": "http://imagor:8000",
"THUMBNAILS_PARITY_IMAGOR_MAGICK_URL": "http://imagor-magick:8000",
},
"commands": [
"go test -tags enable_vips ./services/thumbnails/pkg/parity/...",
],
},
{
"name": "codacy-coverage",
"image": CODACY_COVERAGE_REPORTER,
Expand Down Expand Up @@ -2650,7 +2636,7 @@ def build():
"name": "build",
"image": OC_CI_GOLANG,
"commands": [
"for i in $(seq 3); do make -C opencloud build ENABLE_VIPS=1 && break || sleep 1; done",
"for i in $(seq 3); do make -C opencloud build && break || sleep 1; done",
],
"environment": CI_HTTP_PROXY_ENV,
},
Expand Down
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ FROM quay.io/opencloudeu/golang-ci:1.26 AS build
COPY --from=generate /opencloud /opencloud

WORKDIR /opencloud/opencloud
RUN make go-generate build ENABLE_VIPS=true
RUN make go-generate build

FROM alpine:3.24

RUN apk add --no-cache attr ca-certificates curl mailcap tree vips && \
RUN apk add --no-cache attr ca-certificates curl mailcap tree && \
echo 'hosts: files dns' >| /etc/nsswitch.conf

LABEL maintainer="OpenCloud GmbH <devops@opencloud.eu>" \
Expand Down
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ require (
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/coreos/go-oidc/v3 v3.21.0
github.com/cs3org/go-cs3apis v0.0.0-20260424072047-8d9ef7076ae9
github.com/davidbyttow/govips/v2 v2.20.0
github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
github.com/gabriel-vasile/mimetype v1.4.15
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davidbyttow/govips/v2 v2.20.0 h1:sftLOcLeI+sMPVrBiw/ulWHuMm0kKyTQ2hfnoIYSxwc=
github.com/davidbyttow/govips/v2 v2.20.0/go.mod h1:QK5liLrx7YaHC7xPcYnm4KaQIYvu9xSGrBt9goXrfgo=
github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4=
github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
Expand Down
3 changes: 0 additions & 3 deletions opencloud/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@ NAME := opencloud
TAGS := disable_crypt
GOARCH := $(shell go env GOARCH)
CONFIG_DOCS_BASE_PATH := ../docs
ifdef ENABLE_VIPS
TAGS := ${TAGS},enable_vips
endif

ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI
include ../.bingo/Variables.mk
Expand Down
2 changes: 1 addition & 1 deletion opencloud/docker/Dockerfile.linux.debug.amd64
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ FROM amd64/alpine:edge
ARG VERSION=""
ARG REVISION=""

RUN apk add --no-cache attr bash ca-certificates curl delve inotify-tools libc6-compat mailcap tree vips patch && \
RUN apk add --no-cache attr bash ca-certificates curl delve inotify-tools libc6-compat mailcap tree patch && \
echo 'hosts: files dns' >| /etc/nsswitch.conf

LABEL maintainer="OpenCloud GmbH <devops@opencloud.eu>" \
Expand Down
2 changes: 1 addition & 1 deletion opencloud/docker/Dockerfile.linux.debug.arm64
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ FROM arm64v8/alpine:edge
ARG VERSION=""
ARG REVISION=""

RUN apk add --no-cache attr bash ca-certificates curl delve inotify-tools libc6-compat mailcap tree vips patch && \
RUN apk add --no-cache attr bash ca-certificates curl delve inotify-tools libc6-compat mailcap tree patch && \
echo 'hosts: files dns' >| /etc/nsswitch.conf

LABEL maintainer="OpenCloud GmbH <devops@opencloud.eu>" \
Expand Down
5 changes: 2 additions & 3 deletions opencloud/docker/Dockerfile.multiarch
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ RUN --mount=type=bind,target=/build,rw \
--mount=type=cache,target=/root/.cache \
GOOS="${TARGETOS:-linux}" GOARCH="${TARGETARCH:-amd64}" ; \
make -C ${SRCDIR:-.}/opencloud release-linux-docker-${TARGETARCH} \
ENABLE_VIPS=true DIST=/dist \
DIST=/dist \
VERSION="${VERSION}" EDITION="${EDITION}" \
${STRING:+STRING="${STRING}"}

Expand All @@ -23,8 +23,7 @@ ARG TARGETOS
ARG TARGETARCH

RUN apk add --no-cache attr bash ca-certificates curl imagemagick \
inotify-tools libc6-compat mailcap tree vips \
vips-magick patch && \
inotify-tools libc6-compat mailcap tree patch && \
echo 'hosts: files dns' >| /etc/nsswitch.conf

LABEL maintainer="openCloud GmbH <devops@opencloud.eu>" \
Expand Down
64 changes: 16 additions & 48 deletions services/thumbnails/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ source file to this endpoint and receives the processed image back.
| `POST /unsafe/fit-in/{width}x{height}/filters:{filters}` | Scale to fit within width x height, preserving aspect ratio and never upscaling |
| `POST /unsafe/stretch/{width}x{height}/filters:{filters}` | Resize to the exact width x height without preserving aspect ratio (distorts) |

A `smart` segment behind the box, `POST /unsafe/{width}x{height}/smart/filters:{filters}`, asks for the crop that keeps the region carrying the image rather than the middle, which is what webdav sends for every cropping resize. With libvips that is an attention crop; the pure Go build has nothing of the kind and crops the centre, as the thumbnails service has always done without libvips.
A `smart` segment behind the box, `POST /unsafe/{width}x{height}/smart/filters:{filters}`, asks for the crop that keeps the region carrying the image rather than the middle, which is what webdav sends for every cropping resize. A real imagor answers it with an attention crop; this service has nothing of the kind and crops the centre.

The path is imagor's. The filters segment is optional and may be empty
(`filters:`), and an image key behind it is accepted and ignored, because the
Expand All @@ -34,11 +34,10 @@ resized, while this service treats it as the box. Inside the filters segment,
filter is ignored. `format` itself is optional: without it the input's own format is
preserved, the same way imagor does it. An input this service cannot write
(tiff, bmp) becomes a JPEG; imagor writes tiff back in its own format and only
falls back to JPEG for the ones it cannot write either. Which formats those are
depends on the build: with libvips a webp source stays a webp, the pure Go build
answers it as a JPEG because it has no webp encoder.
falls back to JPEG for the ones it cannot write either. A webp source is
answered as a JPEG, because this service has no webp encoder.

What a build can read is its own backend's answer rather than a list: the library
What this service can read is its decoders' answer rather than a list: the library
that will decode the image reads the header first, so a source it cannot read
answers `406` before anything is decoded, and the maximum dimensions are checked
on that same header.
Expand All @@ -61,13 +60,11 @@ decides it, are the webdav service's business and documented in

The request body is a `multipart/form-data` upload with a single file field
named `image`, typed `image/*`; an untyped part is refused the way imagor's
upload loader refuses it. Output formats are `jpg`, `png`, `gif` and, with
libvips, `webp`; the pure Go build answers `406` for `format(webp)` because it
has no webp encoder. An animated gif asked for a still format is flattened to
upload loader refuses it. Output formats are `jpg`, `png` and `gif`; `format(webp)`
is answered with `406` because this service has no webp encoder. An animated gif asked for a still format is flattened to
its first frame, and a still source asked for `format(gif)` is palettised.

Reading an ICO needs libvips with its magick loader; the pure Go build answers
`406` for one.
An ICO is answered with `406`; imagor reads one only with its magick loader.

An animation is the one source whose frames are all decoded here, so what its
blocks declare is read before any of them is: a gif with more frames than
Expand All @@ -82,7 +79,7 @@ none.
The answers are imagor's: `400` for a malformed request, which includes a
`format` nobody writes (anything but `jpg`, `png`, `gif` and `webp`); `406` for a
source it cannot read, for an upload part that is not typed `image/*`, and for a
format this build has no encoder for (`webp` without libvips); `422` for an image
format this service has no encoder for (`webp`); `422` for an image
above the configured maximum dimensions and for an animation above the frame
limit; `429` when the concurrency limit is reached; `413` for a body above
`THUMBNAILS_MAX_INPUT_IMAGE_FILE_SIZE`.
Expand All @@ -107,41 +104,12 @@ The source and the thumbnail have ceilings of their own, for different reasons.

A real imagor has one pair of knobs for both, `-vips-max-width` and `-vips-max-height`, which bound every image in its pipeline, the loaded source as well as the result. Set them to the source ceiling. They are also more forgiving than this service for formats libvips can shrink while loading: a 9000 pixel wide JPEG passes a 2048 limit there, where this service refuses it on the header.

## Using libvips for Image Processing
## Supported Formats

To improve performance and to support a wider range of image formats, the
thumbnails service is able to utilize the
[libvips library](https://www.libvips.org/) for image processing. Support for
libvips needs to be enabled at buildtime and has a couple of implications:

- With libvips support enabled, it is not possible to create a statically
linked OpenCloud binary.
- Therefore, the libvips shared libraries need to be available at runtime in
the same release that was used to build the OpenCloud binary.
- When using the OpenCloud docker images, the libvips shared libraries are
included in the image and are correctly embedded.

Support of libvips is disabled by default. To enable it, make sure libvips
and its buildtime dependencies are installed in your build environment. For
macOS users, add the build time dependencies via:

```shell
brew install vips pkg-config
export PKG_CONFIG_PATH="/usr/local/opt/libffi/lib/pkgconfig"
```

Then you just need to set the `ENABLE_VIPS` variable on the `make` command:

```shell
make -C opencloud build ENABLE_VIPS=1
```

Or include the `enable_vips` build tag in the `go build` command:

```shell
go build -tags enable_vips -o opencloud -o bin/opencloud ./cmd/opencloud
```

When building a docker image using the Dockerfile in the top-level directory of
OpenCloud, libvips support is enabled and the libvips shared libraries are
included in the resulting docker image.
This service deliberately supports a small set of formats, decoded and encoded in
pure Go: it reads `jpeg`, `png`, `gif`, `bmp`, `tiff` and `webp` and writes `jpg`,
`png` and `gif`. It is meant as a fallback generator that keeps working when
libvips or imagor need to be taken out of a deployment, for example while a
security issue in them is being fixed. A deployment that wants more formats,
webp output or an attention crop runs a real [imagor](https://imagor.net) and
points `WEBDAV_THUMBNAILS_GENERATOR_URL` at it.
2 changes: 0 additions & 2 deletions services/thumbnails/pkg/parity/build_imaging_test.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
//go:build !enable_vips

package parity

const builtinBuild = "imaging"
7 changes: 0 additions & 7 deletions services/thumbnails/pkg/parity/build_vips_test.go

This file was deleted.

7 changes: 3 additions & 4 deletions services/thumbnails/pkg/service/http/v0/push.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,10 +141,9 @@ func (s Thumbnails) pushHandler(w http.ResponseWriter, r *http.Request) {
MaxOutWidth: s.maxOutWidth, MaxOutHeight: s.maxOutHeight,
}

// An animated gif is the one source both builds treat the same way: neither
// libvips nor the pure Go path has anything better for a paletted animation,
// so it never reaches a backend. Asking for a still format flattens it, and
// that is a backend's job again.
// An animated gif never reaches the still image backend, it is resized
// frame by frame. Asking for a still format flattens it, and that is the
// backend's job again.
if bytes.HasPrefix(imgData, gifMagic) && (ext == "gif" || !hasFormat) {
s.serveAnimation(w, imgData, req)

Expand Down
26 changes: 11 additions & 15 deletions services/thumbnails/pkg/service/http/v0/push_imaging.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
//go:build !enable_vips

package svc

import (
Expand All @@ -15,9 +13,7 @@ import (
"github.com/kovidgoyal/imaging/prism/meta/autometa"
)

// The backend this build resizes with. The implementations keep their own names
// so a symbol search finds both builds' versions side by side, the variables are
// what the shared handler calls.
// The backend the service resizes with, as variables the shared handler calls.
var (
loadSource = loadSourceImaging
extForSourceFormat = extForSourceFormatImaging
Expand All @@ -28,7 +24,7 @@ var (
encodeWebP = encodeWebPImaging
)

// imagingSource is a request's body as this build holds it between the header
// imagingSource is a request's body as the service holds it between the header
// read and the resize: the bytes plus what the header said about them. The
// pixels are decoded in resizeSourceImaging, so the dimension guard has its say
// before anything allocates a frame buffer.
Expand All @@ -38,8 +34,8 @@ type imagingSource struct {
format string
}

// loadSourceImaging reads the header through the stdlib decoder registry, which this
// build also decodes with: png, jpeg and gif from the stdlib, bmp, tiff and webp
// loadSourceImaging reads the header through the stdlib decoder registry, which the
// service also decodes with: png, jpeg and gif from the stdlib, bmp, tiff and webp
// from the imaging library.
func loadSourceImaging(data []byte) (*imagingSource, error) {
config, format, err := image.DecodeConfig(bytes.NewReader(data))
Expand All @@ -51,14 +47,14 @@ func loadSourceImaging(data []byte) (*imagingSource, error) {
}

// Width and Height are what the header declared, so the dimension guard can ask
// before the pixels exist. Close is there because the libvips build's image has
// one; this build holds nothing that needs releasing.
// before the pixels exist. Close is a no-op, the source holds nothing that
// needs releasing.
func (s *imagingSource) Width() int { return s.config.Width }
func (s *imagingSource) Height() int { return s.config.Height }
func (s *imagingSource) Close() {}

// extForSourceFormatImaging is the output format for a request that asked for none: the
// input's own where this build can write it, a JPEG for the rest, which is how
// input's own where the service can write it, a JPEG for the rest, which is how
// imagor answers a format it cannot write either.
func extForSourceFormatImaging(src *imagingSource) string {
switch src.format {
Expand Down Expand Up @@ -160,8 +156,8 @@ func resizeSourceImaging(src *imagingSource, req resizeRequest) (image.Image, er
return img, nil
}

// smart() asks for the most interesting window, which needs libvips;
// this build crops the centre for it, the way main always did
// smart() asks for the most interesting window, which real imagor
// finds with libvips; this service crops the centre for it.

return imaging.Thumbnail(img, req.Width, req.Height, imaging.Lanczos), nil
}
Expand All @@ -176,6 +172,6 @@ func encodeGIFImaging(w io.Writer, img image.Image) error {
return gif.Encode(w, palettedAtOrigin(img, palette.Plan9), nil)
}

// encodeWebPImaging refuses: there is no webp encoder in the pure Go build, so a
// deployment that needs webp output runs the libvips build or a real imagor.
// encodeWebPImaging refuses: there is no webp encoder in pure Go, so a
// deployment that needs webp output runs a real imagor.
func encodeWebPImaging(io.Writer, image.Image) error { return errFormatUnsupported }
2 changes: 1 addition & 1 deletion services/thumbnails/pkg/service/http/v0/push_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ var _ = Describe("outputFormatFromFilters", func() {
Entry("jpeg maps onto our jpg", "format(jpeg)", "jpg", true, false),
Entry("png", "format(png)", "png", true, false),
Entry("gif", "format(gif)", "gif", true, false),
Entry("webp, which only the libvips build writes", "format(webp)", "webp", true, false),
Entry("webp, which only imagor writes", "format(webp)", "webp", true, false),
Entry("behind another filter", "no_upscale():format(png)", "png", true, false),
Entry("no format filter at all", "no_upscale()", "", false, false),
Entry("an empty segment", "", "", false, false),
Expand Down
Loading
Loading