feat(thumbnails): Tika-based audio cover art extraction

services/search/README.md

@@ -98,7 +98,7 @@ As soon as Tika is installed and configured, the search service needs to be told
 The following settings must be set:
 
 *   `SEARCH_EXTRACTOR_TYPE=tika`
-*   `SEARCH_EXTRACTOR_TIKA_TIKA_URL=http://YOUR-TIKA.URL`
+*   `SEARCH_EXTRACTOR_TIKA_TIKA_URL=http://YOUR-TIKA.URL` (or the shared `OC_TIKA_URL`, which the thumbnails service reads too)
 
 Additionally, the following optional settings can be set:
 

services/search/pkg/config/content.go

@@ -9,6 +9,6 @@ type Extractor struct {
 
 // ExtractorTika configures the Tika extractor
 type ExtractorTika struct {
-	TikaURL        string `yaml:"tika_url" env:"SEARCH_EXTRACTOR_TIKA_TIKA_URL" desc:"URL of the tika server." introductionVersion:"1.0.0"`
+	TikaURL        string `yaml:"tika_url" env:"OC_TIKA_URL;SEARCH_EXTRACTOR_TIKA_TIKA_URL" desc:"URL of the tika server." introductionVersion:"1.0.0"`
 	CleanStopWords bool   `yaml:"clean_stop_words" env:"SEARCH_EXTRACTOR_TIKA_CLEAN_STOP_WORDS" desc:"Defines if stop words should be cleaned or not. See the documentation for more details." introductionVersion:"1.0.0"`
 }

services/thumbnails/README.md

@@ -30,6 +30,11 @@ Thumbnails can be generated from the following source file types:
 -   tiff
 -   bmp
 -   txt
+-   camera raw files: nef, cr2, pef, arw, sr2, srf, dng
+
+For camera raw files the thumbnail is generated from the JPEG preview the camera embedded in the file, the raw sensor data itself is not developed. Raw files without an embedded preview cannot be thumbnailed. Preview extraction is done by an Apache Tika server, so raw thumbnails require `THUMBNAILS_PREPROCESSOR_TIKA_TIKA_URL` (or the shared `OC_TIKA_URL`, which the search service reads too) to be set.
+
+For audio files the thumbnail is the embedded cover art. When a Tika server is configured it is extracted via Tika (preferring the tagged front cover); otherwise it is read in-process. Setting `THUMBNAILS_PREPROCESSOR_AUDIO_PROCESSOR=builtin` forces the in-process extractor even when a Tika server is configured; that `builtin` processor is deprecated and will be removed in a future major release.
 
 The thumbnail service retrieves source files using the information provided by the backend. The Linux backend identifies source files usually based on the extension.
 

services/thumbnails/pkg/config/config.go

@@ -46,4 +46,18 @@ type Thumbnail struct {
 	MaxInputWidth         int               `yaml:"max_input_width" env:"THUMBNAILS_MAX_INPUT_WIDTH" desc:"The maximum width of an input image which is being processed." introductionVersion:"1.0.0"`
 	MaxInputHeight        int               `yaml:"max_input_height" env:"THUMBNAILS_MAX_INPUT_HEIGHT" desc:"The maximum height of an input image which is being processed." introductionVersion:"1.0.0"`
 	MaxInputImageFileSize string            `yaml:"max_input_image_file_size" env:"THUMBNAILS_MAX_INPUT_IMAGE_FILE_SIZE" desc:"The maximum file size of an input image which is being processed. Usable common abbreviations: [KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, PiB, EB, EiB], example: 2GB." introductionVersion:"1.0.0"`
+	Preprocessor          Preprocessor      `yaml:"preprocessor"`
+}
+
+// Preprocessor configures the preprocessors that turn a source file into a
+// thumbnailable image.
+type Preprocessor struct {
+	Tika Tika `yaml:"tika"`
+	// AudioProcessor selects the audio cover-art extractor.
+	AudioProcessor string `yaml:"audio_processor" env:"THUMBNAILS_PREPROCESSOR_AUDIO_PROCESSOR" desc:"The processor for extracting cover art from audio files. When a Tika server is configured it is used by default. Set this to 'builtin' to force the in-process extractor even when a Tika server is configured. The 'builtin' processor is deprecated and will be removed in a future major release." introductionVersion:"%%NEXT%%" deprecationVersion:"%%NEXT%%" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"THUMBNAILS_PREPROCESSOR_AUDIO_PROCESSOR only selects the deprecated in-process audio extractor; once it is removed audio cover art always uses Tika when configured." deprecationReplacement:""`
+}
+
+// Tika configures preview extraction via an Apache Tika server.
+type Tika struct {
+	TikaURL string `yaml:"tika_url" env:"OC_TIKA_URL;THUMBNAILS_PREPROCESSOR_TIKA_TIKA_URL" desc:"URL of a Tika server used to extract embedded previews from raw images. When empty, raw images are not thumbnailed." introductionVersion:"%%NEXT%%"`
 }

services/thumbnails/pkg/errors/error.go

@@ -17,6 +17,8 @@ var (
 	ErrNoConverterForExtractedImageFromGgsFile = errors.New("thumbnails: could not find converter for image extracted from ggs file")
 	// ErrNoConverterForExtractedImageFromAudioFile defines an error when the extracted image from an audio file could not be converted
 	ErrNoConverterForExtractedImageFromAudioFile = errors.New("thumbnails: could not find converter for image extracted from audio file")
+	// ErrNoEmbeddedImage is returned when a source file carries no extractable embedded image (a raw without a preview, an audio file without cover art)
+	ErrNoEmbeddedImage = errors.New("thumbnails: no embedded image found")
 	// ErrCS3AuthorizationMissing defines an error when the CS3 authorization is missing
 	ErrCS3AuthorizationMissing = errors.New("thumbnails: cs3source - authorization missing")
 )

services/thumbnails/pkg/preprocessor/preprocessor.go

@@ -4,6 +4,7 @@ import (
 	"archive/zip"
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"image"
@@ -26,14 +27,14 @@ import (
 
 // FileConverter is the interface for the file converter
 type FileConverter interface {
-	Convert(r io.Reader) (any, error)
+	Convert(ctx context.Context, r io.Reader) (any, error)
 }
 
 // GifDecoder is a converter for the gif file
 type GifDecoder struct{}
 
 // Convert reads the gif file and returns the thumbnail image
-func (i GifDecoder) Convert(r io.Reader) (any, error) {
+func (i GifDecoder) Convert(_ context.Context, r io.Reader) (any, error) {
 	img, err := gif.DecodeAll(r)
 	if err != nil {
 		return nil, errors.Wrap(err, `could not decode the image`)
@@ -45,7 +46,7 @@ func (i GifDecoder) Convert(r io.Reader) (any, error) {
 type GgsDecoder struct{ thumbnailpath string }
 
 // Convert reads the ggs file and returns the thumbnail image
-func (g GgsDecoder) Convert(r io.Reader) (any, error) {
+func (g GgsDecoder) Convert(ctx context.Context, r io.Reader) (any, error) {
 	var buf bytes.Buffer
 	_, err := io.Copy(&buf, r)
 	if err != nil {
@@ -65,7 +66,7 @@ func (g GgsDecoder) Convert(r io.Reader) (any, error) {
 			if converter == nil {
 				return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromGgsFile
 			}
-			img, err := converter.Convert(thumbnail)
+			img, err := converter.Convert(ctx, thumbnail)
 			if err != nil {
 				return nil, errors.Wrap(err, `could not decode the image`)
 			}
@@ -75,11 +76,12 @@ func (g GgsDecoder) Convert(r io.Reader) (any, error) {
 	return nil, errors.Errorf("%s not found", g.thumbnailpath)
 }
 
-// AudioDecoder is a converter for the audio file
+// AudioDecoder extracts an audio file's embedded cover art from its ID3 tag. It is
+// deprecated in favour of the Tika extractor and slated for removal in the next major.
 type AudioDecoder struct{}
 
 // Convert reads the audio file and extracts the thumbnail image from the id3 tag
-func (i AudioDecoder) Convert(r io.Reader) (any, error) {
+func (i AudioDecoder) Convert(ctx context.Context, r io.Reader) (any, error) {
 	b, err := io.ReadAll(r)
 	if err != nil {
 		return nil, err
@@ -99,7 +101,7 @@ func (i AudioDecoder) Convert(r io.Reader) (any, error) {
 		return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromAudioFile
 	}
 
-	return converter.Convert(bytes.NewReader(picture.Data))
+	return converter.Convert(ctx, bytes.NewReader(picture.Data))
 }
 
 // TxtToImageConverter is a converter for the text file
@@ -108,7 +110,7 @@ type TxtToImageConverter struct {
 }
 
 // Convert reads the text file and renders it into a thumbnail image
-func (t TxtToImageConverter) Convert(r io.Reader) (any, error) {
+func (t TxtToImageConverter) Convert(_ context.Context, r io.Reader) (any, error) {
 	img := image.NewRGBA(image.Rect(0, 0, 640, 480))
 
 	imgBounds := img.Bounds()
@@ -203,7 +205,7 @@ type GGPStruct struct {
 type GgpDecoder struct{}
 
 // Convert reads the ggp file and returns the first thumbnail image
-func (j GgpDecoder) Convert(r io.Reader) (any, error) {
+func (j GgpDecoder) Convert(_ context.Context, r io.Reader) (any, error) {
 	ggp := &GGPStruct{}
 	err := json.NewDecoder(r).Decode(ggp)
 	if err != nil {
@@ -338,12 +340,29 @@ func ForType(mimeType string, opts map[string]any) FileConverter {
 		return GgpDecoder{}
 	case "image/gif":
 		return GifDecoder{}
-	case "audio/flac":
-		fallthrough
-	case "audio/mpeg":
-		fallthrough
-	case "audio/ogg":
+	case "audio/flac", "audio/mpeg", "audio/ogg":
+		// Tika is used whenever configured; "builtin" forces the in-process extractor
+		url, _ := opts["tikaURL"].(string)
+		if p, _ := opts["audioProcessor"].(string); url != "" && p != "builtin" {
+			filename, _ := opts["filename"].(string)
+			return TikaDecoder{tikaURL: url, filename: filename, mimeType: mimeType}
+		}
 		return AudioDecoder{}
+	case "image/x-nikon-nef",
+		"image/x-nikon-nrw",
+		"image/x-canon-cr2",
+		"image/x-pentax-pef",
+		"image/x-sony-arw",
+		"image/x-sony-sr2",
+		"image/x-sony-srf",
+		"image/x-adobe-dng":
+		// raw previews come from Tika, routed by the filename; no URL -> default
+		url, _ := opts["tikaURL"].(string)
+		if url == "" {
+			return ImageDecoder{}
+		}
+		filename, _ := opts["filename"].(string)
+		return TikaDecoder{tikaURL: url, filename: filename}
 	default:
 		return ImageDecoder{}
 	}

services/thumbnails/pkg/preprocessor/preprocessor_imaging.go

@@ -3,6 +3,7 @@
 package preprocessor
 
 import (
+	"context"
 	"io"
 
 	"github.com/kovidgoyal/imaging"
@@ -13,7 +14,7 @@ import (
 type ImageDecoder struct{}
 
 // Convert reads the image file and returns the thumbnail image
-func (i ImageDecoder) Convert(r io.Reader) (any, error) {
+func (i ImageDecoder) Convert(_ context.Context, r io.Reader) (any, error) {
 	img, err := imaging.Decode(r, imaging.AutoOrientation(true))
 	if err != nil {
 		return nil, errors.Wrap(err, `could not decode the image`)

services/thumbnails/pkg/preprocessor/preprocessor_test.go

@@ -2,6 +2,7 @@ package preprocessor
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"os"
 	"testing"
@@ -19,7 +20,7 @@ func TestImageDecoder(t *testing.T) {
 	RunSpecs(t, "ImageDecoder Suite")
 }
 
-var _ = Describe("ImageDecoder", func() {
+var _ = Describe("preprocessor", func() {
 	Describe("ImageDecoder", func() {
 		var fileReader io.Reader
 		BeforeEach(func() {
@@ -32,14 +33,14 @@ var _ = Describe("ImageDecoder", func() {
 
 		It("should decode an image", func() {
 			decoder := ImageDecoder{}
-			img, err := decoder.Convert(fileReader)
+			img, err := decoder.Convert(context.TODO(), fileReader)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(img).ToNot(BeNil())
 		})
 
 		It("should return an error if the image is invalid", func() {
 			decoder := ImageDecoder{}
-			img, err := decoder.Convert(bytes.NewReader([]byte("not an image")))
+			img, err := decoder.Convert(context.TODO(), bytes.NewReader([]byte("not an image")))
 			Expect(err).To(HaveOccurred())
 			Expect(img).To(BeNil())
 		})
@@ -57,14 +58,14 @@ var _ = Describe("ImageDecoder", func() {
 
 		It("should decode a gif", func() {
 			decoder := GifDecoder{}
-			img, err := decoder.Convert(fileReader)
+			img, err := decoder.Convert(context.TODO(), fileReader)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(img).ToNot(BeNil())
 		})
 
 		It("should return an error if the gif is invalid", func() {
 			decoder := GifDecoder{}
-			img, err := decoder.Convert(bytes.NewReader([]byte("not a gif")))
+			img, err := decoder.Convert(context.TODO(), bytes.NewReader([]byte("not a gif")))
 			Expect(err).To(HaveOccurred())
 			Expect(img).To(BeNil())
 		})
@@ -82,14 +83,14 @@ var _ = Describe("ImageDecoder", func() {
 
 		It("should decode a ggs", func() {
 			decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
-			img, err := decoder.Convert(fileReader)
+			img, err := decoder.Convert(context.TODO(), fileReader)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(img).ToNot(BeNil())
 		})
 
 		It("should return an error if the ggs is invalid", func() {
 			decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
-			img, err := decoder.Convert(bytes.NewReader([]byte("not a ggs")))
+			img, err := decoder.Convert(context.TODO(), bytes.NewReader([]byte("not a ggs")))
 			Expect(err).To(HaveOccurred())
 			Expect(img).To(BeNil())
 		})
@@ -104,7 +105,7 @@ var _ = Describe("ImageDecoder", func() {
 			}
 			fileReader = bytes.NewReader(fileContent)
 			decoder := AudioDecoder{}
-			img, err := decoder.Convert(fileReader)
+			img, err := decoder.Convert(context.TODO(), fileReader)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(img).ToNot(BeNil())
 		})
@@ -115,13 +116,13 @@ var _ = Describe("ImageDecoder", func() {
 			}
 			fileReader = bytes.NewReader(fileContent)
 			decoder := AudioDecoder{}
-			img, err := decoder.Convert(fileReader)
+			img, err := decoder.Convert(context.TODO(), fileReader)
 			Expect(err).To(HaveOccurred())
 			Expect(img).To(BeNil())
 		})
 		It("should return an error if the audio is invalid", func() {
 			decoder := AudioDecoder{}
-			img, err := decoder.Convert(bytes.NewReader([]byte("not an audio")))
+			img, err := decoder.Convert(context.TODO(), bytes.NewReader([]byte("not an audio")))
 			Expect(err).To(HaveOccurred())
 			Expect(img).To(BeNil())
 		})
@@ -145,7 +146,7 @@ var _ = Describe("ImageDecoder", func() {
 			}
 		})
 		It("should decode a text", func() {
-			img, err := decoder.Convert(bytes.NewReader([]byte("This is a test text")))
+			img, err := decoder.Convert(context.TODO(), bytes.NewReader([]byte("This is a test text")))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(img).ToNot(BeNil())
 		})
@@ -156,7 +157,7 @@ var _ = Describe("ImageDecoder", func() {
 					DefaultFont: "/some/unknown/font.otf",
 				},
 			}
-			img, err := decoder.Convert(bytes.NewReader([]byte("This is a test text")))
+			img, err := decoder.Convert(context.TODO(), bytes.NewReader([]byte("This is a test text")))
 			Expect(err).To(HaveOccurred())
 			Expect(img).To(BeNil())
 		})
@@ -173,17 +174,46 @@ var _ = Describe("ImageDecoder", func() {
 			Expect(decoder).To(BeAssignableToTypeOf(GifDecoder{}))
 		})
 
+		rawTypes := []string{
+			"image/x-nikon-nef", "image/x-nikon-nrw", "image/x-canon-cr2",
+			"image/x-pentax-pef", "image/x-sony-arw", "image/x-sony-sr2",
+			"image/x-sony-srf", "image/x-adobe-dng",
+		}
+
+		It("should return a TikaDecoder for every registered raw type when a Tika URL is set", func() {
+			opts := map[string]any{"tikaURL": "http://tika:9998"}
+			for _, mt := range rawTypes {
+				Expect(ForType(mt, opts)).To(BeAssignableToTypeOf(TikaDecoder{}), mt)
+			}
+		})
+
+		It("should fall back to the default decoder for raw types without a Tika URL", func() {
+			for _, mt := range rawTypes {
+				Expect(ForType(mt, nil)).To(BeAssignableToTypeOf(ImageDecoder{}), mt)
+			}
+		})
+
 		It("should return an GgsDecoder for ggs types", func() {
 			decoder := ForType("application/vnd.geogebra.ggs", nil)
 			// This will not return the expected ggsDecoder, but an ImageDecoder since ggs contains an embedded png.
 			Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
 		})
 
-		It("should return an AudioDecoder for audio types", func() {
+		It("should return an AudioDecoder for audio types without a Tika URL", func() {
 			decoder := ForType("audio/mpeg", nil)
 			Expect(decoder).To(BeAssignableToTypeOf(AudioDecoder{}))
 		})
 
+		It("should return a TikaDecoder for audio types when a Tika URL is set", func() {
+			opts := map[string]any{"tikaURL": "http://tika:9998"}
+			Expect(ForType("audio/mpeg", opts)).To(BeAssignableToTypeOf(TikaDecoder{}))
+		})
+
+		It("should force the AudioDecoder when the builtin processor is selected", func() {
+			opts := map[string]any{"tikaURL": "http://tika:9998", "audioProcessor": "builtin"}
+			Expect(ForType("audio/mpeg", opts)).To(BeAssignableToTypeOf(AudioDecoder{}))
+		})
+
 		It("should return an TxtToImageConverter for text types", func() {
 			decoder := ForType("text/plain", nil)
 			Expect(decoder).To(BeAssignableToTypeOf(TxtToImageConverter{}))

services/thumbnails/pkg/preprocessor/preprocessor_vips.go

@@ -3,6 +3,7 @@
 package preprocessor
 
 import (
+	"context"
 	"io"
 
 	"github.com/davidbyttow/govips/v2/vips"
@@ -14,7 +15,7 @@ func init() {
 
 type ImageDecoder struct{}
 
-func (v ImageDecoder) Convert(r io.Reader) (interface{}, error) {
+func (v ImageDecoder) Convert(_ context.Context, r io.Reader) (interface{}, error) {
 	img, err := vips.NewImageFromReader(r)
 	return img, err
 }