CHANGELOG.md

@@ -9,7 +9,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 *Nothing yet.*
 
-## [v1.3.0] - September 8, 2023
+## [1.3.1] - September 8, 2023
+
+### Fixed
+
+* Fixed media purge API not being able to delete thumbnails.
+* Fixed thumbnails being attempted for disabled media types.
+* Fixed SVG and other non-dimensional media failing to be usefully thumbnailed in some cases.
+
+## [1.3.0] - September 8, 2023
 
 ### Mandatory Configuration Change
 
@@ -439,7 +447,8 @@ a large database (more than about 100k uploaded files), run the following steps
 * Various other features that would be expected like maximum/minimum size controls, rate limiting, etc. Check out the
   sample config for a better idea of what else is possible.
 
-[unreleased]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.0...HEAD
+[unreleased]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.1...HEAD
+[1.3.1]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.0...v1.3.1
 [1.3.0]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.13...v1.3.0
 [1.2.13]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.12...v1.2.13
 [1.2.12]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.11...v1.2.12

build-dist.sh

@@ -17,8 +17,8 @@ do
   do
     pth="$os-$arch"
     mkdir $PWD/bin/$pth
-    GOOS=$os GOARCH=$arch GOBIN=$PWD/bin go build -o $PWD/bin/$pth -a -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.GitCommit=$(git rev-list -1 HEAD) -X github.com/turt2live/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/...
-    GOOS=$os GOARCH=$arch GOBIN=$PWD/bin go build -pgo=pgo_media_repo.pprof -o $PWD/bin/$pth -a -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.GitCommit=$(git rev-list -1 HEAD) -X github.com/turt2live/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/media_repo
+    GOOS=$os GOARCH=$arch GOBIN=$PWD/bin go build -o $PWD/bin/$pth -a -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/...
+    GOOS=$os GOARCH=$arch GOBIN=$PWD/bin go build -pgo=pgo_media_repo.pprof -o $PWD/bin/$pth -a -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/media_repo
     cd $PWD/bin/$pth
     if [ "$arch" == "amd64" ]; then
       arch="x64"

build.sh

@@ -4,5 +4,5 @@ set -ex
 
 GOBIN=$PWD/bin go install -v ./cmd/compile_assets
 $PWD/bin/compile_assets
-GOBIN=$PWD/bin go install -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.GitCommit=$(git rev-list -1 HEAD) -X github.com/turt2live/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/...
-GOBIN=$PWD/bin go install -pgo=pgo_media_repo.pprof -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.GitCommit=$(git rev-list -1 HEAD) -X github.com/turt2live/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/media_repo
+GOBIN=$PWD/bin go install -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/...
+GOBIN=$PWD/bin go install -pgo=pgo_media_repo.pprof -ldflags "-X github.com/turt2live/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/media_repo

cmd/gdpr_import/main.go

 package main
 
 import (
-	"errors"
 	"flag"
 	"os"
 	"path"
@@ -10,6 +9,7 @@ import (
 	"github.com/turt2live/matrix-media-repo/archival/v2archive"
 	"github.com/turt2live/matrix-media-repo/common/assets"
 	"github.com/turt2live/matrix-media-repo/common/config"
+	"github.com/turt2live/matrix-media-repo/common/import_cmdline"
 	"github.com/turt2live/matrix-media-repo/common/logging"
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
 	"github.com/turt2live/matrix-media-repo/common/runtime"
@@ -37,7 +37,7 @@ func main() {
 	assets.SetupMigrations(*migrationsPath)
 
 	if ids.GetMachineId() == 0 {
-		panic(errors.New("expected custom machine ID for import process (unsafe to import as Machine 0)"))
+		import_cmdline.AskMachineId()
 	}
 
 	var err error

cmd/import_synapse/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/turt2live/matrix-media-repo/common/assets"
 	"github.com/turt2live/matrix-media-repo/common/config"
+	"github.com/turt2live/matrix-media-repo/common/import_cmdline"
 	"github.com/turt2live/matrix-media-repo/common/logging"
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
 	"github.com/turt2live/matrix-media-repo/common/runtime"
@@ -52,7 +53,7 @@ func main() {
 	assets.SetupMigrations(*migrationsPath)
 
 	if ids.GetMachineId() == 0 {
-		panic(errors.New("expected custom machine ID for import process (unsafe to import as Machine 0)"))
+		import_cmdline.AskMachineId()
 	}
 
 	var realPsqlPassword string

common/import_cmdline/ask_machine_id.go

+package import_cmdline
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/turt2live/matrix-media-repo/util/ids"
+	"golang.org/x/term"
+)
+
+func AskMachineId() {
+	fmt.Println("The importer runs as a MMR worker and needs to have a dedicated MACHINE_ID. See https://docs.t2bot.io/matrix-media-repo/deployment/horizontal_scaling.html for details on what a MACHINE_ID is.")
+	if !term.IsTerminal(int(os.Stdin.Fd())) {
+		fmt.Println("Please specify a MACHINE_ID environment variable.")
+		os.Exit(2)
+		return // for good measure
+	}
+	fmt.Println("If you don't use horizontal scaling, you can use '1' as the machine ID. Otherwise, please enter an unused machine ID in your environment.")
+	fmt.Printf("Machine ID: ")
+	var machineId int64
+	if _, err := fmt.Scanf("%d", &machineId); err != nil {
+		panic(err)
+	}
+	if err := ids.SetMachineId(machineId); err != nil {
+		panic(err)
+	}
+}

common/runtime/init.go

@@ -55,6 +55,7 @@ func LoadDatastores() {
 	for id, found := range dsMap {
 		if !found {
 			logrus.Errorf("No configured datastore for ID %s found - please check your configuration and restart.", id)
+			logrus.Error("See https://docs.t2bot.io/matrix-media-repo/upgrading/130.html for details")
 			fatal = true
 		}
 	}

common/version/version.go

@@ -2,6 +2,7 @@ package version
 
 import (
 	"fmt"
+	"runtime/debug"
 
 	"github.com/sirupsen/logrus"
 )
@@ -10,9 +11,20 @@ var GitCommit string
 var Version string
 
 func SetDefaults() {
+	build, infoOk := debug.ReadBuildInfo()
+
 	if GitCommit == "" {
 		GitCommit = ".dev"
+		if infoOk {
+			for _, setting := range build.Settings {
+				if setting.Key == "vcs.revision" {
+					GitCommit = setting.Value
+					break
+				}
+			}
+		}
 	}
+
 	if Version == "" {
 		Version = "unknown"
 	}

config.sample.yaml

@@ -410,6 +410,7 @@ thumbnails:
     - "image/apng"
     - "image/gif"
     - "image/heif"
+    - "image/heic"
     - "image/webp"
     - "image/bmp"
     - "image/tiff"

database/table_thumbnails.go

@@ -30,7 +30,7 @@ const insertThumbnail = "INSERT INTO thumbnails (origin, media_id, content_type,
 const selectThumbnailByLocationExists = "SELECT TRUE FROM thumbnails WHERE datastore_id = $1 AND location = $2 LIMIT 1;"
 const selectThumbnailsForMedia = "SELECT origin, media_id, content_type, width, height, method, animated, sha256_hash, size_bytes, creation_ts, datastore_id, location FROM thumbnails WHERE origin = $1 AND media_id = $2;"
 const selectOldThumbnails = "SELECT origin, media_id, content_type, width, height, method, animated, sha256_hash, size_bytes, creation_ts, datastore_id, location FROM thumbnails WHERE sha256_hash IN (SELECT t2.sha256_hash FROM thumbnails AS t2 WHERE t2.creation_ts < $1);"
-const deleteThumbnail = "DELETE FROM thumbnails WHERE origin = $1 AND media_id = $2 AND content_type = $3 AND width = $4 AND height = $5 AND method = $6 AND animated = $7 AND sha256_hash = $8 AND size_bytes = $9 AND creation_ts = $10 AND datastore_id = $11 AND location = $11;"
+const deleteThumbnail = "DELETE FROM thumbnails WHERE origin = $1 AND media_id = $2 AND content_type = $3 AND width = $4 AND height = $5 AND method = $6 AND animated = $7 AND sha256_hash = $8 AND size_bytes = $9 AND creation_ts = $10 AND datastore_id = $11 AND location = $12;"
 const updateThumbnailLocation = "UPDATE thumbnails SET datastore_id = $3, location = $4 WHERE datastore_id = $1 AND location = $2;"
 const selectThumbnailsByLocation = "SELECT origin, media_id, content_type, width, height, method, animated, sha256_hash, size_bytes, creation_ts, datastore_id, location FROM thumbnails WHERE datastore_id = $1 AND location = $2;"
 

tasks/exec.go

@@ -26,7 +26,7 @@ func executeEnable() {
 		if notiferCh != nil {
 			go func() {
 				for val := range notiferCh {
-					tryBeginTask(int(val), true)
+					go tryBeginTask(int(val), true)
 				}
 				notiferCh = nil
 			}()

tasks/schedule.go

@@ -124,7 +124,7 @@ func scheduleUnfinished() {
 		return
 	}
 	for _, task := range tasks {
-		beginTask(task)
+		go beginTask(task)
 	}
 }
 

thumbnailing/thumbnail.go

@@ -26,6 +26,10 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h
 		ctx.Log.Debugf("Unsupported content type '%s'", contentType)
 		return nil, ErrUnsupported
 	}
+	if !util.ArrayContains(ctx.Config.Thumbnails.Types, contentType) {
+		ctx.Log.Debugf("Disabled content type '%s'", contentType)
+		return nil, ErrUnsupported
+	}
 
 	generator, reconstructed := i.GetGenerator(imgStream, contentType, animated)
 	if generator == nil {
@@ -49,7 +53,7 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h
 	// TODO: Why does AdjustProperties even take `canAnimate` if it's always been hardcoded to `false`? (see git blame on this comment)
 	var shouldThumbnail bool
 	shouldThumbnail, width, height, _, method = u.AdjustProperties(w, h, width, height, animated, false, method)
-	if !shouldThumbnail {
+	if !shouldThumbnail && dimensional {
 		return nil, common.ErrMediaDimensionsTooSmall
 	}
 

thumbnailing/u/dimensions.go

@@ -14,7 +14,7 @@ func AdjustProperties(srcWidth int, srcHeight int, desiredWidth int, desiredHeig
 		} else if canAnimate {
 			return true, srcWidth, srcHeight, false, method
 		} else {
-			return false, 0, 0, false, method
+			return false, desiredWidth, desiredHeight, false, method
 		}
 	}
 	return true, desiredWidth, desiredHeight, wantAnimated, method

util/ids/snowflake.go

 package ids
 
 import (
+	"errors"
 	"os"
 	"strconv"
 
@@ -30,3 +31,17 @@ func makeSnowflake() (*snowflake.Node, error) {
 	sfnode = node
 	return sfnode, nil
 }
+
+func SetMachineId(id int64) error {
+	if err := os.Setenv("MACHINE_ID", strconv.FormatInt(id, 10)); err != nil {
+		return err
+	}
+	sfnode = nil
+	if GetMachineId() != id {
+		return errors.New("unexpected error setting machine ID")
+	}
+	if _, err := makeSnowflake(); err != nil {
+		return err
+	}
+	return nil
+}