.github/workflows/build-libheif.sh

@@ -3,9 +3,10 @@ set -ex
 sudo apt-get install -y git cmake make pkg-config libx265-dev libde265-dev libjpeg-dev libtool
 git clone https://github.com/strukturag/libheif.git
 cd libheif
+git checkout v1.17.1
 mkdir build
 cd build
-cmake ..
+cmake --preset=release ..
 make
 sudo make install
 sudo ldconfig

CHANGELOG.md

@@ -9,6 +9,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 *Nothing yet.*
 
+## [1.3.3] - October 31, 2023
+
+### Fixed
+
+* Improved handling when encountering an error attempting to populate Redis during uploads.
+* Fixed `Range` requests failing by default by internally setting a default chunk size of 10mb.
+* Stop logging "no exif data".
+* Fixed admin API requests not working when authenticating as the shared secret user.
+
+### Changed
+
+* Updated dependencies. Manually compiled deployments may need to recompile `libheif` as well.
+
 ## [1.3.2] - September 13, 2023
 
 ### Fixed
@@ -453,7 +466,8 @@ a large database (more than about 100k uploaded files), run the following steps
 * Various other features that would be expected like maximum/minimum size controls, rate limiting, etc. Check out the
   sample config for a better idea of what else is possible.
 
-[unreleased]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.2...HEAD
+[unreleased]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.3...HEAD
+[1.3.3]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.2...v1.3.3
 [1.3.2]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.1...v1.3.2
 [1.3.1]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.0...v1.3.1
 [1.3.0]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.13...v1.3.0

Dockerfile

@@ -3,10 +3,24 @@
 FROM golang:1.20-alpine AS builder
 
 # Install build dependencies
-RUN apk add --no-cache git musl-dev dos2unix build-base libde265-dev libheif-dev
+RUN apk add --no-cache git musl-dev dos2unix build-base libde265-dev
 
 WORKDIR /opt
 COPY . /opt
+
+# Build libheif manually
+RUN apk add --no-cache build-base libtool cmake libjpeg-turbo-dev x265-dev ffmpeg-dev zlib-dev
+RUN git clone https://github.com/strukturag/libheif.git
+WORKDIR /opt/libheif
+RUN git checkout v1.17.1
+RUN mkdir build
+WORKDIR /opt/libheif/build
+RUN cmake --preset=release ..
+RUN make
+RUN make install
+WORKDIR /opt
+
+# Run remaining build steps
 RUN dos2unix ./build.sh ./docker/run.sh && chmod 744 ./build.sh
 RUN ./build.sh
 

api/_routers/97-require-repo-admin.go

@@ -17,7 +17,7 @@ func RequireRepoAdmin(generator GeneratorWithUserFn) GeneratorFn {
 				panic(errors.New("safety check failed: Repo admin access check received empty user ID"))
 			}
 
-			if !util.IsGlobalAdmin(user.UserId) {
+			if !user.IsShared && !util.IsGlobalAdmin(user.UserId) {
 				return _responses.AuthFailed()
 			}
 

assets/providers.json

@@ -346,6 +346,19 @@
             }
         ]
     },
+    {
+        "provider_name": "Balsamiq Cloud",
+        "provider_url": "https://balsamiq.cloud/",
+        "endpoints": [
+            {
+                "schemes": [
+                    "https://balsamiq.cloud/*"
+                ],
+                "url": "https://balsamiq.cloud/oembed",
+                "discovery": true
+            }
+        ]
+    },
     {
         "provider_name": "Beams.fm",
         "provider_url": "http://beams.fm",
@@ -993,6 +1006,23 @@
             }
         ]
     },
+    {
+        "provider_name": "EchoesHQ",
+        "provider_url": "https://echoeshq.com",
+        "endpoints": [
+            {
+                "schemes": [
+                    "http://app.echoeshq.com/embed/*"
+                ],
+                "url": "https://api.echoeshq.com/oembed",
+                "formats": [
+                    "json",
+                    "xml"
+                ],
+                "discovery": true
+            }
+        ]
+    },
     {
         "provider_name": "eduMedia",
         "provider_url": "https://www.edumedia-sciences.com/",
@@ -1347,22 +1377,6 @@
             }
         ]
     },
-    {
-        "provider_name": "Gfycat",
-        "provider_url": "https://gfycat.com/",
-        "endpoints": [
-            {
-                "schemes": [
-                    "http://gfycat.com/*",
-                    "http://www.gfycat.com/*",
-                    "https://gfycat.com/*",
-                    "https://www.gfycat.com/*"
-                ],
-                "url": "https://api.gfycat.com/v1/oembed",
-                "discovery": true
-            }
-        ]
-    },
     {
         "provider_name": "Gifnote",
         "provider_url": "https://www.gifnote.com/",

common/config/conf_domain.go

@@ -18,8 +18,9 @@ func NewDefaultDomainConfig() DomainRepoConfig {
 			AdminApiKind:    "matrix",
 		},
 		Downloads: DownloadsConfig{
-			MaxSizeBytes:        104857600, // 100mb
-			FailureCacheMinutes: 15,
+			MaxSizeBytes:               104857600, // 100mb
+			FailureCacheMinutes:        15,
+			DefaultRangeChunkSizeBytes: 10485760, // 10mb
 		},
 		UrlPreviews: UrlPreviewsConfig{
 			Enabled:          true,

common/config/conf_main.go

@@ -44,8 +44,9 @@ func NewDefaultMainConfig() MainRepoConfig {
 		Admins:      []string{},
 		Downloads: MainDownloadsConfig{
 			DownloadsConfig: DownloadsConfig{
-				MaxSizeBytes:        104857600, // 100mb
-				FailureCacheMinutes: 15,
+				MaxSizeBytes:               104857600, // 100mb
+				FailureCacheMinutes:        15,
+				DefaultRangeChunkSizeBytes: 10485760, // 10mb
 			},
 			NumWorkers: 10,
 			ExpireDays: 0,

dev/docker-compose.yaml

@@ -35,7 +35,7 @@ services:
       - media_proxy
   media_homeserver:
     container_name: "media_repo_synapse"
-    image: "matrixdotorg/synapse:v1.91.1"
+    image: "matrixdotorg/synapse:v1.95.0"
     restart: unless-stopped
     volumes:
       - ./synapse-db:/data
@@ -55,7 +55,7 @@ services:
       - media_proxy
   media_element:
     container_name: "media_repo_element"
-    image: "vectorim/element-web:v1.11.40"
+    image: "vectorim/element-web:v1.11.47"
     restart: unless-stopped
     volumes:
       - ./element-config.json:/app/config.json

go.mod

@@ -18,49 +18,49 @@ require (
 	github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c
 	github.com/faiface/beep v1.1.0
 	github.com/fogleman/gg v1.3.0
-	github.com/fsnotify/fsnotify v1.6.0
-	github.com/gabriel-vasile/mimetype v1.4.2
-	github.com/getsentry/sentry-go v0.23.0
+	github.com/fsnotify/fsnotify v1.7.0
+	github.com/gabriel-vasile/mimetype v1.4.3
+	github.com/getsentry/sentry-go v0.25.0
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
 	github.com/hashicorp/go-hclog v1.5.0
-	github.com/hashicorp/go-plugin v1.5.0
+	github.com/hashicorp/go-plugin v1.5.2
 	github.com/k3a/html2text v1.2.1
 	github.com/kettek/apng v0.0.0-20220823221153-ff692776a607
 	github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f
 	github.com/lib/pq v1.10.9
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_golang v1.17.0
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
 	github.com/rubyist/circuitbreaker v2.2.1+incompatible
 	github.com/ryanuber/go-glob v1.0.0
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d
 	github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a
 	github.com/sirupsen/logrus v1.9.3
-	golang.org/x/crypto v0.12.0 // indirect
-	golang.org/x/image v0.12.0
-	golang.org/x/net v0.14.0
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/image v0.13.0
+	golang.org/x/net v0.17.0
 )
 
 require (
-	github.com/docker/docker v24.0.5+incompatible
+	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-connections v0.4.0
-	github.com/go-redsync/redsync/v4 v4.8.2
+	github.com/go-redsync/redsync/v4 v4.10.0
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/minio/minio-go/v7 v7.0.63
-	github.com/panjf2000/ants/v2 v2.8.1
-	github.com/redis/go-redis/v9 v9.1.0
+	github.com/panjf2000/ants/v2 v2.8.2
+	github.com/redis/go-redis/v9 v9.2.1
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/stretchr/testify v1.8.4
-	github.com/strukturag/libheif v1.16.2
+	github.com/strukturag/libheif v1.17.1
 	github.com/t2bot/go-singleflight-streams v0.0.8
 	github.com/t2bot/go-typed-singleflight v0.0.3
 	github.com/t2bot/gotd-contrib v0.0.0-20230907202504-d21987ea2957
 	github.com/t2bot/pgo-fleet/embedded v1.0.1
-	github.com/testcontainers/testcontainers-go v0.23.0
-	github.com/testcontainers/testcontainers-go/modules/postgres v0.23.0
-	golang.org/x/sync v0.3.0
-	golang.org/x/term v0.12.0
+	github.com/testcontainers/testcontainers-go v0.26.0
+	github.com/testcontainers/testcontainers-go/modules/postgres v0.26.0
+	golang.org/x/sync v0.4.0
+	golang.org/x/term v0.13.0
 )
 
 require (
@@ -68,23 +68,27 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/Jeffail/gabs v1.4.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.11.2 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/containerd/containerd v1.7.5 // indirect
+	github.com/containerd/containerd v1.7.8 // indirect
+	github.com/containerd/log v0.1.0 // indirect
 	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349 // indirect
 	github.com/fatih/color v1.15.0 // indirect
-	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-errors/errors v1.5.1 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/uuid v1.3.1 // indirect
+	github.com/google/uuid v1.4.0 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
 	github.com/hajimehoshi/go-mp3 v0.3.4 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -94,14 +98,15 @@ require (
 	github.com/jfreymuth/vorbis v1.0.2 // indirect
 	github.com/jonboulle/clockwork v0.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.2 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/mewkiz/flac v1.0.8 // indirect
-	github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/mewkiz/flac v1.0.9 // indirect
+	github.com/mewkiz/pkg v0.0.0-20231012081350-95d6616c5403 // indirect
 	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
@@ -109,20 +114,26 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc4 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/opencontainers/runc v1.1.9 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rs/xid v1.5.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.9 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/smartystreets/assertions v1.0.0 // indirect
-	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
@@ -146,7 +157,7 @@ require (
 	github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect
 	github.com/tebeka/strftime v0.1.3 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/grpc v1.57.0 // indirect
+	google.golang.org/grpc v1.59.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1
 )

pipelines/_steps/upload/redis_async.go

@@ -3,6 +3,7 @@ package upload
 import (
 	"io"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/turt2live/matrix-media-repo/common/rcontext"
 	"github.com/turt2live/matrix-media-repo/redislib"
 )
@@ -18,6 +19,7 @@ func PopulateCacheAsync(ctx rcontext.RequestContext, reader io.Reader, size int6
 		err = redislib.StoreMedia(ctx, sha256hash, reader, size)
 		if err != nil {
 			ctx.Log.Debug("Not populating cache due to error: ", err)
+			sentry.CaptureException(err)
 			return
 		}
 	}()

redislib/cache.go

@@ -39,19 +39,31 @@ func StoreMedia(ctx rcontext.RequestContext, hash string, content io.Reader, siz
 		return err
 	}
 
+	cleanup := func() {
+		if delErr := DeleteMedia(ctx, hash); delErr != nil {
+			ctx.Log.Warn("Error while attempting to clean up cache during another error: ", delErr)
+			sentry.CaptureException(delErr)
+		}
+	}
+
 	buf := make([]byte, appendBufferSize)
 	for {
 		read, err := content.Read(buf)
-		eof := errors.Is(err, io.EOF)
+		eof := false
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				eof = true
+			} else {
+				cleanup()
+				return err
+			}
+		}
 		if read > 0 {
 			if err = ring.ForEachShard(ctx.Context, func(ctx2 context.Context, client *redis.Client) error {
 				res := client.Append(ctx2, hash, string(buf[0:read]))
 				return res.Err()
 			}); err != nil {
-				if delErr := DeleteMedia(ctx, hash); delErr != nil {
-					ctx.Log.Warn("Error while attempting to clean up cache during another error: ", delErr)
-					sentry.CaptureException(delErr)
-				}
+				cleanup()
 				return err
 			}
 		}

test/test_internals/deps_minio.go

@@ -101,7 +101,7 @@ func MakeMinio(depNet *NetworkDep) (*MinioDep, error) {
 	// Use an intermediary container to set up the minio instance
 	_, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
 		ContainerRequest: testcontainers.ContainerRequest{
-			Image:      "docker.io/alpine:3.18.3",
+			Image:      "docker.io/alpine:3.18.4",
 			Networks:   []string{depNet.NetId},
 			WaitingFor: wait.ForLog("This line marks WaitFor as done").WithStartupTimeout(120 * time.Second),
 			Mounts: []testcontainers.ContainerMount{

test/test_internals/deps_synapse.go

@@ -113,7 +113,7 @@ func MakeSynapse(domainName string, depNet *NetworkDep) (*SynapseDep, error) {
 	}
 	synContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
 		ContainerRequest: testcontainers.ContainerRequest{
-			Image:        "docker.io/matrixdotorg/synapse:v1.91.1",
+			Image:        "docker.io/matrixdotorg/synapse:v1.94.0",
 			ExposedPorts: []string{"8008/tcp"},
 			Mounts: []testcontainers.ContainerMount{
 				testcontainers.BindMount(f.Name(), "/data/homeserver.yaml"),

thumbnailing/u/exif.go

@@ -17,6 +17,9 @@ type ExifOrientation struct {
 func GetExifOrientation(img io.Reader) (*ExifOrientation, error) {
 	rawExif, err := exif.SearchAndExtractExifWithReader(img)
 	if err != nil {
+		if errors.Is(err, exif.ErrNoExif) {
+			return nil, nil
+		}
 		return nil, errors.New("exif: error reading possible exif data: " + err.Error())
 	}