Add optional S3 prefixes (#582)

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 * New datastore option to ignore Redis cache when downloading media served by a `publicBaseUrl`. This can help ensure more requests get redirected to the CDN.
 * `HEAD /download` is now supported, as per [MSC4120](https://github.com/matrix-org/matrix-spec-proposals/pull/4120).
+* S3 datastores can now specify a `prefixLength` to improve S3 performance on some providers. See `config.sample.yaml` for details.
 
 ### Fixed
 

config.sample.yaml

@@ -212,6 +212,10 @@ datastores:
       # when `publicBaseUrl` is unset. Defaults to false (cached media will be served by MMR
       # before redirection if present).
       #redirectWhenCached: true
+      # The size of the prefix (path component) to use when storing media in S3. This can
+      # help improve download speeds in some S3 providers. Should not be set to higher than
+      # 16 to avoid future incompatibilities with MMR. Defaults to zero (no prefix).
+      #prefixLength: 3
 
 # Options for controlling archives. Archives are exports of a particular user's content for
 # the purpose of GDPR or moving media to a different server.

datastores/s3.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/sirupsen/logrus"
 	"github.com/t2bot/matrix-media-repo/common/config"
 	"github.com/t2bot/matrix-media-repo/common/rcontext"
 )
@@ -21,6 +22,7 @@ type s3 struct {
 	bucket             string
 	publicBaseUrl      string
 	redirectWhenCached bool
+	prefixLength       int
 }
 
 func ResetS3Clients() {
@@ -41,6 +43,7 @@ func getS3(ds config.DatastoreConfig) (*s3, error) {
 	useSslStr, hasSsl := ds.Options["ssl"]
 	publicBaseUrl := ds.Options["publicBaseUrl"]
 	redirectWhenCachedStr, hasRedirectWhenCached := ds.Options["redirectWhenCached"]
+	prefixLengthStr, hasPrefixLength := ds.Options["prefixLength"]
 
 	if !hasStorageClass {
 		storageClass = "STANDARD"
@@ -56,6 +59,17 @@ func getS3(ds config.DatastoreConfig) (*s3, error) {
 		redirectWhenCached, _ = strconv.ParseBool(redirectWhenCachedStr)
 	}
 
+	prefixLength := 0
+	if hasPrefixLength && prefixLengthStr != "" {
+		prefixLength, _ = strconv.Atoi(prefixLengthStr)
+		if prefixLength < 0 {
+			prefixLength = 0
+		}
+		if prefixLength > 16 {
+			logrus.Warnf("Prefix length %d is greater than 16 for datastore %s - this may cause future incompatibilities", prefixLength, ds.Id)
+		}
+	}
+
 	var err error
 	var client *minio.Client
 	client, err = minio.New(endpoint, &minio.Options{
@@ -73,6 +87,7 @@ func getS3(ds config.DatastoreConfig) (*s3, error) {
 		bucket:             bucket,
 		publicBaseUrl:      publicBaseUrl,
 		redirectWhenCached: redirectWhenCached,
+		prefixLength:       prefixLength,
 	}
 	s3clients.Store(ds.Id, s3c)
 	return s3c, nil

datastores/upload.go

@@ -38,6 +38,10 @@ func Upload(ctx rcontext.RequestContext, ds config.DatastoreConfig, data io.Read
 			return "", err
 		}
 
+		if s3c.prefixLength > 0 {
+			objectName = objectName[:s3c.prefixLength] + "/" + objectName[s3c.prefixLength:]
+		}
+
 		metrics.S3Operations.With(prometheus.Labels{"operation": "PutObject"}).Inc()
 		var info minio.UploadInfo
 		info, err = s3c.client.PutObject(ctx.Context, s3c.bucket, objectName, tee, size, minio.PutObjectOptions{StorageClass: s3c.storageClass, ContentType: contentType})