.dockerignore

@@ -7,9 +7,18 @@
 /config
 /gdpr-data
 /ipfs
+/dev/conduit-db
+/dev/psql
+/vcpkg
+/libheif
+
+# Generated files
+assets.bin.go
 
 media-repo*.yaml
 homeserver.yaml
+s3-probably-safe-to-delete.txt
+/test.test
 
 # Binaries for programs and plugins
 *.exe

.github/workflows/build-libheif.sh

+#!/bin/bash
+set -ex
+sudo apt-get install -y git cmake make pkg-config libx265-dev libde265-dev libjpeg-dev libtool
+git clone https://github.com/strukturag/libheif.git
+cd libheif
+mkdir build
+cd build
+cmake ..
+make
+sudo make install
+sudo ldconfig

.github/workflows/download_pgo.yml

+name: "Update PGO performance profile"
+on:
+#  schedule:
+#    - cron: "0 0 * * 2" # Every Tuesday at 00:00
+  workflow_dispatch:
+jobs:
+  update_pgo:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      PGO_MERGE: ${{ secrets.PGO_MERGE }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: "Download new pgo_media_repo.pprof"
+        run: "curl -sv --fail -X POST -H \"Authorization: Bearer ${PGO_MERGE}\" https://pgo-mmr.t2host.io/v1/merge?and_combine=true > pgo_media_repo.pprof"
+      - uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: "Update pgo_media_repo.pprof"

.github/workflows/download_providers.yml

+name: "Update oEmbed providers"
+on:
+#  schedule:
+#    - cron: "0 0 * * 2" # Every Tuesday at 00:00
+  workflow_dispatch:
+jobs:
+  update_providers:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: "Download new providers.json"
+        run: "curl -s --fail https://oembed.com/providers.json > assets/providers.json"
+      - uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: "Update providers.json"

.github/workflows/main.yml

@@ -3,11 +3,46 @@ on:
   push:
 jobs:
   build:
-    name: 'Build Go 1.18'
+    name: 'Go Build (1.20)'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
         with:
-          go-version: '1.18'
-      - run: './build.sh'
+          go-version: '1.20'
+      - name: "Install libheif"
+        run: "chmod +x ./.github/workflows/build-libheif.sh && ./.github/workflows/build-libheif.sh"
+      - run: './build.sh' # verify the thing compiles
+  static:
+    name: 'Go Static (1.20)'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+      - name: "Install libheif"
+        run: "chmod +x ./.github/workflows/build-libheif.sh && ./.github/workflows/build-libheif.sh"
+      - name: "Prepare: compile assets"
+        run: "GOBIN=$PWD/bin go install -v ./cmd/compile_assets"
+      - name: "Run: compile assets"
+        run: "$PWD/bin/compile_assets"
+      - name: "Prepare: staticcheck"
+        run: 'go install honnef.co/go/tools/cmd/staticcheck@latest'
+      - run: 'go vet ./cmd/...'
+      - run: 'staticcheck ./cmd/...'
+  test:
+    name: 'Go Test (1.20)'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+      - name: "Prepare: compile assets"
+        run: "GOBIN=$PWD/bin go install -v ./cmd/compile_assets"
+      - name: "Run: compile assets"
+        run: "$PWD/bin/compile_assets"
+      - name: "Run: tests"
+        run: "go test -c -v ./test && ./test.test '-test.v'" # cheat and work around working directory issues
+        timeout-minutes: 30

.gitignore

@@ -9,6 +9,8 @@
 /ipfs
 /dev/conduit-db
 /dev/psql
+/vcpkg
+/libheif
 
 # Generated files
 assets.bin.go
@@ -16,6 +18,7 @@ assets.bin.go
 media-repo*.yaml
 homeserver.yaml
 s3-probably-safe-to-delete.txt
+/test.test
 
 # Binaries for programs and plugins
 *.exe

CHANGELOG.md

@@ -9,6 +9,62 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 *Nothing yet.*
 
+## [v1.3.0] - September 8, 2023
+
+### Mandatory Configuration Change
+
+**Please see [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/upgrading/130.html) for details.**
+
+### Security Fixes
+
+* Fix improper usage of `Content-Disposition: inline` and related `Content-Type` safety ([CVE-2023-41318](https://www.cve.org/CVERecord?id=CVE-2023-41318), [GHSA-5crw-6j7v-xc72](https://github.com/turt2live/matrix-media-repo/security/advisories/GHSA-5crw-6j7v-xc72)).
+
+### Deprecations
+
+* The `GET /_matrix/media/unstable/local_copy/:server/:mediaId` (and `unstable/io.t2bot.media` variant) endpoint is deprecated and scheduled for removal. If you are using this endpoint, please comment on [this issue](https://github.com/turt2live/matrix-media-repo/issues/422) to explain your use case.
+
+### Added
+
+* Added a `federation.ignoredHosts` config option to block media from individual homeservers.
+* Support for [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246) (async uploads) is added, with per-user quota limiting options.
+* Support for [MSC4034](https://github.com/matrix-org/matrix-spec-proposals/pull/4034) (self-serve usage information) is added, alongside a new "maximum file count" quota limit.
+* The `GET /_synapse/admin/v1/statistics/users/media` [endpoint](https://matrix-org.github.io/synapse/v1.88/admin_api/statistics.html#users-media-usage-statistics) from Synapse is now supported at the same path for local server admins.
+* Thumbnailing support for:
+  * BMP images.
+  * TIFF images.
+  * HEIC images.
+* New metrics:
+  * HTTP response times.
+  * Age of downloaded/accessed media.
+* Support for [PGO](https://go.dev/doc/pgo) builds has been enabled via [pgo-fleet](https://github.com/t2bot/pgo-fleet).
+
+### Removed
+
+* IPFS support has been removed due to maintenance burden.
+* Exports initiated through the admin API no longer support `?include_data=false`. Exports will always contain data.
+* Server-side blurhash calculation has been removed. Clients and bridges already calculate blurhashes locally where applicable. 
+
+### Changed
+
+* **Mandatory configuration change**: You must add datastore IDs to your datastore configuration, as matrix-media-repo will no longer manage datastores for you.
+* If compiling `matrix-media-repo`, note that new external dependencies are required. See [the docs](https://docs.t2bot.io/matrix-media-repo/installing/method/compilation.html).
+  * Docker images already contain these dependencies. 
+* Datastores no longer use the `enabled` flag set on them. Use `forKinds: []` instead to disable a datastore's usage.
+* Per-user upload quotas now do not allow users to exceed the maximum values, even by 1 byte. Previously, users could exceed the limits by a little bit.
+* Updated to Go 1.19, then Go 1.20 in the same release cycle.
+* New CGO dependencies are required. See [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/installing/method/compilation.html) for details.
+* Logs are now less noisy by default.
+* Connected homeservers must support at least Matrix 1.1 on the Client-Server API. Servers over federation are not affected.
+* The example Grafana dashboard has been updated.
+
+### Fixed
+
+* URL previews now follow redirects properly.
+* Overall memory usage is improved, particularly during media uploads and API-initiated imports.
+  * Note: If you use plugins then memory usage will still be somewhat high due to temporary caching of uploads.
+  * Note: This affects RSS primarily. VSZ and other memory metrics may be higher than expected due to how Go releases memory to the OS. This is fixed when there's memory pressure.
+* Fixed shutdown stall if the config was reloaded more than once while running.
+
 ## [1.2.13] - February 12, 2023
 
 ### Deprecations
@@ -69,7 +125,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Updated support for post-[MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069) homeservers.
 * Updated the built-in oEmbed `providers.json`
 
-# [1.2.10] - December 23rd, 2021
+## [1.2.10] - December 23rd, 2021
 
 ### Deprecation notices
 
@@ -383,7 +439,8 @@ a large database (more than about 100k uploaded files), run the following steps
 * Various other features that would be expected like maximum/minimum size controls, rate limiting, etc. Check out the
   sample config for a better idea of what else is possible.
 
-[unreleased]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.13...HEAD
+[unreleased]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.0...HEAD
+[1.3.0]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.13...v1.3.0
 [1.2.13]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.12...v1.2.13
 [1.2.12]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.11...v1.2.12
 [1.2.11]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.10...v1.2.11

Dockerfile

 # ---- Stage 0 ----
 # Builds media repo binaries
-FROM golang:1.18-alpine AS builder
+FROM golang:1.20-alpine AS builder
 
 # Install build dependencies
-RUN apk add --no-cache git musl-dev dos2unix build-base
+RUN apk add --no-cache git musl-dev dos2unix build-base libde265-dev libheif-dev
 
 WORKDIR /opt
 COPY . /opt
-RUN dos2unix ./build.sh ./docker/run.sh
+RUN dos2unix ./build.sh ./docker/run.sh && chmod 744 ./build.sh
 RUN ./build.sh
 
+# the label is applied last so we don't pollute the image list with a weird amount of labelled images
+LABEL io.t2bot.mmr.cleanup="true"
+
 # ---- Stage 1 ----
 # Final runtime stage.
 FROM alpine
 
 RUN mkdir /plugins
-COPY --from=builder /opt/bin/plugin_antispam_ocr /plugins/
-COPY --from=builder /opt/bin/media_repo /opt/bin/import_synapse /opt/bin/export_synapse_for_import /opt/bin/gdpr_export /opt/bin/gdpr_import /opt/bin/s3_consistency_check /usr/local/bin/
-
 RUN apk add --no-cache \
         su-exec \
         ca-certificates \
@@ -25,9 +25,12 @@ RUN apk add --no-cache \
         imagemagick \
         ffmpeg
 
+COPY --from=builder /opt/bin/plugin_antispam_ocr /plugins/
+COPY --from=builder /opt/bin/media_repo /opt/bin/import_synapse /opt/bin/export_synapse_for_import /opt/bin/gdpr_export /opt/bin/gdpr_import /opt/bin/s3_consistency_check /usr/local/bin/
+
 COPY ./config.sample.yaml /etc/media-repo.yaml.sample
 COPY ./docker/run.sh /usr/local/bin/
-RUN dos2unix /usr/local/bin/run.sh
+RUN dos2unix /usr/local/bin/run.sh && chmod 744 /usr/local/bin/run.sh
 
 ENV REPO_CONFIG=/data/media-repo.yaml
 

README.md

 # matrix-media-repo
 
-[![Build status](https://badge.buildkite.com/4205079064098cf0abf5179ea4784f1c9113e875b8fcbde1a2.svg)](https://buildkite.com/t2bot/matrix-media-repo)
-
 matrix-media-repo is a highly customizable multi-domain media repository for Matrix. Intended for medium to large environments
 consisting of several homeservers, this media repo de-duplicates media (including remote media) while being fully compliant
 with the specification. 

api/_apimeta/auth.go

+package _apimeta
+
+import (
+	"net/http"
+
+	"github.com/getsentry/sentry-go"
+
+	"github.com/turt2live/matrix-media-repo/common/rcontext"
+	"github.com/turt2live/matrix-media-repo/matrix"
+	"github.com/turt2live/matrix-media-repo/util"
+)
+
+type UserInfo struct {
+	UserId      string
+	AccessToken string
+	IsShared    bool
+}
+
+func GetRequestUserAdminStatus(r *http.Request, rctx rcontext.RequestContext, user UserInfo) (bool, bool) {
+	isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
+	isLocalAdmin, err := matrix.IsUserAdmin(rctx, r.Host, user.AccessToken, r.RemoteAddr)
+	if err != nil {
+		sentry.CaptureException(err)
+		rctx.Log.Debug("Error verifying local admin: ", err)
+		return isGlobalAdmin, false
+	}
+
+	return isGlobalAdmin, isLocalAdmin
+}

api/auth_cache/auth_cache.go → api/_auth_cache/auth_cache.go

-package auth_cache
+package _auth_cache
 
 import (
 	"errors"
@@ -94,7 +94,7 @@ func GetUserId(ctx rcontext.RequestContext, accessToken string, appserviceUserId
 		if token.err != nil {
 			return "", token.err
 		}
-		ctx.Log.Infof("Access token belongs to %s", token.userId)
+		ctx.Log.Debugf("Access token belongs to %s", token.userId)
 		return token.userId, nil
 	}
 
@@ -109,7 +109,7 @@ func GetUserId(ctx rcontext.RequestContext, accessToken string, appserviceUserId
 		}
 
 		if r.SenderUserId != "" && (r.SenderUserId == appserviceUserId || appserviceUserId == "") {
-			ctx.Log.Infof("Access token belongs to appservice (sender user ID): %s", r.Id)
+			ctx.Log.Debugf("Access token belongs to appservice (sender user ID): %s", r.Id)
 			cacheToken(ctx, accessToken, appserviceUserId, r.SenderUserId, nil)
 			return r.SenderUserId, nil
 		}
@@ -121,7 +121,7 @@ func GetUserId(ctx rcontext.RequestContext, accessToken string, appserviceUserId
 				regexCache[n.Regex] = regex
 			}
 			if regex.MatchString(appserviceUserId) {
-				ctx.Log.Infof("Access token belongs to appservice: %s", r.Id)
+				ctx.Log.Debugf("Access token belongs to appservice: %s", r.Id)
 				cacheToken(ctx, accessToken, appserviceUserId, appserviceUserId, nil)
 				return appserviceUserId, nil
 			}
@@ -144,10 +144,10 @@ func cacheToken(ctx rcontext.RequestContext, accessToken string, appserviceUserI
 }
 
 func checkTokenWithHomeserver(ctx rcontext.RequestContext, accessToken string, appserviceUserId string, withCache bool) (string, error) {
-	ctx.Log.Info("Checking access token with homeserver")
+	ctx.Log.Debug("Checking access token with homeserver")
 	hsUserId, err := matrix.GetUserIdFromToken(ctx, ctx.Request.Host, accessToken, appserviceUserId, ctx.Request.RemoteAddr)
 	if withCache {
-		ctx.Log.Info("Caching access token response from homeserver")
+		ctx.Log.Debug("Caching access token response from homeserver")
 		cacheToken(ctx, accessToken, appserviceUserId, hsUserId, err)
 	}
 	return hsUserId, err

api/_debug/pprof.go

+package _debug
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/pprof"
+
+	"github.com/julienschmidt/httprouter"
+)
+
+func BindPprofEndpoints(httpMux *httprouter.Router, secret string) {
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/", pprofServe(pprof.Index, secret))
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/allocs", pprofServe(pprof.Index, secret))
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/block", pprofServe(pprof.Index, secret))
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/cmdline", pprofServe(pprof.Index, secret))
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/goroutine", pprofServe(pprof.Index, secret))
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/heap", pprofServe(pprof.Index, secret))
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/mutex", pprofServe(pprof.Index, secret))
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/profile", pprofServe(pprof.Profile, secret))
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/threadcreate", pprofServe(pprof.Index, secret))
+	httpMux.Handler("GET", "/_matrix/media/unstable/io.t2bot/debug/pprof/trace", pprofServe(pprof.Trace, secret))
+}
+
+type generatorFn = func(w http.ResponseWriter, r *http.Request)
+
+type requestContainer struct {
+	secret string
+	fn     generatorFn
+}
+
+func pprofServe(fn generatorFn, secret string) http.Handler {
+	return &requestContainer{
+		secret: secret,
+		fn:     fn,
+	}
+}
+
+func (c *requestContainer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	auth := r.Header.Get("Authorization")
+	if auth == "" {
+		auth = r.URL.Query().Get("access_token")
+	}
+	if auth != ("Bearer " + c.secret) {
+		// Order is important: Set headers before sending responses
+		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
+		w.WriteHeader(http.StatusUnauthorized)
+
+		encoder := json.NewEncoder(w)
+		_ = encoder.Encode(&map[string]bool{"success": false})
+		return
+	}
+
+	// otherwise authed fine
+	r.URL.Path = r.URL.Path[len("/_matrix/media/unstable/io.t2bot"):]
+	c.fn(w, r)
+}

api/_responses/content.go

+package _responses
+
+import "io"
+
+type EmptyResponse struct{}
+
+type HtmlResponse struct {
+	HTML string
+}
+
+type DownloadResponse struct {
+	ContentType       string
+	Filename          string
+	SizeBytes         int64
+	Data              io.ReadCloser
+	TargetDisposition string
+}
+
+type StreamDataResponse struct {
+	Stream io.Reader
+}
+
+func MakeQuarantinedImageResponse(stream io.ReadCloser) *DownloadResponse {
+	return &DownloadResponse{
+		ContentType:       "image/png",
+		Filename:          "not_allowed.png",
+		SizeBytes:         -1,
+		Data:              stream,
+		TargetDisposition: "inline",
+	}
+}

api/responses.go → api/_responses/errors.go

-package api
+package _responses
 
 import "github.com/turt2live/matrix-media-repo/common"
 
-type EmptyResponse struct{}
-
-type DoNotCacheResponse struct {
-	Payload interface{}
-}
-
-type HtmlResponse struct {
-	HTML string
-}
-
 type ErrorResponse struct {
 	Code         string `json:"errcode"`
 	Message      string `json:"error"`
@@ -22,6 +12,10 @@ func InternalServerError(message string) *ErrorResponse {
 	return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeUnknown}
 }
 
+func BadGatewayError(message string) *ErrorResponse {
+	return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeUnknown}
+}
+
 func MethodNotAllowed() *ErrorResponse {
 	return &ErrorResponse{common.ErrCodeUnknown, "Method Not Allowed", common.ErrCodeMethodNotAllowed}
 }
@@ -46,6 +40,10 @@ func AuthFailed() *ErrorResponse {
 	return &ErrorResponse{common.ErrCodeUnknownToken, "Authentication Failed", common.ErrCodeUnknownToken}
 }
 
+func MediaBlocked() *ErrorResponse {
+	return &ErrorResponse{common.ErrCodeNotFound, "Media blocked or not found", common.ErrCodeForbidden}
+}
+
 func GuestAuthFailed() *ErrorResponse {
 	return &ErrorResponse{common.ErrCodeNoGuests, "Guests cannot use this endpoint", common.ErrCodeNoGuests}
 }
@@ -57,3 +55,7 @@ func BadRequest(message string) *ErrorResponse {
 func QuotaExceeded() *ErrorResponse {
 	return &ErrorResponse{common.ErrCodeForbidden, "Quota Exceeded", common.ErrCodeQuotaExceeded}
 }
+
+func NotYetUploaded() *ErrorResponse {
+	return &ErrorResponse{common.ErrCodeNotYetUploaded, "Media not yet uploaded", common.ErrCodeNotYetUploaded}
+}

api/_responses/meta.go

+package _responses
+
+type DoNotCacheResponse struct {
+	Payload interface{}
+}

api/_routers/00-install-params.go

+package _routers
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"regexp"
+
+	"github.com/julienschmidt/httprouter"
+)
+
+func localCompile(expr string) *regexp.Regexp {
+	r, err := regexp.Compile(expr)
+	if err != nil {
+		panic(errors.New("error compiling expression: " + expr + " | " + err.Error()))
+	}
+	return r
+}
+
+var ServerNameRegex = localCompile("[a-zA-Z0-9.:\\-_]+")
+
+//var NumericIdRegex = localCompile("[0-9]+")
+
+func GetParam(name string, r *http.Request) string {
+	p := httprouter.ParamsFromContext(r.Context())
+	if p == nil {
+		return ""
+	}
+	return p.ByName(name)
+}
+
+func ForceSetParam(name string, val string, r *http.Request) *http.Request {
+	params := httprouter.ParamsFromContext(r.Context())
+	wasSet := false
+	for _, p := range params {
+		if p.Key == name {
+			p.Value = val
+			wasSet = true
+			break
+		}
+	}
+	if !wasSet {
+		params = append(params, httprouter.Param{
+			Key:   name,
+			Value: val,
+		})
+	}
+	return r.WithContext(context.WithValue(r.Context(), httprouter.ParamsKey, params))
+}

api/_routers/01-install_metadata.go

+package _routers
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+
+	"github.com/sirupsen/logrus"
+	"github.com/turt2live/matrix-media-repo/common"
+	"github.com/turt2live/matrix-media-repo/util"
+)
+
+type RequestCounter struct {
+	lastId uint64
+}
+
+func (c *RequestCounter) NextId() string {
+	strId := strconv.FormatUint(c.lastId, 10)
+	c.lastId = c.lastId + 1
+
+	return "REQ-" + strId
+}
+
+type InstallMetadataRouter struct {
+	next       http.Handler
+	ignoreHost bool
+	actionName string
+	counter    *RequestCounter
+}
+
+func NewInstallMetadataRouter(ignoreHost bool, actionName string, counter *RequestCounter, next http.Handler) *InstallMetadataRouter {
+	return &InstallMetadataRouter{
+		next:       next,
+		ignoreHost: ignoreHost,
+		actionName: actionName,
+		counter:    counter,
+	}
+}
+
+func (i *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	requestId := i.counter.NextId()
+	logger := logrus.WithFields(logrus.Fields{
+		"method":        r.Method,
+		"host":          r.Host,
+		"resource":      r.URL.Path,
+		"contentType":   r.Header.Get("Content-Type"),
+		"contentLength": r.ContentLength,
+		"queryString":   util.GetLogSafeQueryString(r),
+		"requestId":     requestId,
+		"remoteAddr":    r.RemoteAddr,
+		"userAgent":     r.UserAgent(),
+	})
+
+	ctx := r.Context()
+	ctx = context.WithValue(ctx, common.ContextRequestStartTime, util.NowMillis())
+	ctx = context.WithValue(ctx, common.ContextRequestId, requestId)
+	ctx = context.WithValue(ctx, common.ContextAction, i.actionName)
+	ctx = context.WithValue(ctx, common.ContextIgnoreHost, i.ignoreHost)
+	ctx = context.WithValue(ctx, common.ContextLogger, logger)
+	r = r.WithContext(ctx)
+
+	if i.next != nil {
+		i.next.ServeHTTP(w, r)
+	}
+}
+
+func GetActionName(r *http.Request) string {
+	x, ok := r.Context().Value(common.ContextAction).(string)
+	if !ok {
+		return "<UNKNOWN>"
+	}
+	return x
+}
+
+func ShouldIgnoreHost(r *http.Request) bool {
+	x, ok := r.Context().Value(common.ContextIgnoreHost).(bool)
+	if !ok {
+		return false
+	}
+	return x
+}
+
+func GetLogger(r *http.Request) *logrus.Entry {
+	x, ok := r.Context().Value(common.ContextLogger).(*logrus.Entry)
+	if !ok {
+		return nil
+	}
+	return x
+}
+
+func GetRequestDuration(r *http.Request) float64 {
+	x, ok := r.Context().Value(common.ContextRequestStartTime).(int64)
+	if !ok {
+		return -1
+	}
+	return float64(util.NowMillis()-x) / 1000.0
+}

api/_routers/02-install-headers.go

+package _routers
+
+import (
+	"net/http"
+)
+
+type InstallHeadersRouter struct {
+	next http.Handler
+}
+
+func NewInstallHeadersRouter(next http.Handler) *InstallHeadersRouter {
+	return &InstallHeadersRouter{next: next}
+}
+
+func (i *InstallHeadersRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	headers := w.Header()
+	if headers.Get("Allow") != "" {
+		headers.Set("Access-Control-Allow-Methods", headers.Get("Allow"))
+	}
+	headers.Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization")
+	headers.Set("Access-Control-Allow-Origin", "*")
+	headers.Set("Content-Security-Policy", "sandbox; default-src 'none'; script-src 'none'; plugin-types application/pdf; style-src 'unsafe-inline'; media-src 'self'; object-src 'self';")
+	headers.Set("Cross-Origin-Resource-Policy", "cross-origin")
+	headers.Set("X-Content-Security-Policy", "sandbox;")
+	headers.Set("X-Robots-Tag", "noindex, nofollow, noarchive, noimageindex")
+	headers.Set("Server", "matrix-media-repo")
+
+	if i.next != nil {
+		i.next.ServeHTTP(w, r)
+	}
+}

api/_routers/03-host_detection.go

+package _routers
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sebest/xff"
+	"github.com/sirupsen/logrus"
+	"github.com/turt2live/matrix-media-repo/api/_responses"
+	"github.com/turt2live/matrix-media-repo/common"
+	"github.com/turt2live/matrix-media-repo/common/config"
+	"github.com/turt2live/matrix-media-repo/metrics"
+	"github.com/turt2live/matrix-media-repo/util"
+)
+
+type HostRouter struct {
+	next http.Handler
+}
+
+func NewHostRouter(next http.Handler) *HostRouter {
+	return &HostRouter{next: next}
+}
+
+func (h *HostRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Header.Get("X-Forwarded-Host") != "" && config.Get().General.UseForwardedHost {
+		r.Host = r.Header.Get("X-Forwarded-Host")
+	}
+	r.Host = strings.Split(r.Host, ":")[0]
+
+	var raddr string
+	if config.Get().General.TrustAnyForward {
+		raddr = r.Header.Get("X-Forwarded-For")
+	} else {
+		raddr = xff.GetRemoteAddr(r)
+	}
+	if raddr == "" {
+		raddr = r.RemoteAddr
+	}
+	host, _, err := net.SplitHostPort(raddr)
+	if err != nil {
+		logrus.Error(err)
+		sentry.CaptureException(err)
+		host = raddr
+	}
+	r.RemoteAddr = host
+
+	ignoreHost := ShouldIgnoreHost(r)
+	isOurs := ignoreHost || util.IsServerOurs(r.Host)
+	if !isOurs {
+		logger := GetLogger(r)
+		metrics.InvalidHttpRequests.With(prometheus.Labels{
+			"action": GetActionName(r),
+			"method": r.Method,
+		}).Inc()
+		logger.Warnf("The server name provided ('%s') in the Host header is not configured, or the request was made directly to the media repo. Please specify a Host header and check your reverse proxy configuration. The request is being rejected.", r.Host)
+		w.WriteHeader(http.StatusBadGateway)
+		if b, err := json.Marshal(_responses.BadGatewayError("Review server logs to continue")); err != nil {
+			panic(errors.New("error preparing BadGatewayError: " + err.Error()))
+		} else {
+			if _, err = w.Write(b); err != nil {
+				panic(errors.New("error sending BadGatewayError: " + err.Error()))
+			}
+		}
+		return // don't call next handler
+	}
+
+	cfg := config.GetDomain(r.Host)
+	if ignoreHost {
+		dc := config.DomainConfigFrom(*config.Get())
+		cfg = &dc
+	}
+
+	ctx := r.Context()
+	ctx = context.WithValue(ctx, common.ContextDomainConfig, cfg)
+	r = r.WithContext(ctx)
+
+	if h.next != nil {
+		h.next.ServeHTTP(w, r)
+	}
+}
+
+func GetDomainConfig(r *http.Request) *config.DomainRepoConfig {
+	x, ok := r.Context().Value(common.ContextDomainConfig).(*config.DomainRepoConfig)
+	if !ok {
+		return nil
+	}
+	return x
+}

api/_routers/04-request-metrics.go

+package _routers
+
+import (
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/turt2live/matrix-media-repo/metrics"
+)
+
+type MetricsRequestRouter struct {
+	next http.Handler
+}
+
+func NewMetricsRequestRouter(next http.Handler) *MetricsRequestRouter {
+	return &MetricsRequestRouter{next: next}
+}
+
+func (m *MetricsRequestRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	metrics.HttpRequests.With(prometheus.Labels{
+		"host":   r.Host,
+		"action": GetActionName(r),
+		"method": r.Method,
+	}).Inc()
+
+	if m.next != nil {
+		m.next.ServeHTTP(w, r)
+	}
+}