Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • tedomum/matrix-media-repo
1 result
Show changes
Commits on Source (63)
Hide whitespace changes
Inline Side-by-side
Showing
with 350 additions and 185 deletions
.dockerignore
View file @ 4fc0d96c
*.key
/webui
/.idea
/bin
......
.github/workflows/binaries.yml 0 → 100644
View file @ 4fc0d96c
name: Build release binaries
on:
push:
#branches: ['travis/release-bin']
tags: ["*"]
jobs:
build:
runs-on: ${{ matrix.os }}
permissions:
contents: read
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
arch:
- amd64
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
- name: Install libheif (linux)
if: runner.os == 'linux'
run: "chmod +x ./.github/workflows/build-libheif.sh && ./.github/workflows/build-libheif.sh"
- name: Install libheif (windows)
if: runner.os == 'windows'
run: |
choco install pkgconfiglite
git clone https://github.com/Microsoft/vcpkg.git
cd vcpkg
.\bootstrap-vcpkg.bat
.\vcpkg integrate install
.\vcpkg install libde265:x64-windows
.\vcpkg install libheif:x64-windows
cd ..
- name: Dist
shell: bash
run: |
set -ex
pkgconfig=$PWD/vcpkg/installed/x64-windows/lib/pkgconfig
platform=${{ runner.os }}
platform=${platform,,}
mkdir -p $PWD/bin/dist
GOBIN=$PWD/bin go install -v ./cmd/utilities/compile_assets
$PWD/bin/compile_assets
pth="$platform-${{ matrix.arch }}"
mkdir -p $PWD/bin/$pth
PKG_CONFIG_PATH=$pkgconfig GOOS=$platform GOARCH=${{ matrix.arch }} GOBIN=$PWD/bin go build -o $PWD/bin/$pth -a -ldflags "-X github.com/t2bot/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/...
PKG_CONFIG_PATH=$pkgconfig GOOS=$platform GOARCH=${{ matrix.arch }} GOBIN=$PWD/bin go build -pgo=pgo_media_repo.pprof -o $PWD/bin/$pth -a -ldflags "-X github.com/t2bot/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/workers/media_repo
cd $PWD/bin/$pth
arch=${{ matrix.arch }}
if [ "$arch" == "amd64" ]; then
arch="x64"
fi
if [ "$platform" == "windows" ]; then
for file in * ; do mv -v $file ../dist/${file%.*}-win-${arch}.exe; done;
else
for file in * ; do mv -v $file ../dist/${file}-${platform}-${arch}; done;
fi
cd ../../
rm -rv $PWD/bin/dist/compile_assets*
- name: Upload
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}-${{ matrix.arch }}
path: ./bin/dist/*
.github/workflows/build-libheif.sh
View file @ 4fc0d96c
......@@ -3,7 +3,7 @@ set -ex
sudo apt-get install -y git cmake make pkg-config libx265-dev libde265-dev libjpeg-dev libtool
git clone https://github.com/strukturag/libheif.git
cd libheif
git checkout v1.17.1
git checkout v1.17.6
mkdir build
cd build
cmake --preset=release ..
......
.github/workflows/docker.yml 0 → 100644
View file @ 4fc0d96c
name: Create and publish a Docker image
on:
push:
branches: ['main']
tags: ['*']
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
platform:
- linux/amd64
- linux/arm/v6
- linux/arm/v7
- linux/arm64
steps:
- name: Prepare
run: |
platform=${{ matrix.platform }}
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v4
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push by digest
id: build
uses: docker/build-push-action@v5
with:
context: .
platforms: ${{ matrix.platform }}
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
- name: Export digest
run: |
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digests-${{ env.PLATFORM_PAIR }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
merge:
runs-on: ubuntu-latest
needs: [build]
steps:
- name: Download digests
uses: actions/download-artifact@v4
with:
path: /tmp/digests
pattern: digests-*
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Login
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create manifest list and push
working-directory: /tmp/digests
run: |
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
- name: Inspect image
run: |
docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
.github/workflows/download_pgo.yml
View file @ 4fc0d96c
......@@ -11,9 +11,9 @@ jobs:
env:
PGO_MERGE: ${{ secrets.PGO_MERGE }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: "Download new pgo_media_repo.pprof"
run: "curl -sv --fail -X POST -H \"Authorization: Bearer ${PGO_MERGE}\" https://pgo-mmr.t2host.io/v1/merge?and_combine=true > pgo_media_repo.pprof"
- uses: stefanzweifel/git-auto-commit-action@v4
- uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: "Update pgo_media_repo.pprof"
.github/workflows/download_providers.yml
View file @ 4fc0d96c
......@@ -9,9 +9,9 @@ jobs:
permissions:
contents: write
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: "Download new providers.json"
run: "curl -s --fail https://oembed.com/providers.json > assets/providers.json"
- uses: stefanzweifel/git-auto-commit-action@v4
- uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: "Update providers.json"
.github/workflows/main.yml
View file @ 4fc0d96c
......@@ -3,28 +3,28 @@ on:
push:
jobs:
build:
name: 'Go Build (1.20)'
name: 'Go Build'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.20'
go-version: '1.21'
- name: "Install libheif"
run: "chmod +x ./.github/workflows/build-libheif.sh && ./.github/workflows/build-libheif.sh"
- run: './build.sh' # verify the thing compiles
static:
name: 'Go Static (1.20)'
name: 'Go Static'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.20'
go-version: '1.21'
- name: "Install libheif"
run: "chmod +x ./.github/workflows/build-libheif.sh && ./.github/workflows/build-libheif.sh"
- name: "Prepare: compile assets"
run: "GOBIN=$PWD/bin go install -v ./cmd/compile_assets"
run: "GOBIN=$PWD/bin go install -v ./cmd/utilities/compile_assets"
- name: "Run: compile assets"
run: "$PWD/bin/compile_assets"
- name: "Prepare: staticcheck"
......@@ -32,15 +32,15 @@ jobs:
- run: 'go vet ./cmd/...'
- run: 'staticcheck ./cmd/...'
test:
name: 'Go Test (1.20)'
name: 'Go Test'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.20'
go-version: '1.21'
- name: "Prepare: compile assets"
run: "GOBIN=$PWD/bin go install -v ./cmd/compile_assets"
run: "GOBIN=$PWD/bin go install -v ./cmd/utilities/compile_assets"
- name: "Run: compile assets"
run: "$PWD/bin/compile_assets"
- name: "Run: tests"
......
.gitignore
View file @ 4fc0d96c
*.key
/webui
/.idea
/bin
......
CHANGELOG.md
View file @ 4fc0d96c
......@@ -9,6 +9,36 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
*Nothing yet.*
## [v1.3.4] - February 9, 2024
### Added
* Dendrite homeservers can now have their media imported safely, and `adminApiKind` may be set to `dendrite`.
* Exporting MMR's data to Synapse is now possible with `import_to_synapse`. To use it, first run `gdpr_export` or similar.
* Errors encountered during a background task, such as an API-induced export, are exposed as `error_message` in the admin API.
* MMR will follow redirects on federated downloads up to 5 hops.
* S3-backed datastores can have download requests redirected to a public-facing CDN rather than being proxied through MMR. See `publicBaseUrl` under the S3 datastore config.
### Changed
* Exports now use an internal timeout of 10 minutes instead of 1 minute when downloading files. This may still result in errors if downloading from S3 takes too long.
* MMR now requires Go 1.21 for compilation.
* ARM-supported Docker images are now available through [GHCR](https://github.com/t2bot/matrix-media-repo/pkgs/container/matrix-media-repo).
* The Docker Hub (docker.io) builds are deprecated and will not receive updates starting with v1.4.0
* Docker Hub images are not guaranteed to have ARM compatibility.
* The `latest` Docker tag on both Docker Hub and GHCR now points to the latest release instead of the unstable development build.
### Fixed
* Exports created with `s3_urls` now contain valid URLs.
* Exports no longer fail with "The requested range is not satisfiable".
* Exports no longer fail with "index out of range \[0] with length 0".
* Requests requiring authentication, but lack a provided access token, will return HTTP 401 instead of HTTP 500 now.
* Downloads when using a self-hosted MinIO instance are no longer slower than expected.
* The `DELETE /_matrix/media/unstable/admin/export/:exportId` endpoint has been reinstated as described.
* If a server's `downloads.maxSize` is greater than the `uploads.maxSize`, remote media is no longer cut off at `uploads.maxSize`. The media will instead be downloaded at `downloads.maxSize` and error if greater.
* `Content-Type` on `/download` and `/thumbnail` is now brought in line with [MSC2701](https://github.com/matrix-org/matrix-spec-proposals/pull/2701).
## [1.3.3] - October 31, 2023
### Fixed
......@@ -40,15 +70,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Mandatory Configuration Change
**Please see [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/upgrading/130.html) for details.**
**Please see [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/v1.3.3/upgrading/130.html) for details.**
### Security Fixes
* Fix improper usage of `Content-Disposition: inline` and related `Content-Type` safety ([CVE-2023-41318](https://www.cve.org/CVERecord?id=CVE-2023-41318), [GHSA-5crw-6j7v-xc72](https://github.com/turt2live/matrix-media-repo/security/advisories/GHSA-5crw-6j7v-xc72)).
* Fix improper usage of `Content-Disposition: inline` and related `Content-Type` safety ([CVE-2023-41318](https://www.cve.org/CVERecord?id=CVE-2023-41318), [GHSA-5crw-6j7v-xc72](https://github.com/t2bot/matrix-media-repo/security/advisories/GHSA-5crw-6j7v-xc72)).
### Deprecations
* The `GET /_matrix/media/unstable/local_copy/:server/:mediaId` (and `unstable/io.t2bot.media` variant) endpoint is deprecated and scheduled for removal. If you are using this endpoint, please comment on [this issue](https://github.com/turt2live/matrix-media-repo/issues/422) to explain your use case.
* The `GET /_matrix/media/unstable/local_copy/:server/:mediaId` (and `unstable/io.t2bot.media` variant) endpoint is deprecated and scheduled for removal. If you are using this endpoint, please comment on [this issue](https://github.com/t2bot/matrix-media-repo/issues/422) to explain your use case.
### Added
......@@ -74,12 +104,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Changed
* **Mandatory configuration change**: You must add datastore IDs to your datastore configuration, as matrix-media-repo will no longer manage datastores for you.
* If compiling `matrix-media-repo`, note that new external dependencies are required. See [the docs](https://docs.t2bot.io/matrix-media-repo/installing/method/compilation.html).
* If compiling `matrix-media-repo`, note that new external dependencies are required. See [the docs](https://docs.t2bot.io/matrix-media-repo/v1.3.3/installing/method/compilation.html).
* Docker images already contain these dependencies.
* Datastores no longer use the `enabled` flag set on them. Use `forKinds: []` instead to disable a datastore's usage.
* Per-user upload quotas now do not allow users to exceed the maximum values, even by 1 byte. Previously, users could exceed the limits by a little bit.
* Updated to Go 1.19, then Go 1.20 in the same release cycle.
* New CGO dependencies are required. See [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/installing/method/compilation.html) for details.
* New CGO dependencies are required. See [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/v1.3.3/installing/method/compilation.html) for details.
* Logs are now less noisy by default.
* Connected homeservers must support at least Matrix 1.1 on the Client-Server API. Servers over federation are not affected.
* The example Grafana dashboard has been updated.
......@@ -105,7 +135,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Changed
* Swap out the HEIF library for better support towards [ARM64 Docker Images](https://github.com/turt2live/matrix-media-repo/issues/365).
* Swap out the HEIF library for better support towards [ARM64 Docker Images](https://github.com/t2bot/matrix-media-repo/issues/365).
* The development environment now uses Synapse as a homeserver. Test accounts will need recreating.
* Updated to Go 1.18
* Improved error message when thumbnailer cannot determine image dimensions.
......@@ -204,7 +234,7 @@ caching that is now supported properly by this release, or disable caching if no
### Security advisories
This release includes a fix for [CVE-2021-29453](https://github.com/turt2live/matrix-media-repo/security/advisories/GHSA-j889-h476-hh9h).
This release includes a fix for [CVE-2021-29453](https://github.com/t2bot/matrix-media-repo/security/advisories/GHSA-j889-h476-hh9h).
Server administrators are recommended to upgrade as soon as possible. This issue is considered to be exploited in the wild
due to some deployments being affected unexpectedly.
......@@ -306,7 +336,7 @@ due to some deployments being affected unexpectedly.
release tracks how much a user has uploaded, which might take a while to initially calculate. If you have
a large database (more than about 100k uploaded files), run the following steps before upgrading:
1. The PostgreSQL script described [here](https://github.com/turt2live/matrix-media-repo/blob/a8951b0562debb9f8ae3b6e517bfc3a84d2e627a/migrations/17_add_user_stats_table_up.sql).
1. The PostgreSQL script described [here](https://github.com/t2bot/matrix-media-repo/blob/a8951b0562debb9f8ae3b6e517bfc3a84d2e627a/migrations/17_add_user_stats_table_up.sql).
This can be run while the server is running.
2. If you have no intention of using stats or quotas, you're done (the stats table will be inaccurate). If
you do plan on using either, run `INSERT INTO user_stats SELECT user_id, SUM(size_bytes) FROM media GROUP BY user_id;`
......@@ -351,7 +381,7 @@ a large database (more than about 100k uploaded files), run the following steps
### Added
* Added options to cache access tokens for users. This prevents excessive calls to `/account/whoami` on your homeserver, particularly for appservices.
* [Documentation](https://github.com/turt2live/matrix-media-repo/blob/master/docs/contrib/delegation.md) on how to set up delegation with the media repo and Traefik. Thanks @derEisele!
* [Documentation](https://github.com/t2bot/matrix-media-repo/blob/master/docs/contrib/delegation.md) on how to set up delegation with the media repo and Traefik. Thanks @derEisele!
### Changed
......@@ -466,31 +496,32 @@ a large database (more than about 100k uploaded files), run the following steps
* Various other features that would be expected like maximum/minimum size controls, rate limiting, etc. Check out the
sample config for a better idea of what else is possible.
[unreleased]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.3...HEAD
[1.3.3]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.2...v1.3.3
[1.3.2]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.1...v1.3.2
[1.3.1]: https://github.com/turt2live/matrix-media-repo/compare/v1.3.0...v1.3.1
[1.3.0]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.13...v1.3.0
[1.2.13]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.12...v1.2.13
[1.2.12]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.11...v1.2.12
[1.2.11]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.10...v1.2.11
[1.2.10]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.9...v1.2.10
[1.2.9]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.8...v1.2.9
[1.2.8]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.7...v1.2.8
[1.2.6]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.6...v1.2.7
[1.2.6]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.5...v1.2.6
[1.2.5]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.4...v1.2.5
[1.2.4]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.3...v1.2.4
[1.2.3]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.2...v1.2.3
[1.2.2]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.1...v1.2.2
[1.2.1]: https://github.com/turt2live/matrix-media-repo/compare/v1.2.0...v1.2.1
[1.2.0]: https://github.com/turt2live/matrix-media-repo/compare/v1.1.3...v1.2.0
[1.1.3]: https://github.com/turt2live/matrix-media-repo/compare/v1.1.2...v1.1.3
[1.1.2]: https://github.com/turt2live/matrix-media-repo/compare/v1.1.1...v1.1.2
[1.1.1]: https://github.com/turt2live/matrix-media-repo/compare/v1.1.0...v1.1.1
[1.1.0]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.2...v1.1.0
[1.0.2]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.1...v1.0.2
[1.0.1]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.0...v1.0.1
[1.0.0]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.0-rc.2...v1.0.0
[1.0.0-rc.2]: https://github.com/turt2live/matrix-media-repo/compare/v1.0.0-rc.1...v1.0.0-rc.2
[1.0.0-rc.1]: https://github.com/turt2live/matrix-media-repo/releases/tag/v1.0.0-rc.1
[unreleased]: https://github.com/t2bot/matrix-media-repo/compare/v1.3.4...HEAD
[1.3.4]: https://github.com/t2bot/matrix-media-repo/compare/v1.3.3...v1.3.4
[1.3.3]: https://github.com/t2bot/matrix-media-repo/compare/v1.3.2...v1.3.3
[1.3.2]: https://github.com/t2bot/matrix-media-repo/compare/v1.3.1...v1.3.2
[1.3.1]: https://github.com/t2bot/matrix-media-repo/compare/v1.3.0...v1.3.1
[1.3.0]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.13...v1.3.0
[1.2.13]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.12...v1.2.13
[1.2.12]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.11...v1.2.12
[1.2.11]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.10...v1.2.11
[1.2.10]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.9...v1.2.10
[1.2.9]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.8...v1.2.9
[1.2.8]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.7...v1.2.8
[1.2.6]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.6...v1.2.7
[1.2.6]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.5...v1.2.6
[1.2.5]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.4...v1.2.5
[1.2.4]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.3...v1.2.4
[1.2.3]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.2...v1.2.3
[1.2.2]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.1...v1.2.2
[1.2.1]: https://github.com/t2bot/matrix-media-repo/compare/v1.2.0...v1.2.1
[1.2.0]: https://github.com/t2bot/matrix-media-repo/compare/v1.1.3...v1.2.0
[1.1.3]: https://github.com/t2bot/matrix-media-repo/compare/v1.1.2...v1.1.3
[1.1.2]: https://github.com/t2bot/matrix-media-repo/compare/v1.1.1...v1.1.2
[1.1.1]: https://github.com/t2bot/matrix-media-repo/compare/v1.1.0...v1.1.1
[1.1.0]: https://github.com/t2bot/matrix-media-repo/compare/v1.0.2...v1.1.0
[1.0.2]: https://github.com/t2bot/matrix-media-repo/compare/v1.0.1...v1.0.2
[1.0.1]: https://github.com/t2bot/matrix-media-repo/compare/v1.0.0...v1.0.1
[1.0.0]: https://github.com/t2bot/matrix-media-repo/compare/v1.0.0-rc.2...v1.0.0
[1.0.0-rc.2]: https://github.com/t2bot/matrix-media-repo/compare/v1.0.0-rc.1...v1.0.0-rc.2
[1.0.0-rc.1]: https://github.com/t2bot/matrix-media-repo/releases/tag/v1.0.0-rc.1
Dockerfile
View file @ 4fc0d96c
# ---- Stage 0 ----
# Builds media repo binaries
FROM golang:1.20-alpine AS builder
FROM golang:1.21-alpine3.18 AS builder
# Install build dependencies
RUN apk add --no-cache git musl-dev dos2unix build-base libde265-dev
......@@ -12,7 +12,7 @@ COPY . /opt
RUN apk add --no-cache build-base libtool cmake libjpeg-turbo-dev x265-dev ffmpeg-dev zlib-dev
RUN git clone https://github.com/strukturag/libheif.git
WORKDIR /opt/libheif
RUN git checkout v1.17.1
RUN git checkout v1.17.6
RUN mkdir build
WORKDIR /opt/libheif/build
RUN cmake --preset=release ..
......@@ -24,12 +24,9 @@ WORKDIR /opt
RUN dos2unix ./build.sh ./docker/run.sh && chmod 744 ./build.sh
RUN ./build.sh
# the label is applied last so we don't pollute the image list with a weird amount of labelled images
LABEL io.t2bot.mmr.cleanup="true"
# ---- Stage 1 ----
# Final runtime stage.
FROM alpine
FROM alpine:3.18
RUN mkdir /plugins
RUN apk add --no-cache \
......@@ -40,7 +37,19 @@ RUN apk add --no-cache \
ffmpeg
COPY --from=builder /opt/bin/plugin_antispam_ocr /plugins/
COPY --from=builder /opt/bin/media_repo /opt/bin/import_synapse /opt/bin/export_synapse_for_import /opt/bin/gdpr_export /opt/bin/gdpr_import /opt/bin/s3_consistency_check /usr/local/bin/
COPY --from=builder \
/opt/bin/media_repo \
/opt/bin/import_synapse \
/opt/bin/import_dendrite \
/opt/bin/export_synapse_for_import \
/opt/bin/export_dendrite_for_import \
/opt/bin/import_to_synapse \
/opt/bin/gdpr_export \
/opt/bin/gdpr_import \
/opt/bin/s3_consistency_check \
/opt/bin/combine_signing_keys \
/opt/bin/generate_signing_key \
/usr/local/bin/
COPY ./config.sample.yaml /etc/media-repo.yaml.sample
COPY ./docker/run.sh /usr/local/bin/
......
README.md
View file @ 4fc0d96c
# matrix-media-repo
matrix-media-repo is a highly customizable multi-domain media repository for Matrix. Intended for medium to large environments
consisting of several homeservers, this media repo de-duplicates media (including remote media) while being fully compliant
with the specification.
MMR is a highly configurable multi-homeserver media repository for Matrix. It is an optional component of your homeserver
setup, and recommended only for large individual servers or hosting providers with many servers.
Smaller/individual homeservers can still make use of this project's features, though it may be difficult to set up or have
higher than expected resource consumption - please do your research before deploying this as this project may not be useful
for your environment.
**If you're looking for an S3 connector, please consider using [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) instead.**
For help and support, visit [#mediarepo:t2bot.io](https://matrix.to/#/#mediarepo:t2bot.io). Administrator documentation
can be found on [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/).
Smaller homeservers can still set this up, though may find it difficult to deploy or use. A high level of knowledge regarding
the Matrix homeserver stack is assumed.
## Installing
## Documentation and support
For installation instructions, see [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/installing/index.html).
Matrix room: [#media-repo:t2bot.io](https://matrix.to/#/#media-repo:t2bot.io)
## Deployment
For deployment information, see [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/deployment/index.html).
Documentation: [docs.t2bot.io](https://docs.t2bot.io/matrix-media-repo/)
## Developers
To properly run the media repo in a development setting, it must be compiled manually
once to ensure the assets are set up correctly: follow the
[compilation steps](https://docs.t2bot.io/matrix-media-repo/installing/method/compilation.html)
posted on docs.t2bot.io.
MMR requires compiling at least once before it'll run in a development setting. See the [compilation steps](https://docs.t2bot.io/matrix-media-repo/unstable/installation/compile)
before continuing.
This project offers a development environment you can use to test against a client and homeserver.
......@@ -34,7 +27,7 @@ As a first-time setup, run:
docker run --rm -it -v ./dev/synapse-db:/data -e SYNAPSE_SERVER_NAME=localhost -e SYNAPSE_REPORT_STATS=no matrixdotorg/synapse:latest generate
```
Then you can run `docker-compose -f dev/docker-compose.yaml up` to always bring the service online. The homeserver will
Then you can run `docker compose -f dev/docker-compose.yaml up` to always bring the service online. The homeserver will
be behind an nginx reverse proxy which routes media requests to `http://host.docker.internal:8001`. To test accurately,
it is recommended to add the following homeserver configuration to your media repo config:
......@@ -59,86 +52,5 @@ database:
Note that the postgresql image is *insecure* and not recommended for production use. It also does not follow best practices
for database management - use at your own risk.
## Importing media from synapse
Media is imported by connecting to your synapse database and downloading all the content from the homeserver. This is so
you have a backup of the media repository still with synapse. **Do not point traffic at the media repo until after the
import is complete.**
**Note**: the database options provided on the command line are for the Synapse database. The media repo will use the
connection string in the media-repo.yaml config when trying to store the Synapse media.
**Note**: the import script is not available to the Docker container. Binaries of the script are included with every
release though if you want to avoid building it yourself.
1. Build the media repo (as stated above)
2. Edit/setup `media-repo.yaml` per the install instructions above
3. Run `bin/import_synapse`. The usage is below.
```
Usage of import_synapse:
-baseUrl string
The base URL to access your homeserver with (default "http://localhost:8008")
-config string
The path to the media repo configuration (with the database section completed) (default "media-repo.yaml")
-dbHost string
The PostgresSQL hostname for your Synapse database (default "localhost")
-dbName string
The name of your Synapse database (default "synapse")
-dbPassword string
The password for your Synapse's PostgreSQL database. Can be omitted to be prompted when run
-dbPort int
The port for your Synapse's PostgreSQL database (default 5432)
-dbUsername string
The username for your Synapse's PostgreSQL database (default "synapse")
-migrations string
The absolute path the media repo's migrations folder (default "./migrations")
-serverName string
The name of your homeserver (eg: matrix.org) (default "localhost")
-workers int
The number of workers to use when downloading media. Using multiple workers risks deduplication not working as efficiently. (default 1)
```
Assuming the media repository, postgres database, and synapse are all on the same host, the command to run would look something like: `bin/import_synapse -serverName myserver.com -dbUsername my_database_user -dbName synapse`
4. Wait for the import to complete. The script will automatically deduplicate media.
5. Point traffic to the media repository.
## Export and import user data
The admin API for this is specified in [docs/admin.md](./docs/admin.md), though they can be difficult to use for scripts.
The `bin/gdpr_export` and `bin/gdpr_import` binaries do the process for you, and do so in memory but against the real
media repo database and datastores - this moves the resource intensiveness to the binary you're running instead of the
media repo instance, but still makes reads and writes to your database and datastores. For example, when exporting a
user's data the binary will pull all the data locally and write it to disk for you, but during that process the user's
export is accessible via the main media repo too. The export is deleted if the binary is successful at exporting the
data.
**Note**: Imports done through this method can affect other homeservers! For example, a user's data export could contain
an entry for a homeserver other than their own, which the media repo will happily import. Always validate the manifest
of an import before running it!
Ensuring you have your media repo config available, here's the help for each binary:
```
Usage of gdpr_export:
-config string
The path to the configuration (default "media-repo.yaml")
-destination string
The directory for where export files should be placed (default "./gdpr-data")
-entity string
The user ID or server name to export
-migrations string
The absolute path for the migrations folder (default "./migrations")
-templates string
The absolute path for the templates folder (default "./templates")
```
```
Usage of gdpr_import:
-config string
The path to the configuration (default "media-repo.yaml")
-directory string
The directory for where the entity's exported files are (default "./gdpr-data")
-migrations string
The absolute path for the migrations folder (default "./migrations")
-verify
If set, no media will be imported and instead be tested to see if they've been imported already
```
**Note**: Running the Go tests requires Docker, and may pollute your cached images with tons of layers. It is suggested to
clean these images up manually from time to time, or rely on an ephemeral build system instead.
SECURITY.md 0 → 100644
View file @ 4fc0d96c
# Security Policy
## Supported Versions
The most current release and releases within the last 6 weeks are supported.
## Reporting a Vulnerability
Please report vulnerabilities through [t2bot.io's Security Disclosure Policy](https://t2bot.io/docs/legal/security-disclosure-policy-v1/)
api/_apimeta/auth.go
View file @ 4fc0d96c
......@@ -5,9 +5,9 @@ import (
"github.com/getsentry/sentry-go"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/matrix"
"github.com/turt2live/matrix-media-repo/util"
"github.com/t2bot/matrix-media-repo/common/rcontext"
"github.com/t2bot/matrix-media-repo/matrix"
"github.com/t2bot/matrix-media-repo/util"
)
type UserInfo struct {
......
api/_auth_cache/auth_cache.go
View file @ 4fc0d96c
......@@ -8,8 +8,8 @@ import (
"time"
"github.com/patrickmn/go-cache"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/matrix"
"github.com/t2bot/matrix-media-repo/common/rcontext"
"github.com/t2bot/matrix-media-repo/matrix"
)
var tokenCache = cache.New(0*time.Second, 30*time.Second)
......
api/_responses/errors.go
View file @ 4fc0d96c
package _responses
import "github.com/turt2live/matrix-media-repo/common"
import "github.com/t2bot/matrix-media-repo/common"
type ErrorResponse struct {
Code string `json:"errcode"`
......
api/_responses/redirect.go 0 → 100644
View file @ 4fc0d96c
package _responses
type RedirectResponse struct {
ToUrl string
}
func Redirect(url string) *RedirectResponse {
return &RedirectResponse{ToUrl: url}
}
api/_routers/01-install_metadata.go
View file @ 4fc0d96c
......@@ -6,8 +6,8 @@ import (
"strconv"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/util"
"github.com/t2bot/matrix-media-repo/common"
"github.com/t2bot/matrix-media-repo/util"
)
type RequestCounter struct {
......
api/_routers/03-host_detection.go
View file @ 4fc0d96c
......@@ -12,11 +12,11 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/sebest/xff"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/api/_responses"
"github.com/turt2live/matrix-media-repo/common"
"github.com/turt2live/matrix-media-repo/common/config"
"github.com/turt2live/matrix-media-repo/metrics"
"github.com/turt2live/matrix-media-repo/util"
"github.com/t2bot/matrix-media-repo/api/_responses"
"github.com/t2bot/matrix-media-repo/common"
"github.com/t2bot/matrix-media-repo/common/config"
"github.com/t2bot/matrix-media-repo/metrics"
"github.com/t2bot/matrix-media-repo/util"
)
type HostRouter struct {
......
api/_routers/04-request-metrics.go
View file @ 4fc0d96c
......@@ -4,7 +4,7 @@ import (
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/turt2live/matrix-media-repo/metrics"
"github.com/t2bot/matrix-media-repo/metrics"
)
type MetricsRequestRouter struct {
......
api/_routers/97-optional-access-token.go
View file @ 4fc0d96c
......@@ -6,13 +6,13 @@ import (
"github.com/getsentry/sentry-go"
"github.com/sirupsen/logrus"
"github.com/turt2live/matrix-media-repo/api/_apimeta"
"github.com/turt2live/matrix-media-repo/api/_auth_cache"
"github.com/turt2live/matrix-media-repo/api/_responses"
"github.com/turt2live/matrix-media-repo/common/config"
"github.com/turt2live/matrix-media-repo/common/rcontext"
"github.com/turt2live/matrix-media-repo/matrix"
"github.com/turt2live/matrix-media-repo/util"
"github.com/t2bot/matrix-media-repo/api/_apimeta"
"github.com/t2bot/matrix-media-repo/api/_auth_cache"
"github.com/t2bot/matrix-media-repo/api/_responses"
"github.com/t2bot/matrix-media-repo/common/config"
"github.com/t2bot/matrix-media-repo/common/rcontext"
"github.com/t2bot/matrix-media-repo/matrix"
"github.com/t2bot/matrix-media-repo/util"
)
func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn {
......