Skip to content
Snippets Groups Projects
Commit 081a53ae authored by Travis Ralston's avatar Travis Ralston
Browse files

Don't rely on StatObject/FileExists checks

Fixes https://github.com/turt2live/matrix-media-repo/issues/446

We instead just generate something unique enough.
parent 0748ba8b
No related branches found
No related tags found
No related merge requests found
......@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"io"
"net/http"
"os"
"path"
......@@ -22,8 +21,14 @@ func Upload(ctx rcontext.RequestContext, ds config.DatastoreConfig, data io.Read
defer data.Close()
hasher := sha256.New()
tee := io.TeeReader(data, hasher)
var objectName string
var err error
objectName, err := ids.NewUniqueId()
if err != nil {
return "", err
}
// Suffix the ID so file paths are correctly bucketed
objectName = fmt.Sprintf("%sidv2fmt", objectName)
var uploadedBytes int64
if ds.Type == "s3" {
......@@ -33,31 +38,6 @@ func Upload(ctx rcontext.RequestContext, ds config.DatastoreConfig, data io.Read
return "", err
}
// Ensure unique ID
exists := true
attempts := 0
for exists {
objectName, err = ids.NewUniqueId()
if err != nil {
return "", err
}
attempts++
if attempts > 10 {
return "", errors.New("failed to generate suitable object name for S3 store")
}
metrics.S3Operations.With(prometheus.Labels{"operation": "StatObject"}).Inc()
_, err = s3c.client.StatObject(ctx.Context, s3c.bucket, objectName, minio.StatObjectOptions{})
if err != nil {
var merr minio.ErrorResponse
if errors.As(err, &merr) {
if merr.Code == "NoSuchKey" || merr.StatusCode == http.StatusNotFound {
exists = false
}
}
}
}
metrics.S3Operations.With(prometheus.Labels{"operation": "PutObject"}).Inc()
var info minio.UploadInfo
info, err = s3c.client.PutObject(ctx.Context, s3c.bucket, objectName, tee, size, minio.PutObjectOptions{StorageClass: s3c.storageClass, ContentType: contentType})
......@@ -71,34 +51,12 @@ func Upload(ctx rcontext.RequestContext, ds config.DatastoreConfig, data io.Read
var targetDir string
var targetFile string
// Ensure unique ID
exists := true
attempts := 0
for exists {
objectName, err = ids.NewUniqueId()
if err != nil {
return "", err
}
attempts++
if attempts > 10 {
return "", errors.New("failed to generate suitable file name for persistence")
}
firstContainer = objectName[0:2]
secondContainer = objectName[2:4]
fileName = objectName[4:]
objectName = path.Join(firstContainer, secondContainer, fileName)
targetDir = path.Join(basePath, firstContainer, secondContainer)
targetFile = path.Join(targetDir, fileName)
_, err = os.Stat(targetFile)
if err != nil && !os.IsNotExist(err) {
return "", err
} else if err != nil && os.IsNotExist(err) {
exists = false
}
}
firstContainer = objectName[0:2]
secondContainer = objectName[2:4]
fileName = objectName[4:]
objectName = path.Join(firstContainer, secondContainer, fileName)
targetDir = path.Join(basePath, firstContainer, secondContainer)
targetFile = path.Join(targetDir, fileName)
// Persist file
var file *os.File
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment