diff options
-rw-r--r-- | api/emoji_test.go | 2 | ||||
-rw-r--r-- | api/file.go | 10 | ||||
-rw-r--r-- | api/file_test.go | 53 | ||||
-rw-r--r-- | api4/file.go | 8 | ||||
-rw-r--r-- | api4/file_test.go | 8 | ||||
-rw-r--r-- | app/emoji.go | 10 | ||||
-rw-r--r-- | app/file.go | 257 | ||||
-rw-r--r-- | app/file_test.go | 55 | ||||
-rw-r--r-- | app/import.go | 6 | ||||
-rw-r--r-- | app/slackimport.go | 3 | ||||
-rw-r--r-- | app/user.go | 6 | ||||
-rw-r--r-- | cmd/platform/server.go | 2 | ||||
-rw-r--r-- | i18n/en.json | 36 | ||||
-rw-r--r-- | model/file_info.go | 3 | ||||
-rw-r--r-- | store/sql_file_info_store.go | 40 | ||||
-rw-r--r-- | store/sql_file_info_store_test.go | 38 | ||||
-rw-r--r-- | store/store.go | 1 | ||||
-rw-r--r-- | utils/file.go | 313 | ||||
-rw-r--r-- | utils/file_test.go | 122 | ||||
-rw-r--r-- | utils/time.go | 4 |
20 files changed, 678 insertions, 299 deletions
diff --git a/api/emoji_test.go b/api/emoji_test.go index 600f7975e..4de5f41dd 100644 --- a/api/emoji_test.go +++ b/api/emoji_test.go @@ -266,7 +266,7 @@ func TestDeleteEmoji(t *testing.T) { func createTestEmoji(t *testing.T, emoji *model.Emoji, imageData []byte) *model.Emoji { emoji = store.Must(app.Srv.Store.Emoji().Save(emoji)).(*model.Emoji) - if err := app.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil { + if err := utils.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil { store.Must(app.Srv.Store.Emoji().Delete(emoji.Id, time.Now().Unix())) t.Fatalf("failed to write image: %v", err.Error()) } diff --git a/api/file.go b/api/file.go index 342430ed1..1eab30e76 100644 --- a/api/file.go +++ b/api/file.go @@ -96,7 +96,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := app.ReadFile(info.Path); err != nil { + if data, err := utils.ReadFile(info.Path); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil { @@ -118,7 +118,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := app.ReadFile(info.ThumbnailPath); err != nil { + if data, err := utils.ReadFile(info.ThumbnailPath); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, w, r); err != nil { @@ -140,7 +140,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := app.ReadFile(info.PreviewPath); err != nil { + if data, err := utils.ReadFile(info.PreviewPath); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, w, r); err != nil { @@ -190,7 +190,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := app.ReadFile(info.Path); err != nil { + if data, err := utils.ReadFile(info.Path); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil { @@ -285,7 +285,7 @@ func getPublicFileOld(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := app.ReadFile(info.Path); err != nil { + if data, err := utils.ReadFile(info.Path); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil { diff --git a/api/file_test.go b/api/file_test.go index 0e0a5772c..282cff2ec 100644 --- a/api/file_test.go +++ b/api/file_test.go @@ -81,20 +81,22 @@ func TestUploadFile(t *testing.T) { t.Fatal("file preview path should be set in database") } + date := time.Now().Format("20060102") + // This also makes sure that the relative path provided above is sanitized out - expectedPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test.png", team.Id, channel.Id, user.Id, info.Id) + expectedPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test.png", date, team.Id, channel.Id, user.Id, info.Id) if info.Path != expectedPath { t.Logf("file is saved in %v", info.Path) t.Fatalf("file should've been saved in %v", expectedPath) } - expectedThumbnailPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", team.Id, channel.Id, user.Id, info.Id) + expectedThumbnailPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", date, team.Id, channel.Id, user.Id, info.Id) if info.ThumbnailPath != expectedThumbnailPath { t.Logf("file thumbnail is saved in %v", info.ThumbnailPath) t.Fatalf("file thumbnail should've been saved in %v", expectedThumbnailPath) } - expectedPreviewPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_preview.jpg", team.Id, channel.Id, user.Id, info.Id) + expectedPreviewPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_preview.jpg", date, team.Id, channel.Id, user.Id, info.Id) if info.PreviewPath != expectedPreviewPath { t.Logf("file preview is saved in %v", info.PreviewPath) t.Fatalf("file preview should've been saved in %v", expectedPreviewPath) @@ -466,7 +468,6 @@ func TestGetPublicFileOld(t *testing.T) { utils.Cfg.FileSettings.EnablePublicLink = true *utils.Cfg.FileSettings.PublicLinkSalt = model.NewId() - Client := th.BasicClient channel := th.BasicChannel var fileId string @@ -474,7 +475,16 @@ func TestGetPublicFileOld(t *testing.T) { if err != nil { t.Fatal(err) } else { - fileId = Client.MustGeneric(Client.UploadPostAttachment(data, channel.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id + //fileId = Client.MustGeneric(Client.UploadPostAttachment(data, channel.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id + fileId = model.NewId() + fileInfo := model.FileInfo{ + Id: fileId, + CreateAt: model.GetMillis(), + CreatorId: th.BasicUser.Id, + Path: fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", th.BasicTeam.Id, channel.Id, th.BasicUser.Id, fileId, "test.png"), + } + store.Must(app.Srv.Store.FileInfo().Save(&fileInfo)) + uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", th.BasicTeam.Id, channel.Id, th.BasicUser.Id, fileId), "test.png") } // Hacky way to assign file to a post (usually would be done by CreatePost call) @@ -619,7 +629,9 @@ func TestMigrateFilenamesToFileInfos(t *testing.T) { t.Fatal(err) } else { fileId1 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id + uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", th.BasicTeam.Id, channel1.Id, user1.Id, fileId1), "test.png") fileId2 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id + uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", th.BasicTeam.Id, channel1.Id, user1.Id, fileId2), "test.png") } // Bypass the Client whenever possible since we're trying to simulate a pre-3.5 post @@ -686,6 +698,25 @@ func TestMigrateFilenamesToFileInfos(t *testing.T) { } } +func uploadFileOld(t *testing.T, data []byte, dest string, filename string) { + os.MkdirAll(dest, os.ModePerm) + eFile, err := os.Create(dest + "/" + filename) + if err != nil { + t.Fatal(err) + } + defer eFile.Close() + + _, err = io.Copy(eFile, bytes.NewReader(data)) // first var shows number of bytes + if err != nil { + t.Fatal(err) + } + + err = eFile.Sync() + if err != nil { + t.Fatal(err) + } +} + func TestFindTeamIdForFilename(t *testing.T) { th := Setup().InitBasic() @@ -717,9 +748,11 @@ func TestFindTeamIdForFilename(t *testing.T) { t.Fatal(err) } else { fileId1 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id + uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", team1.Id, channel1.Id, user1.Id, fileId1), "test.png") Client.SetTeamId(team2.Id) fileId2 = Client.MustGeneric(Client.UploadPostAttachment(data, channel2.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id + uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", team2.Id, channel2.Id, user1.Id, fileId2), "test.png") Client.SetTeamId(team1.Id) } @@ -732,6 +765,7 @@ func TestFindTeamIdForFilename(t *testing.T) { })).(*model.Post) if teamId := app.FindTeamIdForFilename(post1, post1.Filenames[0]); teamId != team1.Id { + t.Log(teamId) t.Fatal("file should've been found under team1") } @@ -773,6 +807,7 @@ func TestGetInfoForFilename(t *testing.T) { t.Fatal(err) } else { fileId1 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id + uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", team1.Id, channel1.Id, user1.Id, fileId1), "test.png") path = store.Must(app.Srv.Store.FileInfo().Get(fileId1)).(*model.FileInfo).Path thumbnailPath = store.Must(app.Srv.Store.FileInfo().Get(fileId1)).(*model.FileInfo).ThumbnailPath previewPath = store.Must(app.Srv.Store.FileInfo().Get(fileId1)).(*model.FileInfo).PreviewPath @@ -786,6 +821,8 @@ func TestGetInfoForFilename(t *testing.T) { Filenames: []string{fmt.Sprintf("/%s/%s/%s/%s", channel1.Id, user1.Id, fileId1, "test.png")}, })).(*model.Post) + date := time.Now().Format("20060102") + if info := app.GetInfoForFilename(post1, team1.Id, post1.Filenames[0]); info == nil { t.Fatal("info shouldn't be nil") } else if info.Id == "" { @@ -794,11 +831,11 @@ func TestGetInfoForFilename(t *testing.T) { t.Fatal("incorrect user id") } else if info.PostId != post1.Id { t.Fatal("incorrect user id") - } else if info.Path != path { + } else if fmt.Sprintf("%s/%s", date, info.Path) != path { t.Fatal("incorrect path") - } else if info.ThumbnailPath != thumbnailPath { + } else if fmt.Sprintf("%s/%s", date, info.ThumbnailPath) != thumbnailPath { t.Fatal("incorrect thumbnail path") - } else if info.PreviewPath != previewPath { + } else if fmt.Sprintf("%s/%s", date, info.PreviewPath) != previewPath { t.Fatal("incorrect preview path") } else if info.Name != "test.png" { t.Fatal("incorrect name") diff --git a/api4/file.go b/api4/file.go index 0607c1942..7dc13dafc 100644 --- a/api4/file.go +++ b/api4/file.go @@ -123,7 +123,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - data, err := app.ReadFile(info.Path) + data, err := utils.ReadFile(info.Path) if err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound @@ -165,7 +165,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := app.ReadFile(info.ThumbnailPath); err != nil { + if data, err := utils.ReadFile(info.ThumbnailPath); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, forceDownload, w, r); err != nil { @@ -237,7 +237,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := app.ReadFile(info.PreviewPath); err != nil { + if data, err := utils.ReadFile(info.PreviewPath); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, forceDownload, w, r); err != nil { @@ -299,7 +299,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := app.ReadFile(info.Path); err != nil { + if data, err := utils.ReadFile(info.Path); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, info.MimeType, data, true, w, r); err != nil { diff --git a/api4/file_test.go b/api4/file_test.go index c2bc926e1..a2673dc8e 100644 --- a/api4/file_test.go +++ b/api4/file_test.go @@ -71,20 +71,22 @@ func TestUploadFile(t *testing.T) { t.Fatal("file preview path should be set in database") } + date := time.Now().Format("20060102") + // This also makes sure that the relative path provided above is sanitized out - expectedPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test.png", FILE_TEAM_ID, channel.Id, user.Id, info.Id) + expectedPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test.png", date, FILE_TEAM_ID, channel.Id, user.Id, info.Id) if info.Path != expectedPath { t.Logf("file is saved in %v", info.Path) t.Fatalf("file should've been saved in %v", expectedPath) } - expectedThumbnailPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", FILE_TEAM_ID, channel.Id, user.Id, info.Id) + expectedThumbnailPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", date, FILE_TEAM_ID, channel.Id, user.Id, info.Id) if info.ThumbnailPath != expectedThumbnailPath { t.Logf("file thumbnail is saved in %v", info.ThumbnailPath) t.Fatalf("file thumbnail should've been saved in %v", expectedThumbnailPath) } - expectedPreviewPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_preview.jpg", FILE_TEAM_ID, channel.Id, user.Id, info.Id) + expectedPreviewPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_preview.jpg", date, FILE_TEAM_ID, channel.Id, user.Id, info.Id) if info.PreviewPath != expectedPreviewPath { t.Logf("file preview is saved in %v", info.PreviewPath) t.Fatalf("file preview should've been saved in %v", expectedPreviewPath) diff --git a/app/emoji.go b/app/emoji.go index f1362b798..e01ca97ae 100644 --- a/app/emoji.go +++ b/app/emoji.go @@ -101,7 +101,7 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro if err := gif.EncodeAll(newbuf, resized_gif); err != nil { return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.gif_encode_error", nil, "", http.StatusBadRequest) } - if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { + if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { return err } } @@ -113,13 +113,13 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro if err := png.Encode(newbuf, resized_image); err != nil { return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.encode_error", nil, "", http.StatusBadRequest) } - if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { + if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { return err } } } } else { - if err := WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil { + if err := utils.WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil { return err } } @@ -159,7 +159,7 @@ func GetEmojiImage(emojiId string) (imageByte []byte, imageType string, err *mod } else { var img []byte - if data, err := ReadFile(getEmojiImagePath(emojiId)); err != nil { + if data, err := utils.ReadFile(getEmojiImagePath(emojiId)); err != nil { return nil, "", model.NewAppError("getEmojiImage", "api.emoji.get_image.read.app_error", nil, err.Error(), http.StatusNotFound) } else { img = data @@ -219,7 +219,7 @@ func imageToPaletted(img image.Image) *image.Paletted { } func deleteEmojiImage(id string) { - if err := MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil { + if err := utils.MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil { l4g.Error("Failed to rename image when deleting emoji %v", id) } } diff --git a/app/file.go b/app/file.go index dc7caff41..10fb1425c 100644 --- a/app/file.go +++ b/app/file.go @@ -14,23 +14,21 @@ import ( _ "image/gif" "image/jpeg" "io" - "io/ioutil" "mime/multipart" "net/http" "net/url" - "os" "path/filepath" "strings" "sync" + "time" l4g "github.com/alecthomas/log4go" "github.com/disintegration/imaging" - "github.com/mattermost/platform/model" - "github.com/mattermost/platform/utils" - s3 "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/credentials" "github.com/rwcarlsen/goexif/exif" _ "golang.org/x/image/bmp" + + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" ) const ( @@ -57,222 +55,8 @@ const ( IMAGE_THUMBNAIL_PIXEL_WIDTH = 120 IMAGE_THUMBNAIL_PIXEL_HEIGHT = 100 IMAGE_PREVIEW_PIXEL_WIDTH = 1024 - - TEST_FILE_PATH = "/testfile" ) -// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. -// If signV2 input is false, function always returns signature v4. -// -// Additionally this function also takes a user defined region, if set -// disables automatic region lookup. -func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) { - var creds *credentials.Credentials - if signV2 { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2) - } else { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4) - } - return s3.NewWithCredentials(endpoint, creds, secure, region) -} - -func TestFileConnection() *model.AppError { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint - accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *utils.Cfg.FileSettings.AmazonS3SSL - signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2 - region := utils.Cfg.FileSettings.AmazonS3Region - bucket := utils.Cfg.FileSettings.AmazonS3Bucket - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewLocAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error()) - } - - exists, err := s3Clnt.BucketExists(bucket) - if err != nil { - return model.NewLocAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error()) - } - - if !exists { - l4g.Warn("Bucket specified does not exist. Attempting to create...") - err := s3Clnt.MakeBucket(bucket, region) - if err != nil { - l4g.Error("Unable to create bucket.") - return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) - } - } - l4g.Info("Connection to S3 or minio is good. Bucket exists.") - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - f := []byte("testingwrite") - if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil { - return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError) - } - os.Remove(utils.Cfg.FileSettings.Directory + TEST_FILE_PATH) - l4g.Info("Able to write files to local storage.") - } else { - return model.NewLocAppError("TestFileConnection", "No file driver selected.", nil, "") - } - - return nil -} - -func ReadFile(path string) ([]byte, *model.AppError) { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint - accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *utils.Cfg.FileSettings.AmazonS3SSL - signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2 - region := utils.Cfg.FileSettings.AmazonS3Region - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) - } - bucket := utils.Cfg.FileSettings.AmazonS3Bucket - minioObject, err := s3Clnt.GetObject(bucket, path) - defer minioObject.Close() - if err != nil { - return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) - } - if f, err := ioutil.ReadAll(minioObject); err != nil { - return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) - } else { - return f, nil - } - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewLocAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error()) - } else { - return f, nil - } - } else { - return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented) - } -} - -func MoveFile(oldPath, newPath string) *model.AppError { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint - accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *utils.Cfg.FileSettings.AmazonS3SSL - signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2 - region := utils.Cfg.FileSettings.AmazonS3Region - encrypt := false - if *utils.Cfg.FileSettings.AmazonS3SSE && utils.IsLicensed() && *utils.License().Features.Compliance { - encrypt = true - } - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error()) - } - bucket := utils.Cfg.FileSettings.AmazonS3Bucket - - source := s3.NewSourceInfo(bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt)) - if err != nil { - return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error()) - } - if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) - } - if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil { - return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) - } - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+newPath), 0774); err != nil { - return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error()) - } - - if err := os.Rename(utils.Cfg.FileSettings.Directory+oldPath, utils.Cfg.FileSettings.Directory+newPath); err != nil { - return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error()) - } - } else { - return model.NewLocAppError("moveFile", "api.file.move_file.configured.app_error", nil, "") - } - - return nil -} - -func WriteFile(f []byte, path string) *model.AppError { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint - accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *utils.Cfg.FileSettings.AmazonS3SSL - signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2 - region := utils.Cfg.FileSettings.AmazonS3Region - encrypt := false - if *utils.Cfg.FileSettings.AmazonS3SSE && utils.IsLicensed() && *utils.License().Features.Compliance { - encrypt = true - } - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) - } - - bucket := utils.Cfg.FileSettings.AmazonS3Bucket - ext := filepath.Ext(path) - metaData := S3Metadata(encrypt, "binary/octet-stream") - if model.IsFileExtImage(ext) { - metaData = S3Metadata(encrypt, model.GetImageMimeType(ext)) - } - - _, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil) - if err != nil { - return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) - } - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil { - return err - } - } else { - return model.NewLocAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "") - } - - return nil -} - -func writeFileLocally(f []byte, path string) *model.AppError { - if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { - directory, _ := filepath.Abs(filepath.Dir(path)) - return model.NewLocAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error()) - } - - if err := ioutil.WriteFile(path, f, 0644); err != nil { - return model.NewLocAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error()) - } - - return nil -} - -func openFileWriteStream(path string) (io.Writer, *model.AppError) { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.s3.app_error", nil, "") - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+path), 0774); err != nil { - return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.creating_dir.app_error", nil, err.Error()) - } - - if fileHandle, err := os.Create(utils.Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.local_server.app_error", nil, err.Error()) - } else { - fileHandle.Chmod(0644) - return fileHandle, nil - } - } - - return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.configured.app_error", nil, "") -} - -func closeFileWriteStream(file io.Writer) { - file.(*os.File).Close() -} - func GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo { // Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension} split := strings.SplitN(filename, "/", 5) @@ -295,7 +79,7 @@ func GetInfoForFilename(post *model.Post, teamId string, filename string) *model // Open the file and populate the fields of the FileInfo var info *model.FileInfo - if data, err := ReadFile(path); err != nil { + if data, err := utils.ReadFile(path); err != nil { l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.file_not_found.error"), post.Id, filename, path, err) return nil } else { @@ -337,7 +121,7 @@ func FindTeamIdForFilename(post *model.Post, filename string) string { } else { for _, team := range teams { path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name) - if _, err := ReadFile(path); err == nil { + if _, err := utils.ReadFile(path); err == nil { // Found the team that this file was posted from return team.Id } @@ -484,7 +268,7 @@ func UploadFiles(teamId string, channelId string, userId string, fileHeaders []* io.Copy(buf, file) data := buf.Bytes() - info, err := DoUploadFile(teamId, channelId, userId, fileHeader.Filename, data) + info, err := DoUploadFile(time.Now(), teamId, channelId, userId, fileHeader.Filename, data) if err != nil { return nil, err } @@ -507,7 +291,7 @@ func UploadFiles(teamId string, channelId string, userId string, fileHeaders []* return resStruct, nil } -func DoUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { +func DoUploadFile(now time.Time, teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { filename := filepath.Base(rawFilename) info, err := model.GetInfoForBytes(filename, data) @@ -519,7 +303,7 @@ func DoUploadFile(teamId string, channelId string, userId string, rawFilename st info.Id = model.NewId() info.CreatorId = userId - pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/" + pathPrefix := now.Format("20060102") + "/teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/" info.Path = pathPrefix + filename if info.IsImage() { @@ -535,7 +319,7 @@ func DoUploadFile(teamId string, channelId string, userId string, rawFilename st info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" } - if err := WriteFile(data, info.Path); err != nil { + if err := utils.WriteFile(data, info.Path); err != nil { return nil, err } @@ -652,7 +436,7 @@ func generateThumbnailImage(img image.Image, thumbnailPath string, width int, he return } - if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil { + if err := utils.WriteFile(buf.Bytes(), thumbnailPath); err != nil { l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err) return } @@ -674,7 +458,7 @@ func generatePreviewImage(img image.Image, previewPath string, width int) { return } - if err := WriteFile(buf.Bytes(), previewPath); err != nil { + if err := utils.WriteFile(buf.Bytes(), previewPath); err != nil { l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err) return } @@ -687,20 +471,3 @@ func GetFileInfo(fileId string) (*model.FileInfo, *model.AppError) { return result.Data.(*model.FileInfo), nil } } - -func S3Metadata(encrypt bool, contentType string) map[string][]string { - metaData := make(map[string][]string) - if contentType != "" { - metaData["Content-Type"] = []string{"contentType"} - } - if encrypt { - metaData["x-amz-server-side-encryption"] = []string{"AES256"} - } - return metaData -} - -func CopyMetadata(encrypt bool) map[string]string { - metaData := make(map[string]string) - metaData["x-amz-server-side-encryption"] = "AES256" - return metaData -} diff --git a/app/file_test.go b/app/file_test.go index 683b574b8..962661039 100644 --- a/app/file_test.go +++ b/app/file_test.go @@ -4,9 +4,12 @@ package app import ( + "fmt" "testing" + "time" "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" ) func TestGeneratePublicLinkHash(t *testing.T) { @@ -31,3 +34,55 @@ func TestGeneratePublicLinkHash(t *testing.T) { t.Fatal("hashes for the same file with different salts should not be equal") } } + +func TestDoUploadFile(t *testing.T) { + Setup() + + teamId := model.NewId() + channelId := model.NewId() + userId := model.NewId() + filename := "test" + data := []byte("abcd") + + info1, err := DoUploadFile(time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data) + if err != nil { + t.Fatal(err) + } else { + defer func() { + <-Srv.Store.FileInfo().PermanentDelete(info1.Id) + utils.RemoveFile(info1.Path) + }() + } + + if info1.Path != fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info1.Id, filename) { + t.Fatal("stored file at incorrect path", info1.Path) + } + + info2, err := DoUploadFile(time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data) + if err != nil { + t.Fatal(err) + } else { + defer func() { + <-Srv.Store.FileInfo().PermanentDelete(info2.Id) + utils.RemoveFile(info2.Path) + }() + } + + if info2.Path != fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info2.Id, filename) { + t.Fatal("stored file at incorrect path", info2.Path) + } + + info3, err := DoUploadFile(time.Date(2008, 3, 5, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data) + if err != nil { + t.Fatal(err) + } else { + defer func() { + <-Srv.Store.FileInfo().PermanentDelete(info3.Id) + utils.RemoveFile(info3.Path) + }() + } + + if info3.Path != fmt.Sprintf("20080305/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info3.Id, filename) { + t.Fatal("stored file at incorrect path", info3.Path) + } +} diff --git a/app/import.go b/app/import.go index fb7d43cdf..d404dbadd 100644 --- a/app/import.go +++ b/app/import.go @@ -12,9 +12,11 @@ import ( "regexp" "strings" "sync" + "time" "unicode/utf8" l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/model" "github.com/mattermost/platform/store" "github.com/mattermost/platform/utils" @@ -1486,12 +1488,12 @@ func OldImportChannel(channel *model.Channel) *model.Channel { } } -func OldImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { +func OldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { buf := bytes.NewBuffer(nil) io.Copy(buf, file) data := buf.Bytes() - fileInfo, err := DoUploadFile(teamId, channelId, userId, fileName, data) + fileInfo, err := DoUploadFile(timestamp, teamId, channelId, userId, fileName, data) if err != nil { return nil, err } diff --git a/app/slackimport.go b/app/slackimport.go index 4470b8323..e57a3a3d1 100644 --- a/app/slackimport.go +++ b/app/slackimport.go @@ -401,7 +401,8 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin } defer openFile.Close() - uploadedFile, err := OldImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name)) + timestamp := utils.TimeFromMillis(SlackConvertTimeStamp(sPost.TimeStamp)) + uploadedFile, err := OldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name)) if err != nil { l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) return nil, false diff --git a/app/user.go b/app/user.go index 813421a5c..40e32c282 100644 --- a/app/user.go +++ b/app/user.go @@ -758,7 +758,7 @@ func GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) { } else { path := "users/" + user.Id + "/profile.png" - if data, err := ReadFile(path); err != nil { + if data, err := utils.ReadFile(path); err != nil { readFailed = true if img, err = CreateProfileImage(user.Username, user.Id); err != nil { @@ -766,7 +766,7 @@ func GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) { } if user.LastPictureUpdate == 0 { - if err := WriteFile(img, path); err != nil { + if err := utils.WriteFile(img, path); err != nil { return nil, false, err } } @@ -819,7 +819,7 @@ func SetProfileImage(userId string, imageData *multipart.FileHeader) *model.AppE path := "users/" + userId + "/profile.png" - if err := WriteFile(buf.Bytes(), path); err != nil { + if err := utils.WriteFile(buf.Bytes(), path); err != nil { return model.NewLocAppError("SetProfileImage", "api.user.upload_profile_user.upload_profile.app_error", nil, "") } diff --git a/cmd/platform/server.go b/cmd/platform/server.go index ac8f3baff..f5be5a5fc 100644 --- a/cmd/platform/server.go +++ b/cmd/platform/server.go @@ -67,7 +67,7 @@ func runServer(configFileLocation string) { *utils.Cfg.ServiceSettings.EnableDeveloper = true } - if err := app.TestFileConnection(); err != nil { + if err := utils.TestFileConnection(); err != nil { l4g.Error("Problem with file storage settings: " + err.Error()) } diff --git a/i18n/en.json b/i18n/en.json index 900a6865f..338eba46e 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1234,32 +1234,40 @@ "translation": "Unable to move file locally." }, { - "id": "api.file.open_file_write_stream.configured.app_error", + "id": "api.file.read_file.configured.app_error", "translation": "File storage not configured properly. Please configure for either S3 or local server file storage." }, { - "id": "api.file.open_file_write_stream.creating_dir.app_error", - "translation": "Encountered an error creating the directory for the new file" + "id": "api.file.read_file.get.app_error", + "translation": "Unable to get file from S3" }, { - "id": "api.file.open_file_write_stream.local_server.app_error", - "translation": "Encountered an error writing to local server storage" + "id": "api.file.read_file.reading_local.app_error", + "translation": "Encountered an error reading from local server storage" }, { - "id": "api.file.open_file_write_stream.s3.app_error", - "translation": "S3 is not supported." + "id": "utils.file.remove_file.local.app_error", + "translation": "Encountered an error removing file from local server file storage." }, { - "id": "api.file.read_file.configured.app_error", + "id": "utils.file.remove_file.s3.app_error", + "translation": "Encountered an error removing file from S3." + }, + { + "id": "utils.file.remove_file.configured.app_error", "translation": "File storage not configured properly. Please configure for either S3 or local server file storage." }, { - "id": "api.file.read_file.get.app_error", - "translation": "Unable to get file from S3" + "id": "utils.file.remove_directory.local.app_error", + "translation": "Encountered an error removing directory from local server file storage." }, { - "id": "api.file.read_file.reading_local.app_error", - "translation": "Encountered an error reading from local server storage" + "id": "utils.file.remove_directory.s3.app_error", + "translation": "Encountered an error removing directory from S3." + }, + { + "id": "utils.file.remove_directory.configured.app_error", + "translation": "File storage not configured properly. Please configure for either S3 or local server file storage." }, { "id": "api.file.upload_file.bad_parse.app_error", @@ -5576,6 +5584,10 @@ "translation": "We couldn't save or update the file info" }, { + "id": "store.sql_file_info.permanent_delete.app_error", + "translation": "We couldn't permanently delete the file info" + }, + { "id": "store.sql_job.delete.app_error", "translation": "We couldn't delete the job" }, diff --git a/model/file_info.go b/model/file_info.go index 8b5684127..3071e6d4b 100644 --- a/model/file_info.go +++ b/model/file_info.go @@ -80,6 +80,9 @@ func (o *FileInfo) PreSave() { if o.CreateAt == 0 { o.CreateAt = GetMillis() + } + + if o.UpdateAt < o.CreateAt { o.UpdateAt = o.CreateAt } } diff --git a/store/sql_file_info_store.go b/store/sql_file_info_store.go index 3fe62121c..4bf6c9c73 100644 --- a/store/sql_file_info_store.go +++ b/store/sql_file_info_store.go @@ -97,7 +97,7 @@ func (fs SqlFileInfoStore) Get(id string) StoreChannel { if err == sql.ErrNoRows { result.Err = model.NewAppError("SqlFileInfoStore.Get", "store.sql_file_info.get.app_error", nil, "id="+id+", "+err.Error(), http.StatusNotFound) } else { - result.Err = model.NewLocAppError("SqlFileInfoStore.Get", "store.sql_file_info.get.app_error", nil, "id="+id+", "+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.Get", "store.sql_file_info.get.app_error", nil, "id="+id+", "+err.Error(), http.StatusInternalServerError) } } else { result.Data = info @@ -127,7 +127,7 @@ func (fs SqlFileInfoStore) GetByPath(path string) StoreChannel { Path = :Path AND DeleteAt = 0 LIMIT 1`, map[string]interface{}{"Path": path}); err != nil { - result.Err = model.NewLocAppError("SqlFileInfoStore.GetByPath", "store.sql_file_info.get_by_path.app_error", nil, "path="+path+", "+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.GetByPath", "store.sql_file_info.get_by_path.app_error", nil, "path="+path+", "+err.Error(), http.StatusInternalServerError) } else { result.Data = info } @@ -139,7 +139,7 @@ func (fs SqlFileInfoStore) GetByPath(path string) StoreChannel { return storeChannel } -func (s SqlFileInfoStore) InvalidateFileInfosForPostCache(postId string) { +func (fs SqlFileInfoStore) InvalidateFileInfosForPostCache(postId string) { fileInfoCache.Remove(postId) } @@ -190,8 +190,8 @@ func (fs SqlFileInfoStore) GetForPost(postId string, readFromMaster bool, allowF AND DeleteAt = 0 ORDER BY CreateAt`, map[string]interface{}{"PostId": postId}); err != nil { - result.Err = model.NewLocAppError("SqlFileInfoStore.GetForPost", - "store.sql_file_info.get_for_post.app_error", nil, "post_id="+postId+", "+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.GetForPost", + "store.sql_file_info.get_for_post.app_error", nil, "post_id="+postId+", "+err.Error(), http.StatusInternalServerError) } else { if len(infos) > 0 { fileInfoCache.AddWithExpiresInSecs(postId, infos, FILE_INFO_CACHE_SEC) @@ -221,8 +221,8 @@ func (fs SqlFileInfoStore) AttachToPost(fileId, postId string) StoreChannel { WHERE Id = :Id AND PostId = ''`, map[string]interface{}{"PostId": postId, "Id": fileId}); err != nil { - result.Err = model.NewLocAppError("SqlFileInfoStore.AttachToPost", - "store.sql_file_info.attach_to_post.app_error", nil, "post_id="+postId+", file_id="+fileId+", err="+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.AttachToPost", + "store.sql_file_info.attach_to_post.app_error", nil, "post_id="+postId+", file_id="+fileId+", err="+err.Error(), http.StatusInternalServerError) } storeChannel <- result @@ -245,8 +245,8 @@ func (fs SqlFileInfoStore) DeleteForPost(postId string) StoreChannel { DeleteAt = :DeleteAt WHERE PostId = :PostId`, map[string]interface{}{"DeleteAt": model.GetMillis(), "PostId": postId}); err != nil { - result.Err = model.NewLocAppError("SqlFileInfoStore.DeleteForPost", - "store.sql_file_info.delete_for_post.app_error", nil, "post_id="+postId+", err="+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.DeleteForPost", + "store.sql_file_info.delete_for_post.app_error", nil, "post_id="+postId+", err="+err.Error(), http.StatusInternalServerError) } else { result.Data = postId } @@ -257,3 +257,25 @@ func (fs SqlFileInfoStore) DeleteForPost(postId string) StoreChannel { return storeChannel } + +func (fs SqlFileInfoStore) PermanentDelete(fileId string) StoreChannel { + storeChannel := make(StoreChannel, 1) + + go func() { + result := StoreResult{} + + if _, err := fs.GetMaster().Exec( + `DELETE FROM + FileInfo + WHERE + Id = :FileId`, map[string]interface{}{"FileId": fileId}); err != nil { + result.Err = model.NewAppError("SqlFileInfoStore.PermanentDelete", + "store.sql_file_info.permanent_delete.app_error", nil, "file_id="+fileId+", err="+err.Error(), http.StatusInternalServerError) + } + + storeChannel <- result + close(storeChannel) + }() + + return storeChannel +} diff --git a/store/sql_file_info_store_test.go b/store/sql_file_info_store_test.go index daec06269..fecd862c0 100644 --- a/store/sql_file_info_store_test.go +++ b/store/sql_file_info_store_test.go @@ -25,6 +25,9 @@ func TestFileInfoSaveGet(t *testing.T) { } else { info = returned } + defer func() { + <-store.FileInfo().PermanentDelete(info.Id) + }() if result := <-store.FileInfo().Get(info.Id); result.Err != nil { t.Fatal(result.Err) @@ -43,6 +46,9 @@ func TestFileInfoSaveGet(t *testing.T) { if result := <-store.FileInfo().Get(info2.Id); result.Err == nil { t.Fatal("shouldn't have gotten deleted file") } + defer func() { + <-store.FileInfo().PermanentDelete(info2.Id) + }() } func TestFileInfoSaveGetByPath(t *testing.T) { @@ -60,6 +66,9 @@ func TestFileInfoSaveGetByPath(t *testing.T) { } else { info = returned } + defer func() { + <-store.FileInfo().PermanentDelete(info.Id) + }() if result := <-store.FileInfo().GetByPath(info.Path); result.Err != nil { t.Fatal(result.Err) @@ -78,6 +87,9 @@ func TestFileInfoSaveGetByPath(t *testing.T) { if result := <-store.FileInfo().GetByPath(info2.Id); result.Err == nil { t.Fatal("shouldn't have gotten deleted file") } + defer func() { + <-store.FileInfo().PermanentDelete(info2.Id) + }() } func TestFileInfoGetForPost(t *testing.T) { @@ -112,6 +124,9 @@ func TestFileInfoGetForPost(t *testing.T) { for i, info := range infos { infos[i] = Must(store.FileInfo().Save(info)).(*model.FileInfo) + defer func(id string) { + <-store.FileInfo().PermanentDelete(id) + }(infos[i].Id) } if result := <-store.FileInfo().GetForPost(postId, true, false); result.Err != nil { @@ -143,6 +158,9 @@ func TestFileInfoAttachToPost(t *testing.T) { CreatorId: userId, Path: "file.txt", })).(*model.FileInfo) + defer func() { + <-store.FileInfo().PermanentDelete(info1.Id) + }() if len(info1.PostId) != 0 { t.Fatal("file shouldn't have a PostId") @@ -162,6 +180,9 @@ func TestFileInfoAttachToPost(t *testing.T) { CreatorId: userId, Path: "file.txt", })).(*model.FileInfo) + defer func() { + <-store.FileInfo().PermanentDelete(info2.Id) + }() if result := <-store.FileInfo().AttachToPost(info2.Id, postId); result.Err != nil { t.Fatal(result.Err) @@ -208,6 +229,9 @@ func TestFileInfoDeleteForPost(t *testing.T) { for i, info := range infos { infos[i] = Must(store.FileInfo().Save(info)).(*model.FileInfo) + defer func(id string) { + <-store.FileInfo().PermanentDelete(id) + }(infos[i].Id) } if result := <-store.FileInfo().DeleteForPost(postId); result.Err != nil { @@ -218,3 +242,17 @@ func TestFileInfoDeleteForPost(t *testing.T) { t.Fatal("shouldn't have returned any file infos") } } + +func TestFileInfoPermanentDelete(t *testing.T) { + Setup() + + info := Must(store.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Path: "file.txt", + })).(*model.FileInfo) + + if result := <-store.FileInfo().PermanentDelete(info.Id); result.Err != nil { + t.Fatal(result.Err) + } +} diff --git a/store/store.go b/store/store.go index e86b5f116..0aa4d5c41 100644 --- a/store/store.go +++ b/store/store.go @@ -386,6 +386,7 @@ type FileInfoStore interface { InvalidateFileInfosForPostCache(postId string) AttachToPost(fileId string, postId string) StoreChannel DeleteForPost(postId string) StoreChannel + PermanentDelete(fileId string) StoreChannel } type ReactionStore interface { diff --git a/utils/file.go b/utils/file.go new file mode 100644 index 000000000..efed1e954 --- /dev/null +++ b/utils/file.go @@ -0,0 +1,313 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + l4g "github.com/alecthomas/log4go" + s3 "github.com/minio/minio-go" + "github.com/minio/minio-go/pkg/credentials" + + "github.com/mattermost/platform/model" +) + +const ( + TEST_FILE_PATH = "/testfile" +) + +// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. +// If signV2 input is false, function always returns signature v4. +// +// Additionally this function also takes a user defined region, if set +// disables automatic region lookup. +func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) { + var creds *credentials.Credentials + if signV2 { + creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2) + } else { + creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4) + } + return s3.NewWithCredentials(endpoint, creds, secure, region) +} + +func TestFileConnection() *model.AppError { + if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + endpoint := Cfg.FileSettings.AmazonS3Endpoint + accessKey := Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *Cfg.FileSettings.AmazonS3SSL + signV2 := *Cfg.FileSettings.AmazonS3SignV2 + region := Cfg.FileSettings.AmazonS3Region + bucket := Cfg.FileSettings.AmazonS3Bucket + + s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) + if err != nil { + return model.NewLocAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error()) + } + + exists, err := s3Clnt.BucketExists(bucket) + if err != nil { + return model.NewLocAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error()) + } + + if !exists { + l4g.Warn("Bucket specified does not exist. Attempting to create...") + err := s3Clnt.MakeBucket(bucket, region) + if err != nil { + l4g.Error("Unable to create bucket.") + return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) + } + } + l4g.Info("Connection to S3 or minio is good. Bucket exists.") + } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + f := []byte("testingwrite") + if err := writeFileLocally(f, Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil { + return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError) + } + os.Remove(Cfg.FileSettings.Directory + TEST_FILE_PATH) + l4g.Info("Able to write files to local storage.") + } else { + return model.NewLocAppError("TestFileConnection", "No file driver selected.", nil, "") + } + + return nil +} + +func ReadFile(path string) ([]byte, *model.AppError) { + if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + endpoint := Cfg.FileSettings.AmazonS3Endpoint + accessKey := Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *Cfg.FileSettings.AmazonS3SSL + signV2 := *Cfg.FileSettings.AmazonS3SignV2 + region := Cfg.FileSettings.AmazonS3Region + s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) + if err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) + } + bucket := Cfg.FileSettings.AmazonS3Bucket + minioObject, err := s3Clnt.GetObject(bucket, path) + defer minioObject.Close() + if err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) + } + if f, err := ioutil.ReadAll(minioObject); err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) + } else { + return f, nil + } + } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + if f, err := ioutil.ReadFile(Cfg.FileSettings.Directory + path); err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error()) + } else { + return f, nil + } + } else { + return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented) + } +} + +func MoveFile(oldPath, newPath string) *model.AppError { + if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + endpoint := Cfg.FileSettings.AmazonS3Endpoint + accessKey := Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *Cfg.FileSettings.AmazonS3SSL + signV2 := *Cfg.FileSettings.AmazonS3SignV2 + region := Cfg.FileSettings.AmazonS3Region + encrypt := false + if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance { + encrypt = true + } + s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) + if err != nil { + return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error()) + } + bucket := Cfg.FileSettings.AmazonS3Bucket + + source := s3.NewSourceInfo(bucket, oldPath, nil) + destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt)) + if err != nil { + return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error()) + } + if err = s3Clnt.CopyObject(destination, source); err != nil { + return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) + } + if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil { + return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) + } + } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + if err := os.MkdirAll(filepath.Dir(Cfg.FileSettings.Directory+newPath), 0774); err != nil { + return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error()) + } + + if err := os.Rename(Cfg.FileSettings.Directory+oldPath, Cfg.FileSettings.Directory+newPath); err != nil { + return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error()) + } + } else { + return model.NewLocAppError("moveFile", "api.file.move_file.configured.app_error", nil, "") + } + + return nil +} + +func WriteFile(f []byte, path string) *model.AppError { + if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + endpoint := Cfg.FileSettings.AmazonS3Endpoint + accessKey := Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *Cfg.FileSettings.AmazonS3SSL + signV2 := *Cfg.FileSettings.AmazonS3SignV2 + region := Cfg.FileSettings.AmazonS3Region + encrypt := false + if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance { + encrypt = true + } + + s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) + if err != nil { + return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) + } + + bucket := Cfg.FileSettings.AmazonS3Bucket + ext := filepath.Ext(path) + metaData := S3Metadata(encrypt, "binary/octet-stream") + if model.IsFileExtImage(ext) { + metaData = S3Metadata(encrypt, model.GetImageMimeType(ext)) + } + + _, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil) + if err != nil { + return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) + } + } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + if err := writeFileLocally(f, Cfg.FileSettings.Directory+path); err != nil { + return err + } + } else { + return model.NewLocAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "") + } + + return nil +} + +func writeFileLocally(f []byte, path string) *model.AppError { + if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { + directory, _ := filepath.Abs(filepath.Dir(path)) + return model.NewLocAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error()) + } + + if err := ioutil.WriteFile(path, f, 0644); err != nil { + return model.NewLocAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error()) + } + + return nil +} + +func RemoveFile(path string) *model.AppError { + if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + endpoint := Cfg.FileSettings.AmazonS3Endpoint + accessKey := Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *Cfg.FileSettings.AmazonS3SSL + signV2 := *Cfg.FileSettings.AmazonS3SignV2 + region := Cfg.FileSettings.AmazonS3Region + + s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) + if err != nil { + return model.NewLocAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error()) + } + + bucket := Cfg.FileSettings.AmazonS3Bucket + if err := s3Clnt.RemoveObject(bucket, path); err != nil { + return model.NewLocAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error()) + } + } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + if err := os.Remove(Cfg.FileSettings.Directory + path); err != nil { + return model.NewLocAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error()) + } + } else { + return model.NewLocAppError("RemoveFile", "utils.file.remove_file.configured.app_error", nil, "") + } + + return nil +} + +func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { + out := make(chan string, 1) + + go func() { + defer close(out) + + for { + info, done := <-in + + if !done { + break + } + + out <- info.Key + } + }() + + return out +} + +func RemoveDirectory(path string) *model.AppError { + if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { + endpoint := Cfg.FileSettings.AmazonS3Endpoint + accessKey := Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *Cfg.FileSettings.AmazonS3SSL + signV2 := *Cfg.FileSettings.AmazonS3SignV2 + region := Cfg.FileSettings.AmazonS3Region + + s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) + if err != nil { + return model.NewLocAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error()) + } + + doneCh := make(chan struct{}) + + bucket := Cfg.FileSettings.AmazonS3Bucket + for err := range s3Clnt.RemoveObjects(bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(bucket, path, true, doneCh))) { + if err.Err != nil { + doneCh <- struct{}{} + return model.NewLocAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error()) + } + } + + close(doneCh) + } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { + if err := os.RemoveAll(Cfg.FileSettings.Directory + path); err != nil { + return model.NewLocAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error()) + } + } else { + return model.NewLocAppError("RemoveDirectory", "utils.file.remove_directory.configured.app_error", nil, "") + } + + return nil +} + +func S3Metadata(encrypt bool, contentType string) map[string][]string { + metaData := make(map[string][]string) + if contentType != "" { + metaData["Content-Type"] = []string{"contentType"} + } + if encrypt { + metaData["x-amz-server-side-encryption"] = []string{"AES256"} + } + return metaData +} + +func CopyMetadata(encrypt bool) map[string]string { + metaData := make(map[string]string) + metaData["x-amz-server-side-encryption"] = "AES256" + return metaData +} diff --git a/utils/file_test.go b/utils/file_test.go new file mode 100644 index 000000000..202a5354e --- /dev/null +++ b/utils/file_test.go @@ -0,0 +1,122 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "testing" + + "github.com/mattermost/platform/model" +) + +func TestReadWriteFile(t *testing.T) { + TranslationsPreInit() + LoadConfig("config.json") + InitTranslations(Cfg.LocalizationSettings) + + b := []byte("test") + path := "tests/" + model.NewId() + + if err := WriteFile(b, path); err != nil { + t.Fatal(err) + } + defer RemoveFile(path) + + if read, err := ReadFile(path); err != nil { + t.Fatal(err) + } else if readString := string(read); readString != "test" { + t.Fatal("should've read back contents of file") + } +} + +func TestMoveFile(t *testing.T) { + TranslationsPreInit() + LoadConfig("config.json") + InitTranslations(Cfg.LocalizationSettings) + + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/" + model.NewId() + + if err := WriteFile(b, path1); err != nil { + t.Fatal(err) + } + defer RemoveFile(path1) + + if err := MoveFile(path1, path2); err != nil { + t.Fatal(err) + } + defer RemoveFile(path2) + + if _, err := ReadFile(path1); err == nil { + t.Fatal("file should no longer exist at old path") + } + + if _, err := ReadFile(path2); err != nil { + t.Fatal("file should exist at new path", err) + } +} + +func TestRemoveFile(t *testing.T) { + TranslationsPreInit() + LoadConfig("config.json") + InitTranslations(Cfg.LocalizationSettings) + + b := []byte("test") + path := "tests/" + model.NewId() + + if err := WriteFile(b, path); err != nil { + t.Fatal(err) + } + + if err := RemoveFile(path); err != nil { + t.Fatal(err) + } + + if _, err := ReadFile(path); err == nil { + t.Fatal("should've removed file") + } + + if err := WriteFile(b, "tests2/foo"); err != nil { + t.Fatal(err) + } + if err := WriteFile(b, "tests2/bar"); err != nil { + t.Fatal(err) + } + if err := WriteFile(b, "tests2/asdf"); err != nil { + t.Fatal(err) + } + if err := RemoveDirectory("tests2"); err != nil { + t.Fatal(err) + } +} + +func TestRemoveDirectory(t *testing.T) { + TranslationsPreInit() + LoadConfig("config.json") + InitTranslations(Cfg.LocalizationSettings) + + b := []byte("test") + + if err := WriteFile(b, "tests2/foo"); err != nil { + t.Fatal(err) + } + if err := WriteFile(b, "tests2/bar"); err != nil { + t.Fatal(err) + } + if err := WriteFile(b, "tests2/aaa"); err != nil { + t.Fatal(err) + } + + if err := RemoveDirectory("tests2"); err != nil { + t.Fatal(err) + } + + if _, err := ReadFile("tests2/foo"); err == nil { + t.Fatal("should've removed file") + } else if _, err := ReadFile("tests2/bar"); err == nil { + t.Fatal("should've removed file") + } else if _, err := ReadFile("tests2/asdf"); err == nil { + t.Fatal("should've removed file") + } +} diff --git a/utils/time.go b/utils/time.go index 7d5afdf8f..baddd7329 100644 --- a/utils/time.go +++ b/utils/time.go @@ -8,6 +8,10 @@ func MillisFromTime(t time.Time) int64 { return t.UnixNano() / int64(time.Millisecond) } +func TimeFromMillis(millis int64) time.Time { + return time.Unix(0, millis*int64(time.Millisecond)) +} + func StartOfDay(t time.Time) time.Time { year, month, day := t.Date() return time.Date(year, month, day, 0, 0, 0, 0, t.Location()) |