diff options
Diffstat (limited to 'app')
-rw-r--r-- | app/emoji.go | 10 | ||||
-rw-r--r-- | app/file.go | 257 | ||||
-rw-r--r-- | app/file_test.go | 55 | ||||
-rw-r--r-- | app/import.go | 6 | ||||
-rw-r--r-- | app/slackimport.go | 3 | ||||
-rw-r--r-- | app/user.go | 6 |
6 files changed, 81 insertions, 256 deletions
diff --git a/app/emoji.go b/app/emoji.go index f1362b798..e01ca97ae 100644 --- a/app/emoji.go +++ b/app/emoji.go @@ -101,7 +101,7 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro if err := gif.EncodeAll(newbuf, resized_gif); err != nil { return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.gif_encode_error", nil, "", http.StatusBadRequest) } - if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { + if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { return err } } @@ -113,13 +113,13 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro if err := png.Encode(newbuf, resized_image); err != nil { return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.encode_error", nil, "", http.StatusBadRequest) } - if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { + if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { return err } } } } else { - if err := WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil { + if err := utils.WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil { return err } } @@ -159,7 +159,7 @@ func GetEmojiImage(emojiId string) (imageByte []byte, imageType string, err *mod } else { var img []byte - if data, err := ReadFile(getEmojiImagePath(emojiId)); err != nil { + if data, err := utils.ReadFile(getEmojiImagePath(emojiId)); err != nil { return nil, "", model.NewAppError("getEmojiImage", "api.emoji.get_image.read.app_error", nil, err.Error(), http.StatusNotFound) } else { img = data @@ -219,7 +219,7 @@ func imageToPaletted(img image.Image) *image.Paletted { } func deleteEmojiImage(id string) { - if err := MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil { + if err := utils.MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil { l4g.Error("Failed to rename image when deleting emoji %v", id) } } diff --git a/app/file.go b/app/file.go index dc7caff41..10fb1425c 100644 --- a/app/file.go +++ b/app/file.go @@ -14,23 +14,21 @@ import ( _ "image/gif" "image/jpeg" "io" - "io/ioutil" "mime/multipart" "net/http" "net/url" - "os" "path/filepath" "strings" "sync" + "time" l4g "github.com/alecthomas/log4go" "github.com/disintegration/imaging" - "github.com/mattermost/platform/model" - "github.com/mattermost/platform/utils" - s3 "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/credentials" "github.com/rwcarlsen/goexif/exif" _ "golang.org/x/image/bmp" + + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" ) const ( @@ -57,222 +55,8 @@ const ( IMAGE_THUMBNAIL_PIXEL_WIDTH = 120 IMAGE_THUMBNAIL_PIXEL_HEIGHT = 100 IMAGE_PREVIEW_PIXEL_WIDTH = 1024 - - TEST_FILE_PATH = "/testfile" ) -// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. -// If signV2 input is false, function always returns signature v4. -// -// Additionally this function also takes a user defined region, if set -// disables automatic region lookup. -func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) { - var creds *credentials.Credentials - if signV2 { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2) - } else { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4) - } - return s3.NewWithCredentials(endpoint, creds, secure, region) -} - -func TestFileConnection() *model.AppError { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint - accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *utils.Cfg.FileSettings.AmazonS3SSL - signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2 - region := utils.Cfg.FileSettings.AmazonS3Region - bucket := utils.Cfg.FileSettings.AmazonS3Bucket - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewLocAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error()) - } - - exists, err := s3Clnt.BucketExists(bucket) - if err != nil { - return model.NewLocAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error()) - } - - if !exists { - l4g.Warn("Bucket specified does not exist. Attempting to create...") - err := s3Clnt.MakeBucket(bucket, region) - if err != nil { - l4g.Error("Unable to create bucket.") - return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) - } - } - l4g.Info("Connection to S3 or minio is good. Bucket exists.") - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - f := []byte("testingwrite") - if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil { - return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError) - } - os.Remove(utils.Cfg.FileSettings.Directory + TEST_FILE_PATH) - l4g.Info("Able to write files to local storage.") - } else { - return model.NewLocAppError("TestFileConnection", "No file driver selected.", nil, "") - } - - return nil -} - -func ReadFile(path string) ([]byte, *model.AppError) { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint - accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *utils.Cfg.FileSettings.AmazonS3SSL - signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2 - region := utils.Cfg.FileSettings.AmazonS3Region - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) - } - bucket := utils.Cfg.FileSettings.AmazonS3Bucket - minioObject, err := s3Clnt.GetObject(bucket, path) - defer minioObject.Close() - if err != nil { - return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) - } - if f, err := ioutil.ReadAll(minioObject); err != nil { - return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) - } else { - return f, nil - } - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewLocAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error()) - } else { - return f, nil - } - } else { - return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented) - } -} - -func MoveFile(oldPath, newPath string) *model.AppError { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint - accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *utils.Cfg.FileSettings.AmazonS3SSL - signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2 - region := utils.Cfg.FileSettings.AmazonS3Region - encrypt := false - if *utils.Cfg.FileSettings.AmazonS3SSE && utils.IsLicensed() && *utils.License().Features.Compliance { - encrypt = true - } - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error()) - } - bucket := utils.Cfg.FileSettings.AmazonS3Bucket - - source := s3.NewSourceInfo(bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt)) - if err != nil { - return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error()) - } - if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) - } - if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil { - return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) - } - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+newPath), 0774); err != nil { - return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error()) - } - - if err := os.Rename(utils.Cfg.FileSettings.Directory+oldPath, utils.Cfg.FileSettings.Directory+newPath); err != nil { - return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error()) - } - } else { - return model.NewLocAppError("moveFile", "api.file.move_file.configured.app_error", nil, "") - } - - return nil -} - -func WriteFile(f []byte, path string) *model.AppError { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint - accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *utils.Cfg.FileSettings.AmazonS3SSL - signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2 - region := utils.Cfg.FileSettings.AmazonS3Region - encrypt := false - if *utils.Cfg.FileSettings.AmazonS3SSE && utils.IsLicensed() && *utils.License().Features.Compliance { - encrypt = true - } - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) - } - - bucket := utils.Cfg.FileSettings.AmazonS3Bucket - ext := filepath.Ext(path) - metaData := S3Metadata(encrypt, "binary/octet-stream") - if model.IsFileExtImage(ext) { - metaData = S3Metadata(encrypt, model.GetImageMimeType(ext)) - } - - _, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil) - if err != nil { - return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) - } - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil { - return err - } - } else { - return model.NewLocAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "") - } - - return nil -} - -func writeFileLocally(f []byte, path string) *model.AppError { - if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { - directory, _ := filepath.Abs(filepath.Dir(path)) - return model.NewLocAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error()) - } - - if err := ioutil.WriteFile(path, f, 0644); err != nil { - return model.NewLocAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error()) - } - - return nil -} - -func openFileWriteStream(path string) (io.Writer, *model.AppError) { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.s3.app_error", nil, "") - } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+path), 0774); err != nil { - return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.creating_dir.app_error", nil, err.Error()) - } - - if fileHandle, err := os.Create(utils.Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.local_server.app_error", nil, err.Error()) - } else { - fileHandle.Chmod(0644) - return fileHandle, nil - } - } - - return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.configured.app_error", nil, "") -} - -func closeFileWriteStream(file io.Writer) { - file.(*os.File).Close() -} - func GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo { // Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension} split := strings.SplitN(filename, "/", 5) @@ -295,7 +79,7 @@ func GetInfoForFilename(post *model.Post, teamId string, filename string) *model // Open the file and populate the fields of the FileInfo var info *model.FileInfo - if data, err := ReadFile(path); err != nil { + if data, err := utils.ReadFile(path); err != nil { l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.file_not_found.error"), post.Id, filename, path, err) return nil } else { @@ -337,7 +121,7 @@ func FindTeamIdForFilename(post *model.Post, filename string) string { } else { for _, team := range teams { path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name) - if _, err := ReadFile(path); err == nil { + if _, err := utils.ReadFile(path); err == nil { // Found the team that this file was posted from return team.Id } @@ -484,7 +268,7 @@ func UploadFiles(teamId string, channelId string, userId string, fileHeaders []* io.Copy(buf, file) data := buf.Bytes() - info, err := DoUploadFile(teamId, channelId, userId, fileHeader.Filename, data) + info, err := DoUploadFile(time.Now(), teamId, channelId, userId, fileHeader.Filename, data) if err != nil { return nil, err } @@ -507,7 +291,7 @@ func UploadFiles(teamId string, channelId string, userId string, fileHeaders []* return resStruct, nil } -func DoUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { +func DoUploadFile(now time.Time, teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { filename := filepath.Base(rawFilename) info, err := model.GetInfoForBytes(filename, data) @@ -519,7 +303,7 @@ func DoUploadFile(teamId string, channelId string, userId string, rawFilename st info.Id = model.NewId() info.CreatorId = userId - pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/" + pathPrefix := now.Format("20060102") + "/teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/" info.Path = pathPrefix + filename if info.IsImage() { @@ -535,7 +319,7 @@ func DoUploadFile(teamId string, channelId string, userId string, rawFilename st info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" } - if err := WriteFile(data, info.Path); err != nil { + if err := utils.WriteFile(data, info.Path); err != nil { return nil, err } @@ -652,7 +436,7 @@ func generateThumbnailImage(img image.Image, thumbnailPath string, width int, he return } - if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil { + if err := utils.WriteFile(buf.Bytes(), thumbnailPath); err != nil { l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err) return } @@ -674,7 +458,7 @@ func generatePreviewImage(img image.Image, previewPath string, width int) { return } - if err := WriteFile(buf.Bytes(), previewPath); err != nil { + if err := utils.WriteFile(buf.Bytes(), previewPath); err != nil { l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err) return } @@ -687,20 +471,3 @@ func GetFileInfo(fileId string) (*model.FileInfo, *model.AppError) { return result.Data.(*model.FileInfo), nil } } - -func S3Metadata(encrypt bool, contentType string) map[string][]string { - metaData := make(map[string][]string) - if contentType != "" { - metaData["Content-Type"] = []string{"contentType"} - } - if encrypt { - metaData["x-amz-server-side-encryption"] = []string{"AES256"} - } - return metaData -} - -func CopyMetadata(encrypt bool) map[string]string { - metaData := make(map[string]string) - metaData["x-amz-server-side-encryption"] = "AES256" - return metaData -} diff --git a/app/file_test.go b/app/file_test.go index 683b574b8..962661039 100644 --- a/app/file_test.go +++ b/app/file_test.go @@ -4,9 +4,12 @@ package app import ( + "fmt" "testing" + "time" "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" ) func TestGeneratePublicLinkHash(t *testing.T) { @@ -31,3 +34,55 @@ func TestGeneratePublicLinkHash(t *testing.T) { t.Fatal("hashes for the same file with different salts should not be equal") } } + +func TestDoUploadFile(t *testing.T) { + Setup() + + teamId := model.NewId() + channelId := model.NewId() + userId := model.NewId() + filename := "test" + data := []byte("abcd") + + info1, err := DoUploadFile(time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data) + if err != nil { + t.Fatal(err) + } else { + defer func() { + <-Srv.Store.FileInfo().PermanentDelete(info1.Id) + utils.RemoveFile(info1.Path) + }() + } + + if info1.Path != fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info1.Id, filename) { + t.Fatal("stored file at incorrect path", info1.Path) + } + + info2, err := DoUploadFile(time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data) + if err != nil { + t.Fatal(err) + } else { + defer func() { + <-Srv.Store.FileInfo().PermanentDelete(info2.Id) + utils.RemoveFile(info2.Path) + }() + } + + if info2.Path != fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info2.Id, filename) { + t.Fatal("stored file at incorrect path", info2.Path) + } + + info3, err := DoUploadFile(time.Date(2008, 3, 5, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data) + if err != nil { + t.Fatal(err) + } else { + defer func() { + <-Srv.Store.FileInfo().PermanentDelete(info3.Id) + utils.RemoveFile(info3.Path) + }() + } + + if info3.Path != fmt.Sprintf("20080305/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info3.Id, filename) { + t.Fatal("stored file at incorrect path", info3.Path) + } +} diff --git a/app/import.go b/app/import.go index fb7d43cdf..d404dbadd 100644 --- a/app/import.go +++ b/app/import.go @@ -12,9 +12,11 @@ import ( "regexp" "strings" "sync" + "time" "unicode/utf8" l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/model" "github.com/mattermost/platform/store" "github.com/mattermost/platform/utils" @@ -1486,12 +1488,12 @@ func OldImportChannel(channel *model.Channel) *model.Channel { } } -func OldImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { +func OldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { buf := bytes.NewBuffer(nil) io.Copy(buf, file) data := buf.Bytes() - fileInfo, err := DoUploadFile(teamId, channelId, userId, fileName, data) + fileInfo, err := DoUploadFile(timestamp, teamId, channelId, userId, fileName, data) if err != nil { return nil, err } diff --git a/app/slackimport.go b/app/slackimport.go index 4470b8323..e57a3a3d1 100644 --- a/app/slackimport.go +++ b/app/slackimport.go @@ -401,7 +401,8 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin } defer openFile.Close() - uploadedFile, err := OldImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name)) + timestamp := utils.TimeFromMillis(SlackConvertTimeStamp(sPost.TimeStamp)) + uploadedFile, err := OldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name)) if err != nil { l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) return nil, false diff --git a/app/user.go b/app/user.go index 813421a5c..40e32c282 100644 --- a/app/user.go +++ b/app/user.go @@ -758,7 +758,7 @@ func GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) { } else { path := "users/" + user.Id + "/profile.png" - if data, err := ReadFile(path); err != nil { + if data, err := utils.ReadFile(path); err != nil { readFailed = true if img, err = CreateProfileImage(user.Username, user.Id); err != nil { @@ -766,7 +766,7 @@ func GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) { } if user.LastPictureUpdate == 0 { - if err := WriteFile(img, path); err != nil { + if err := utils.WriteFile(img, path); err != nil { return nil, false, err } } @@ -819,7 +819,7 @@ func SetProfileImage(userId string, imageData *multipart.FileHeader) *model.AppE path := "users/" + userId + "/profile.png" - if err := WriteFile(buf.Bytes(), path); err != nil { + if err := utils.WriteFile(buf.Bytes(), path); err != nil { return model.NewLocAppError("SetProfileImage", "api.user.upload_profile_user.upload_profile.app_error", nil, "") } |