diff options
Diffstat (limited to 'utils')
-rw-r--r-- | utils/file.go | 359 | ||||
-rw-r--r-- | utils/file_backend.go | 44 | ||||
-rw-r--r-- | utils/file_backend_local.go | 98 | ||||
-rw-r--r-- | utils/file_backend_s3.go | 226 | ||||
-rw-r--r-- | utils/file_backend_test.go | 164 | ||||
-rw-r--r-- | utils/file_test.go | 172 |
6 files changed, 532 insertions, 531 deletions
diff --git a/utils/file.go b/utils/file.go index 6472770a0..13b25bdab 100644 --- a/utils/file.go +++ b/utils/file.go @@ -4,372 +4,13 @@ package utils import ( - "bytes" "fmt" "io" "io/ioutil" - "net/http" "os" "path/filepath" - "strings" - - l4g "github.com/alecthomas/log4go" - s3 "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/credentials" - - "github.com/mattermost/mattermost-server/model" -) - -const ( - TEST_FILE_PATH = "/testfile" ) -// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. -// If signV2 input is false, function always returns signature v4. -// -// Additionally this function also takes a user defined region, if set -// disables automatic region lookup. -func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) { - var creds *credentials.Credentials - if signV2 { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2) - } else { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4) - } - - s3Clnt, err := s3.NewWithCredentials(endpoint, creds, secure, region) - if err != nil { - return nil, err - } - - if *Cfg.FileSettings.AmazonS3Trace { - s3Clnt.TraceOn(os.Stdout) - } - - return s3Clnt, nil -} - -func TestFileConnection() *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - bucket := Cfg.FileSettings.AmazonS3Bucket - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error(), http.StatusInternalServerError) - } - - exists, err := s3Clnt.BucketExists(bucket) - if err != nil { - return model.NewAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error(), http.StatusInternalServerError) - } - - if !exists { - l4g.Warn("Bucket specified does not exist. Attempting to create...") - err := s3Clnt.MakeBucket(bucket, region) - if err != nil { - l4g.Error("Unable to create bucket.") - return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) - } - } - l4g.Info("Connection to S3 or minio is good. Bucket exists.") - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - f := []byte("testingwrite") - if err := writeFileLocally(f, Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil { - return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError) - } - os.Remove(Cfg.FileSettings.Directory + TEST_FILE_PATH) - l4g.Info("Able to write files to local storage.") - } else { - return model.NewAppError("TestFileConnection", "No file driver selected.", nil, "", http.StatusInternalServerError) - } - - return nil -} - -func ReadFile(path string) ([]byte, *model.AppError) { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - bucket := Cfg.FileSettings.AmazonS3Bucket - minioObject, err := s3Clnt.GetObject(bucket, path) - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - defer minioObject.Close() - if f, err := ioutil.ReadAll(minioObject); err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if f, err := ioutil.ReadFile(Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } - } else { - return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented) - } -} - -func MoveFile(oldPath, newPath string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - encrypt := false - if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance { - encrypt = true - } - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - bucket := Cfg.FileSettings.AmazonS3Bucket - - source := s3.NewSourceInfo(bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt)) - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.MkdirAll(filepath.Dir(Cfg.FileSettings.Directory+newPath), 0774); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if err := os.Rename(Cfg.FileSettings.Directory+oldPath, Cfg.FileSettings.Directory+newPath); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - return model.NewAppError("moveFile", "api.file.move_file.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func WriteFile(f []byte, path string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - encrypt := false - if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance { - encrypt = true - } - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - bucket := Cfg.FileSettings.AmazonS3Bucket - ext := filepath.Ext(path) - metaData := S3Metadata(encrypt, "binary/octet-stream") - if model.IsFileExtImage(ext) { - metaData = S3Metadata(encrypt, model.GetImageMimeType(ext)) - } - - _, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil) - if err != nil { - return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := writeFileLocally(f, Cfg.FileSettings.Directory+path); err != nil { - return err - } - } else { - return model.NewAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func writeFileLocally(f []byte, path string) *model.AppError { - if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { - directory, _ := filepath.Abs(filepath.Dir(path)) - return model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError) - } - - if err := ioutil.WriteFile(path, f, 0644); err != nil { - return model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - return nil -} - -func RemoveFile(path string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - bucket := Cfg.FileSettings.AmazonS3Bucket - if err := s3Clnt.RemoveObject(bucket, path); err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.Remove(Cfg.FileSettings.Directory + path); err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - return model.NewAppError("RemoveFile", "utils.file.remove_file.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { - out := make(chan string, 1) - - go func() { - defer close(out) - - for { - info, done := <-in - - if !done { - break - } - - out <- info.Key - } - }() - - return out -} - -// Returns a list of all the directories within the path directory provided. -func ListDirectory(path string) (*[]string, *model.AppError) { - var paths []string - - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - defer close(doneCh) - - bucket := Cfg.FileSettings.AmazonS3Bucket - for object := range s3Clnt.ListObjects(bucket, path, false, doneCh) { - if object.Err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) - } - paths = append(paths, strings.Trim(object.Key, "/")) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if fileInfos, err := ioutil.ReadDir(Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - for _, fileInfo := range fileInfos { - if fileInfo.IsDir() { - paths = append(paths, filepath.Join(path, fileInfo.Name())) - } - } - } - } else { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.configured.app_error", nil, "", http.StatusInternalServerError) - } - - return &paths, nil -} - -func RemoveDirectory(path string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - bucket := Cfg.FileSettings.AmazonS3Bucket - for err := range s3Clnt.RemoveObjects(bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(bucket, path, true, doneCh))) { - if err.Err != nil { - doneCh <- struct{}{} - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) - } - } - - close(doneCh) - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.RemoveAll(Cfg.FileSettings.Directory + path); err != nil { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func S3Metadata(encrypt bool, contentType string) map[string][]string { - metaData := make(map[string][]string) - if contentType != "" { - metaData["Content-Type"] = []string{"contentType"} - } - if encrypt { - metaData["x-amz-server-side-encryption"] = []string{"AES256"} - } - return metaData -} - -func CopyMetadata(encrypt bool) map[string]string { - metaData := make(map[string]string) - metaData["x-amz-server-side-encryption"] = "AES256" - return metaData -} - // CopyFile will copy a file from src path to dst path. // Overwrites any existing files at dst. // Permissions are copied from file at src to the new file at dst. diff --git a/utils/file_backend.go b/utils/file_backend.go new file mode 100644 index 000000000..3469a63fb --- /dev/null +++ b/utils/file_backend.go @@ -0,0 +1,44 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "net/http" + + "github.com/mattermost/mattermost-server/model" +) + +type FileBackend interface { + TestConnection() *model.AppError + + ReadFile(path string) ([]byte, *model.AppError) + MoveFile(oldPath, newPath string) *model.AppError + WriteFile(f []byte, path string) *model.AppError + RemoveFile(path string) *model.AppError + + ListDirectory(path string) (*[]string, *model.AppError) + RemoveDirectory(path string) *model.AppError +} + +func NewFileBackend(settings *model.FileSettings) (FileBackend, *model.AppError) { + switch *settings.DriverName { + case model.IMAGE_DRIVER_S3: + return &S3FileBackend{ + endpoint: settings.AmazonS3Endpoint, + accessKey: settings.AmazonS3AccessKeyId, + secretKey: settings.AmazonS3SecretAccessKey, + secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL, + signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2, + region: settings.AmazonS3Region, + bucket: settings.AmazonS3Bucket, + encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance, + trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace, + }, nil + case model.IMAGE_DRIVER_LOCAL: + return &LocalFileBackend{ + directory: settings.Directory, + }, nil + } + return nil, model.NewAppError("NewFileBackend", "No file driver selected.", nil, "", http.StatusInternalServerError) +} diff --git a/utils/file_backend_local.go b/utils/file_backend_local.go new file mode 100644 index 000000000..b5e67f8f0 --- /dev/null +++ b/utils/file_backend_local.go @@ -0,0 +1,98 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "io/ioutil" + "net/http" + "os" + "path/filepath" + + l4g "github.com/alecthomas/log4go" + + "github.com/mattermost/mattermost-server/model" +) + +const ( + TEST_FILE_PATH = "/testfile" +) + +type LocalFileBackend struct { + directory string +} + +func (b *LocalFileBackend) TestConnection() *model.AppError { + f := []byte("testingwrite") + if err := writeFileLocally(f, filepath.Join(b.directory, TEST_FILE_PATH)); err != nil { + return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError) + } + os.Remove(filepath.Join(b.directory, TEST_FILE_PATH)) + l4g.Info("Able to write files to local storage.") + return nil +} + +func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) { + if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *LocalFileBackend) MoveFile(oldPath, newPath string) *model.AppError { + if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0774); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func (b *LocalFileBackend) WriteFile(f []byte, path string) *model.AppError { + return writeFileLocally(f, filepath.Join(b.directory, path)) +} + +func writeFileLocally(f []byte, path string) *model.AppError { + if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { + directory, _ := filepath.Abs(filepath.Dir(path)) + return model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError) + } + + if err := ioutil.WriteFile(path, f, 0644); err != nil { + return model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func (b *LocalFileBackend) RemoveFile(path string) *model.AppError { + if err := os.Remove(filepath.Join(b.directory, path)); err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *LocalFileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + var paths []string + if fileInfos, err := ioutil.ReadDir(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + for _, fileInfo := range fileInfos { + if fileInfo.IsDir() { + paths = append(paths, filepath.Join(path, fileInfo.Name())) + } + } + } + return &paths, nil +} + +func (b *LocalFileBackend) RemoveDirectory(path string) *model.AppError { + if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil { + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} diff --git a/utils/file_backend_s3.go b/utils/file_backend_s3.go new file mode 100644 index 000000000..ed88dc70c --- /dev/null +++ b/utils/file_backend_s3.go @@ -0,0 +1,226 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + l4g "github.com/alecthomas/log4go" + s3 "github.com/minio/minio-go" + "github.com/minio/minio-go/pkg/credentials" + + "github.com/mattermost/mattermost-server/model" +) + +type S3FileBackend struct { + endpoint string + accessKey string + secretKey string + secure bool + signV2 bool + region string + bucket string + encrypt bool + trace bool +} + +// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. +// If signV2 input is false, function always returns signature v4. +// +// Additionally this function also takes a user defined region, if set +// disables automatic region lookup. +func (b *S3FileBackend) s3New() (*s3.Client, error) { + var creds *credentials.Credentials + if b.signV2 { + creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2) + } else { + creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4) + } + + s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region) + if err != nil { + return nil, err + } + + if b.trace { + s3Clnt.TraceOn(os.Stdout) + } + + return s3Clnt, nil +} + +func (b *S3FileBackend) TestConnection() *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error(), http.StatusInternalServerError) + } + + exists, err := s3Clnt.BucketExists(b.bucket) + if err != nil { + return model.NewAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error(), http.StatusInternalServerError) + } + + if !exists { + l4g.Warn("Bucket specified does not exist. Attempting to create...") + err := s3Clnt.MakeBucket(b.bucket, b.region) + if err != nil { + l4g.Error("Unable to create bucket.") + return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) + } + } + l4g.Info("Connection to S3 or minio is good. Bucket exists.") + return nil +} + +func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + minioObject, err := s3Clnt.GetObject(b.bucket, path) + if err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + defer minioObject.Close() + if f, err := ioutil.ReadAll(minioObject); err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + source := s3.NewSourceInfo(b.bucket, oldPath, nil) + destination, err := s3.NewDestinationInfo(b.bucket, newPath, nil, s3CopyMetadata(b.encrypt)) + if err != nil { + return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.CopyObject(destination, source); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *S3FileBackend) WriteFile(f []byte, path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + ext := filepath.Ext(path) + metaData := s3Metadata(b.encrypt, "binary/octet-stream") + if model.IsFileExtImage(ext) { + metaData = s3Metadata(b.encrypt, model.GetImageMimeType(ext)) + } + + if _, err = s3Clnt.PutObjectWithMetadata(b.bucket, path, bytes.NewReader(f), metaData, nil); err != nil { + return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func (b *S3FileBackend) RemoveFile(path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { + out := make(chan string, 1) + + go func() { + defer close(out) + + for { + info, done := <-in + + if !done { + break + } + + out <- info.Key + } + }() + + return out +} + +func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + var paths []string + + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + doneCh := make(chan struct{}) + + defer close(doneCh) + + for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { + if object.Err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) + } + paths = append(paths, strings.Trim(object.Key, "/")) + } + + return &paths, nil +} + +func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + doneCh := make(chan struct{}) + + for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { + if err.Err != nil { + doneCh <- struct{}{} + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) + } + } + + close(doneCh) + return nil +} + +func s3Metadata(encrypt bool, contentType string) map[string][]string { + metaData := make(map[string][]string) + if contentType != "" { + metaData["Content-Type"] = []string{"contentType"} + } + if encrypt { + metaData["x-amz-server-side-encryption"] = []string{"AES256"} + } + return metaData +} + +func s3CopyMetadata(encrypt bool) map[string]string { + metaData := make(map[string]string) + metaData["x-amz-server-side-encryption"] = "AES256" + return metaData +} diff --git a/utils/file_backend_test.go b/utils/file_backend_test.go new file mode 100644 index 000000000..0989f783c --- /dev/null +++ b/utils/file_backend_test.go @@ -0,0 +1,164 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/mattermost/mattermost-server/model" +) + +type FileBackendTestSuite struct { + suite.Suite + + settings model.FileSettings + backend FileBackend +} + +func TestLocalFileBackendTestSuite(t *testing.T) { + dir, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(dir) + + suite.Run(t, &FileBackendTestSuite{ + settings: model.FileSettings{ + DriverName: model.NewString(model.IMAGE_DRIVER_LOCAL), + Directory: dir, + }, + }) +} + +func TestS3FileBackendTestSuite(t *testing.T) { + s3Host := os.Getenv("CI_HOST") + if s3Host == "" { + s3Host = "dockerhost" + } + + s3Port := os.Getenv("CI_MINIO_PORT") + if s3Port == "" { + s3Port = "9001" + } + + s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) + + suite.Run(t, &FileBackendTestSuite{ + settings: model.FileSettings{ + DriverName: model.NewString(model.IMAGE_DRIVER_S3), + AmazonS3AccessKeyId: "minioaccesskey", + AmazonS3SecretAccessKey: "miniosecretkey", + AmazonS3Bucket: "mattermost-test", + AmazonS3Endpoint: s3Endpoint, + AmazonS3SSL: model.NewBool(false), + }, + }) +} + +func (s *FileBackendTestSuite) SetupTest() { + TranslationsPreInit() + + backend, err := NewFileBackend(&s.settings) + require.Nil(s.T(), err) + s.backend = backend +} + +func (s *FileBackendTestSuite) TestConnection() { + s.Nil(s.backend.TestConnection()) +} + +func (s *FileBackendTestSuite) TestReadWriteFile() { + b := []byte("test") + path := "tests/" + model.NewId() + + s.Nil(s.backend.WriteFile(b, path)) + defer s.backend.RemoveFile(path) + + read, err := s.backend.ReadFile(path) + s.Nil(err) + + readString := string(read) + s.EqualValues(readString, "test") +} + +func (s *FileBackendTestSuite) TestMoveFile() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/" + model.NewId() + + s.Nil(s.backend.WriteFile(b, path1)) + defer s.backend.RemoveFile(path1) + + s.Nil(s.backend.MoveFile(path1, path2)) + defer s.backend.RemoveFile(path2) + + _, err := s.backend.ReadFile(path1) + s.Error(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + +func (s *FileBackendTestSuite) TestRemoveFile() { + b := []byte("test") + path := "tests/" + model.NewId() + + s.Nil(s.backend.WriteFile(b, path)) + s.Nil(s.backend.RemoveFile(path)) + + _, err := s.backend.ReadFile(path) + s.Error(err) + + s.Nil(s.backend.WriteFile(b, "tests2/foo")) + s.Nil(s.backend.WriteFile(b, "tests2/bar")) + s.Nil(s.backend.WriteFile(b, "tests2/asdf")) + s.Nil(s.backend.RemoveDirectory("tests2")) +} + +func (s *FileBackendTestSuite) TestListDirectory() { + b := []byte("test") + path1 := "19700101/" + model.NewId() + path2 := "19800101/" + model.NewId() + + s.Nil(s.backend.WriteFile(b, path1)) + defer s.backend.RemoveFile(path1) + s.Nil(s.backend.WriteFile(b, path2)) + defer s.backend.RemoveFile(path2) + + paths, err := s.backend.ListDirectory("") + s.Nil(err) + + found1 := false + found2 := false + for _, path := range *paths { + if path == "19700101" { + found1 = true + } else if path == "19800101" { + found2 = true + } + } + s.True(found1) + s.True(found2) +} + +func (s *FileBackendTestSuite) TestRemoveDirectory() { + b := []byte("test") + + s.Nil(s.backend.WriteFile(b, "tests2/foo")) + s.Nil(s.backend.WriteFile(b, "tests2/bar")) + s.Nil(s.backend.WriteFile(b, "tests2/aaa")) + + s.Nil(s.backend.RemoveDirectory("tests2")) + + _, err := s.backend.ReadFile("tests2/foo") + s.Error(err) + _, err = s.backend.ReadFile("tests2/bar") + s.Error(err) + _, err = s.backend.ReadFile("tests2/asdf") + s.Error(err) +} diff --git a/utils/file_test.go b/utils/file_test.go index 91e78f24e..6c7e3c462 100644 --- a/utils/file_test.go +++ b/utils/file_test.go @@ -4,7 +4,6 @@ package utils import ( - "fmt" "io/ioutil" "os" "path/filepath" @@ -12,179 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - "github.com/mattermost/mattermost-server/model" ) -type FileTestSuite struct { - suite.Suite - - testDriver string - - // Config to be reset after tests. - driverName string - amazonS3AccessKeyId string - amazonS3SecretAccessKey string - amazonS3Bucket string - amazonS3Endpoint string - amazonS3SSL bool -} - -func TestFileLocalTestSuite(t *testing.T) { - testsuite := FileTestSuite{ - testDriver: model.IMAGE_DRIVER_LOCAL, - } - suite.Run(t, &testsuite) -} - -func TestFileMinioTestSuite(t *testing.T) { - testsuite := FileTestSuite{ - testDriver: model.IMAGE_DRIVER_S3, - } - suite.Run(t, &testsuite) -} - -func (s *FileTestSuite) SetupTest() { - TranslationsPreInit() - LoadGlobalConfig("config.json") - InitTranslations(Cfg.LocalizationSettings) - - // Save state to restore after the test has run. - s.driverName = *Cfg.FileSettings.DriverName - s.amazonS3AccessKeyId = Cfg.FileSettings.AmazonS3AccessKeyId - s.amazonS3SecretAccessKey = Cfg.FileSettings.AmazonS3SecretAccessKey - s.amazonS3Bucket = Cfg.FileSettings.AmazonS3Bucket - s.amazonS3Endpoint = Cfg.FileSettings.AmazonS3Endpoint - s.amazonS3SSL = *Cfg.FileSettings.AmazonS3SSL - - // Set up the state for the tests. - s3Host := os.Getenv("CI_HOST") - if s3Host == "" { - s3Host = "dockerhost" - } - - s3Port := os.Getenv("CI_MINIO_PORT") - if s3Port == "" { - s3Port = "9001" - } - - s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) - if s.testDriver == model.IMAGE_DRIVER_LOCAL { - *Cfg.FileSettings.DriverName = model.IMAGE_DRIVER_LOCAL - } else if s.testDriver == model.IMAGE_DRIVER_S3 { - *Cfg.FileSettings.DriverName = model.IMAGE_DRIVER_S3 - Cfg.FileSettings.AmazonS3AccessKeyId = "minioaccesskey" - Cfg.FileSettings.AmazonS3SecretAccessKey = "miniosecretkey" - Cfg.FileSettings.AmazonS3Bucket = "mattermost-test" - Cfg.FileSettings.AmazonS3Endpoint = s3Endpoint - *Cfg.FileSettings.AmazonS3SSL = false - } else { - s.T().Fatal("Invalid image driver set for test suite.") - } -} - -func (s *FileTestSuite) TearDownTest() { - // Restore the test state. - *Cfg.FileSettings.DriverName = s.driverName - Cfg.FileSettings.AmazonS3AccessKeyId = s.amazonS3AccessKeyId - Cfg.FileSettings.AmazonS3SecretAccessKey = s.amazonS3SecretAccessKey - Cfg.FileSettings.AmazonS3Bucket = s.amazonS3Bucket - Cfg.FileSettings.AmazonS3Endpoint = s.amazonS3Endpoint - *Cfg.FileSettings.AmazonS3SSL = s.amazonS3SSL -} - -func (s *FileTestSuite) TestReadWriteFile() { - b := []byte("test") - path := "tests/" + model.NewId() - - s.Nil(WriteFile(b, path)) - defer RemoveFile(path) - - read, err := ReadFile(path) - s.Nil(err) - - readString := string(read) - s.EqualValues(readString, "test") -} - -func (s *FileTestSuite) TestMoveFile() { - b := []byte("test") - path1 := "tests/" + model.NewId() - path2 := "tests/" + model.NewId() - - s.Nil(WriteFile(b, path1)) - defer RemoveFile(path1) - - s.Nil(MoveFile(path1, path2)) - defer RemoveFile(path2) - - _, err := ReadFile(path1) - s.Error(err) - - _, err = ReadFile(path2) - s.Nil(err) -} - -func (s *FileTestSuite) TestRemoveFile() { - b := []byte("test") - path := "tests/" + model.NewId() - - s.Nil(WriteFile(b, path)) - s.Nil(RemoveFile(path)) - - _, err := ReadFile(path) - s.Error(err) - - s.Nil(WriteFile(b, "tests2/foo")) - s.Nil(WriteFile(b, "tests2/bar")) - s.Nil(WriteFile(b, "tests2/asdf")) - s.Nil(RemoveDirectory("tests2")) -} - -func (s *FileTestSuite) TestListDirectory() { - b := []byte("test") - path1 := "19700101/" + model.NewId() - path2 := "19800101/" + model.NewId() - - s.Nil(WriteFile(b, path1)) - defer RemoveFile(path1) - s.Nil(WriteFile(b, path2)) - defer RemoveFile(path2) - - paths, err := ListDirectory("") - s.Nil(err) - - found1 := false - found2 := false - for _, path := range *paths { - if path == "19700101" { - found1 = true - } else if path == "19800101" { - found2 = true - } - } - s.True(found1) - s.True(found2) -} - -func (s *FileTestSuite) TestRemoveDirectory() { - b := []byte("test") - - s.Nil(WriteFile(b, "tests2/foo")) - s.Nil(WriteFile(b, "tests2/bar")) - s.Nil(WriteFile(b, "tests2/aaa")) - - s.Nil(RemoveDirectory("tests2")) - - _, err := ReadFile("tests2/foo") - s.Error(err) - _, err = ReadFile("tests2/bar") - s.Error(err) - _, err = ReadFile("tests2/asdf") - s.Error(err) -} - func TestCopyDir(t *testing.T) { srcDir, err := ioutil.TempDir("", "src") require.NoError(t, err) |