From 2de6c5394ec3a1cd974aae46c41f61fb0e9f9bd8 Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Fri, 20 Jan 2017 14:47:14 +0000 Subject: Move Slack Import to App Layer. (#5135) --- api/file.go | 191 +------------- api/import.go | 158 ----------- api/slackimport.go | 622 -------------------------------------------- api/slackimport_test.go | 240 ----------------- api/team.go | 2 +- app/file.go | 191 +++++++++++++- app/import.go | 157 +++++++++++ app/slackimport.go | 621 +++++++++++++++++++++++++++++++++++++++++++ app/slackimport_test.go | 240 +++++++++++++++++ cmd/platform/import.go | 4 +- cmd/platform/oldcommands.go | 2 +- 11 files changed, 1214 insertions(+), 1214 deletions(-) delete mode 100644 api/import.go delete mode 100644 api/slackimport.go delete mode 100644 api/slackimport_test.go create mode 100644 app/import.go create mode 100644 app/slackimport.go create mode 100644 app/slackimport_test.go diff --git a/api/file.go b/api/file.go index fdc9a8747..9fda76d8f 100644 --- a/api/file.go +++ b/api/file.go @@ -5,49 +5,20 @@ package api import ( "bytes" - "image" - "image/color" - "image/draw" _ "image/gif" - "image/jpeg" "io" "net/http" "net/url" - "path/filepath" "strconv" - "strings" l4g "github.com/alecthomas/log4go" - "github.com/disintegration/imaging" "github.com/gorilla/mux" "github.com/mattermost/platform/app" "github.com/mattermost/platform/model" "github.com/mattermost/platform/utils" - "github.com/rwcarlsen/goexif/exif" _ "golang.org/x/image/bmp" ) -const ( - /* - EXIF Image Orientations - 1 2 3 4 5 6 7 8 - - 888888 888888 88 88 8888888888 88 88 8888888888 - 88 88 88 88 88 88 88 88 88 88 88 88 - 8888 8888 8888 8888 88 8888888888 8888888888 88 - 88 88 88 88 - 88 88 888888 888888 - */ - Upright = 1 - UprightMirrored = 2 - UpsideDown = 3 - UpsideDownMirrored = 4 - RotatedCWMirrored = 5 - RotatedCCW = 6 - RotatedCCWMirrored = 7 - RotatedCW = 8 -) - func InitFile() { l4g.Debug(utils.T("api.file.init.debug")) @@ -119,7 +90,7 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { io.Copy(buf, file) data := buf.Bytes() - info, err := doUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data) + info, err := app.DoUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data) if err != nil { c.Err = err return @@ -138,169 +109,11 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { } } - handleImages(previewPathList, thumbnailPathList, imageDataList) + app.HandleImages(previewPathList, thumbnailPathList, imageDataList) w.Write([]byte(resStruct.ToJson())) } -func doUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { - filename := filepath.Base(rawFilename) - - info, err := model.GetInfoForBytes(filename, data) - if err != nil { - err.StatusCode = http.StatusBadRequest - return nil, err - } - - info.Id = model.NewId() - info.CreatorId = userId - - pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/" - info.Path = pathPrefix + filename - - if info.IsImage() { - // Check dimensions before loading the whole thing into memory later on - if info.Width*info.Height > model.MaxImageSize { - err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "") - err.StatusCode = http.StatusBadRequest - return nil, err - } - - nameWithoutExtension := filename[:strings.LastIndex(filename, ".")] - info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg" - info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" - } - - if err := app.WriteFile(data, info.Path); err != nil { - return nil, err - } - - if result := <-app.Srv.Store.FileInfo().Save(info); result.Err != nil { - return nil, result.Err - } - - return info, nil -} - -func handleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) { - for i, data := range fileData { - go func(i int, data []byte) { - img, width, height := prepareImage(fileData[i]) - if img != nil { - go generateThumbnailImage(*img, thumbnailPathList[i], width, height) - go generatePreviewImage(*img, previewPathList[i], width) - } - }(i, data) - } -} - -func prepareImage(fileData []byte) (*image.Image, int, int) { - // Decode image bytes into Image object - img, imgType, err := image.Decode(bytes.NewReader(fileData)) - if err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err) - return nil, 0, 0 - } - - width := img.Bounds().Dx() - height := img.Bounds().Dy() - - // Fill in the background of a potentially-transparent png file as white - if imgType == "png" { - dst := image.NewRGBA(img.Bounds()) - draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src) - draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over) - img = dst - } - - // Flip the image to be upright - orientation, _ := getImageOrientation(fileData) - - switch orientation { - case UprightMirrored: - img = imaging.FlipH(img) - case UpsideDown: - img = imaging.Rotate180(img) - case UpsideDownMirrored: - img = imaging.FlipV(img) - case RotatedCWMirrored: - img = imaging.Transpose(img) - case RotatedCCW: - img = imaging.Rotate270(img) - case RotatedCCWMirrored: - img = imaging.Transverse(img) - case RotatedCW: - img = imaging.Rotate90(img) - } - - return &img, width, height -} - -func getImageOrientation(imageData []byte) (int, error) { - if exifData, err := exif.Decode(bytes.NewReader(imageData)); err != nil { - return Upright, err - } else { - if tag, err := exifData.Get("Orientation"); err != nil { - return Upright, err - } else { - orientation, err := tag.Int(0) - if err != nil { - return Upright, err - } else { - return orientation, nil - } - } - } -} - -func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) { - thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth) - thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight) - imgWidth := float64(width) - imgHeight := float64(height) - - var thumbnail image.Image - if imgHeight < thumbHeight && imgWidth < thumbWidth { - thumbnail = img - } else if imgHeight/imgWidth < thumbHeight/thumbWidth { - thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos) - } else { - thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos) - } - - buf := new(bytes.Buffer) - if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err) - return - } - - if err := app.WriteFile(buf.Bytes(), thumbnailPath); err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err) - return - } -} - -func generatePreviewImage(img image.Image, previewPath string, width int) { - var preview image.Image - if width > int(utils.Cfg.FileSettings.PreviewWidth) { - preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos) - } else { - preview = img - } - - buf := new(bytes.Buffer) - - if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err) - return - } - - if err := app.WriteFile(buf.Bytes(), previewPath); err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err) - return - } -} - func getFile(c *Context, w http.ResponseWriter, r *http.Request) { info, err := getFileInfoForRequest(c, r, true) if err != nil { diff --git a/api/import.go b/api/import.go deleted file mode 100644 index b93a061fe..000000000 --- a/api/import.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package api - -import ( - "bytes" - "io" - "regexp" - "unicode/utf8" - - l4g "github.com/alecthomas/log4go" - "github.com/mattermost/platform/app" - "github.com/mattermost/platform/model" - "github.com/mattermost/platform/utils" -) - -// -// Import functions are sutible for entering posts and users into the database without -// some of the usual checks. (IsValid is still run) -// - -func ImportPost(post *model.Post) { - // Workaround for empty messages, which may be the case if they are webhook posts. - firstIteration := true - for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) { - firstIteration = false - var remainder string - if messageRuneCount > model.POST_MESSAGE_MAX_RUNES { - remainder = string(([]rune(post.Message))[model.POST_MESSAGE_MAX_RUNES:]) - post.Message = truncateRunes(post.Message, model.POST_MESSAGE_MAX_RUNES) - } else { - remainder = "" - } - - post.Hashtags, _ = model.ParseHashtags(post.Message) - - if result := <-app.Srv.Store.Post().Save(post); result.Err != nil { - l4g.Debug(utils.T("api.import.import_post.saving.debug"), post.UserId, post.Message) - } - - for _, fileId := range post.FileIds { - if result := <-app.Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil { - l4g.Error(utils.T("api.import.import_post.attach_files.error"), post.Id, post.FileIds, result.Err) - } - } - - post.Id = "" - post.CreateAt++ - post.Message = remainder - } -} - -func ImportUser(team *model.Team, user *model.User) *model.User { - user.MakeNonNil() - - user.Roles = model.ROLE_SYSTEM_USER.Id - - if result := <-app.Srv.Store.User().Save(user); result.Err != nil { - l4g.Error(utils.T("api.import.import_user.saving.error"), result.Err) - return nil - } else { - ruser := result.Data.(*model.User) - - if cresult := <-app.Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil { - l4g.Error(utils.T("api.import.import_user.set_email.error"), cresult.Err) - } - - if err := app.JoinUserToTeam(team, user); err != nil { - l4g.Error(utils.T("api.import.import_user.join_team.error"), err) - } - - return ruser - } -} - -func ImportChannel(channel *model.Channel) *model.Channel { - if result := <-app.Srv.Store.Channel().Save(channel); result.Err != nil { - return nil - } else { - sc := result.Data.(*model.Channel) - - return sc - } -} - -func ImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { - buf := bytes.NewBuffer(nil) - io.Copy(buf, file) - data := buf.Bytes() - - fileInfo, err := doUploadFile(teamId, channelId, userId, fileName, data) - if err != nil { - return nil, err - } - - img, width, height := prepareImage(data) - if img != nil { - generateThumbnailImage(*img, fileInfo.ThumbnailPath, width, height) - generatePreviewImage(*img, fileInfo.PreviewPath, width) - } - - return fileInfo, nil -} - -func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) { - linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) - post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})") - - post.AddProp("from_webhook", "true") - - if _, ok := props["override_username"]; !ok { - post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME) - } - - if len(props) > 0 { - for key, val := range props { - if key == "attachments" { - if list, success := val.([]interface{}); success { - // parse attachment links into Markdown format - for i, aInt := range list { - attachment := aInt.(map[string]interface{}) - if aText, ok := attachment["text"].(string); ok { - aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})") - attachment["text"] = aText - list[i] = attachment - } - if aText, ok := attachment["pretext"].(string); ok { - aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})") - attachment["pretext"] = aText - list[i] = attachment - } - if fVal, ok := attachment["fields"]; ok { - if fields, ok := fVal.([]interface{}); ok { - // parse attachment field links into Markdown format - for j, fInt := range fields { - field := fInt.(map[string]interface{}) - if fValue, ok := field["value"].(string); ok { - fValue = linkWithTextRegex.ReplaceAllString(fValue, "[${2}](${1})") - field["value"] = fValue - fields[j] = field - } - } - attachment["fields"] = fields - list[i] = attachment - } - } - } - post.AddProp(key, list) - } - } else if key != "from_webhook" { - post.AddProp(key, val) - } - } - } - - ImportPost(post) -} diff --git a/api/slackimport.go b/api/slackimport.go deleted file mode 100644 index a7d50b1b9..000000000 --- a/api/slackimport.go +++ /dev/null @@ -1,622 +0,0 @@ -// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package api - -import ( - "archive/zip" - "bytes" - "encoding/json" - "io" - "mime/multipart" - "path/filepath" - "regexp" - "strconv" - "strings" - "unicode/utf8" - - l4g "github.com/alecthomas/log4go" - "github.com/mattermost/platform/app" - "github.com/mattermost/platform/model" - "github.com/mattermost/platform/utils" -) - -type SlackChannel struct { - Id string `json:"id"` - Name string `json:"name"` - Members []string `json:"members"` - Topic map[string]string `json:"topic"` - Purpose map[string]string `json:"purpose"` -} - -type SlackUser struct { - Id string `json:"id"` - Username string `json:"name"` - Profile map[string]string `json:"profile"` -} - -type SlackFile struct { - Id string `json:"id"` - Title string `json:"title"` -} - -type SlackPost struct { - User string `json:"user"` - BotId string `json:"bot_id"` - BotUsername string `json:"username"` - Text string `json:"text"` - TimeStamp string `json:"ts"` - Type string `json:"type"` - SubType string `json:"subtype"` - Comment *SlackComment `json:"comment"` - Upload bool `json:"upload"` - File *SlackFile `json:"file"` - Attachments []SlackAttachment `json:"attachments"` -} - -type SlackComment struct { - User string `json:"user"` - Comment string `json:"comment"` -} - -type SlackAttachment struct { - Id int `json:"id"` - Text string `json:"text"` - Pretext string `json:"pretext"` - Fields []map[string]interface{} `json:"fields"` -} - -func truncateRunes(s string, i int) string { - runes := []rune(s) - if len(runes) > i { - return string(runes[:i]) - } - return s -} - -func SlackConvertTimeStamp(ts string) int64 { - timeString := strings.SplitN(ts, ".", 2)[0] - - timeStamp, err := strconv.ParseInt(timeString, 10, 64) - if err != nil { - l4g.Warn(utils.T("api.slackimport.slack_convert_timestamp.bad.warn")) - return 1 - } - return timeStamp * 1000 // Convert to milliseconds -} - -func SlackConvertChannelName(channelName string) string { - newName := strings.Trim(channelName, "_-") - if len(newName) == 1 { - return "slack-channel-" + newName - } - - return newName -} - -func SlackParseChannels(data io.Reader) ([]SlackChannel, error) { - decoder := json.NewDecoder(data) - - var channels []SlackChannel - if err := decoder.Decode(&channels); err != nil { - l4g.Warn(utils.T("api.slackimport.slack_parse_channels.error")) - return channels, err - } - return channels, nil -} - -func SlackParseUsers(data io.Reader) ([]SlackUser, error) { - decoder := json.NewDecoder(data) - - var users []SlackUser - if err := decoder.Decode(&users); err != nil { - // This actually returns errors that are ignored. - // In this case it is erroring because of a null that Slack - // introduced. So we just return the users here. - return users, err - } - return users, nil -} - -func SlackParsePosts(data io.Reader) ([]SlackPost, error) { - decoder := json.NewDecoder(data) - - var posts []SlackPost - if err := decoder.Decode(&posts); err != nil { - l4g.Warn(utils.T("api.slackimport.slack_parse_posts.error")) - return posts, err - } - return posts, nil -} - -func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map[string]*model.User { - // Log header - log.WriteString(utils.T("api.slackimport.slack_add_users.created")) - log.WriteString("===============\r\n\r\n") - - addedUsers := make(map[string]*model.User) - - // Need the team - var team *model.Team - if result := <-app.Srv.Store.Team().Get(teamId); result.Err != nil { - log.WriteString(utils.T("api.slackimport.slack_import.team_fail")) - return addedUsers - } else { - team = result.Data.(*model.Team) - } - - for _, sUser := range slackusers { - firstName := "" - lastName := "" - if name, ok := sUser.Profile["first_name"]; ok { - firstName = name - } - if name, ok := sUser.Profile["last_name"]; ok { - lastName = name - } - - email := sUser.Profile["email"] - - password := model.NewId() - - // Check for email conflict and use existing user if found - if result := <-app.Srv.Store.User().GetByEmail(email); result.Err == nil { - existingUser := result.Data.(*model.User) - addedUsers[sUser.Id] = existingUser - if err := app.JoinUserToTeam(team, addedUsers[sUser.Id]); err != nil { - log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) - } else { - log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) - } - continue - } - - newUser := model.User{ - Username: sUser.Username, - FirstName: firstName, - LastName: lastName, - Email: email, - Password: password, - } - - if mUser := ImportUser(team, &newUser); mUser != nil { - addedUsers[sUser.Id] = mUser - log.WriteString(utils.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password})) - } else { - log.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username})) - } - } - - return addedUsers -} - -func SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User { - var team *model.Team - if result := <-app.Srv.Store.Team().Get(teamId); result.Err != nil { - log.WriteString(utils.T("api.slackimport.slack_import.team_fail")) - return nil - } else { - team = result.Data.(*model.Team) - } - - password := model.NewId() - username := "slackimportuser_" + model.NewId() - email := username + "@localhost" - - botUser := model.User{ - Username: username, - FirstName: "", - LastName: "", - Email: email, - Password: password, - } - - if mUser := ImportUser(team, &botUser); mUser != nil { - log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password})) - return mUser - } else { - log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username})) - return nil - } -} - -func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) { - for _, sPost := range posts { - switch { - case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"): - if sPost.User == "" { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug")) - continue - } else if users[sPost.User] == nil { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) - continue - } - newPost := model.Post{ - UserId: users[sPost.User].Id, - ChannelId: channel.Id, - Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), - } - if sPost.Upload { - if fileInfo, ok := SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok == true { - newPost.FileIds = append(newPost.FileIds, fileInfo.Id) - newPost.Message = sPost.File.Title - } - } - ImportPost(&newPost) - for _, fileId := range newPost.FileIds { - if result := <-app.Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil { - l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err) - } - } - - case sPost.Type == "message" && sPost.SubType == "file_comment": - if sPost.Comment == nil { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_comment.debug")) - continue - } else if sPost.Comment.User == "" { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) - continue - } else if users[sPost.Comment.User] == nil { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) - continue - } - newPost := model.Post{ - UserId: users[sPost.Comment.User].Id, - ChannelId: channel.Id, - Message: sPost.Comment.Comment, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), - } - ImportPost(&newPost) - case sPost.Type == "message" && sPost.SubType == "bot_message": - if botUser == nil { - l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn")) - continue - } else if sPost.BotId == "" { - l4g.Warn(utils.T("api.slackimport.slack_add_posts.no_bot_id.warn")) - continue - } - - props := make(model.StringInterface) - props["override_username"] = sPost.BotUsername - if len(sPost.Attachments) > 0 { - var mAttachments []interface{} - for _, attachment := range sPost.Attachments { - mAttachments = append(mAttachments, map[string]interface{}{ - "text": attachment.Text, - "pretext": attachment.Pretext, - "fields": attachment.Fields, - }) - } - props["attachments"] = mAttachments - } - - post := &model.Post{ - UserId: botUser.Id, - ChannelId: channel.Id, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), - Message: sPost.Text, - Type: model.POST_SLACK_ATTACHMENT, - } - - ImportIncomingWebhookPost(post, props) - case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"): - if sPost.User == "" { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) - continue - } else if users[sPost.User] == nil { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) - continue - } - newPost := model.Post{ - UserId: users[sPost.User].Id, - ChannelId: channel.Id, - Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), - Type: model.POST_JOIN_LEAVE, - } - ImportPost(&newPost) - case sPost.Type == "message" && sPost.SubType == "me_message": - if sPost.User == "" { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug")) - continue - } else if users[sPost.User] == nil { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) - continue - } - newPost := model.Post{ - UserId: users[sPost.User].Id, - ChannelId: channel.Id, - Message: "*" + sPost.Text + "*", - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), - } - ImportPost(&newPost) - case sPost.Type == "message" && sPost.SubType == "channel_topic": - if sPost.User == "" { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) - continue - } else if users[sPost.User] == nil { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) - continue - } - newPost := model.Post{ - UserId: users[sPost.User].Id, - ChannelId: channel.Id, - Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), - Type: model.POST_HEADER_CHANGE, - } - ImportPost(&newPost) - case sPost.Type == "message" && sPost.SubType == "channel_purpose": - if sPost.User == "" { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) - continue - } else if users[sPost.User] == nil { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) - continue - } - newPost := model.Post{ - UserId: users[sPost.User].Id, - ChannelId: channel.Id, - Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), - Type: model.POST_PURPOSE_CHANGE, - } - ImportPost(&newPost) - case sPost.Type == "message" && sPost.SubType == "channel_name": - if sPost.User == "" { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) - continue - } else if users[sPost.User] == nil { - l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) - continue - } - newPost := model.Post{ - UserId: users[sPost.User].Id, - ChannelId: channel.Id, - Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), - Type: model.POST_DISPLAYNAME_CHANGE, - } - ImportPost(&newPost) - default: - l4g.Warn(utils.T("api.slackimport.slack_add_posts.unsupported.warn"), sPost.Type, sPost.SubType) - } - } -} - -func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId string, channelId string, userId string) (*model.FileInfo, bool) { - if sPost.File != nil { - if file, ok := uploads[sPost.File.Id]; ok == true { - openFile, err := file.Open() - if err != nil { - l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_open_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) - return nil, false - } - defer openFile.Close() - - uploadedFile, err := ImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name)) - if err != nil { - l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) - return nil, false - } - - return uploadedFile, true - } else { - l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_found.warn", map[string]interface{}{"FileId": sPost.File.Id})) - return nil, false - } - } else { - l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_in_json.warn")) - return nil, false - } -} - -func deactivateSlackBotUser(user *model.User) { - _, err := app.UpdateActive(user, false) - if err != nil { - l4g.Warn(utils.T("api.slackimport.slack_deactivate_bot_user.failed_to_deactivate", err)) - } -} - -func addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) { - for _, member := range members { - if user, ok := users[member]; !ok { - log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"})) - } else { - if _, err := app.AddUserToChannel(user, channel); err != nil { - log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username})) - } - } - } -} - -func SlackSanitiseChannelProperties(channel model.Channel) model.Channel { - if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES { - l4g.Warn("api.slackimport.slack_sanitise_channel_properties.display_name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}) - channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES) - } - - if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH { - l4g.Warn("api.slackimport.slack_sanitise_channel_properties.name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}) - channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH] - } - - if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES { - l4g.Warn("api.slackimport.slack_sanitise_channel_properties.purpose_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}) - channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES) - } - - if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES { - l4g.Warn("api.slackimport.slack_sanitise_channel_properties.header_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}) - channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES) - } - - return channel -} - -func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, log *bytes.Buffer) map[string]*model.Channel { - // Write Header - log.WriteString(utils.T("api.slackimport.slack_add_channels.added")) - log.WriteString("=================\r\n\r\n") - - addedChannels := make(map[string]*model.Channel) - for _, sChannel := range slackchannels { - newChannel := model.Channel{ - TeamId: teamId, - Type: model.CHANNEL_OPEN, - DisplayName: sChannel.Name, - Name: SlackConvertChannelName(sChannel.Name), - Purpose: sChannel.Purpose["value"], - Header: sChannel.Topic["value"], - } - newChannel = SlackSanitiseChannelProperties(newChannel) - mChannel := ImportChannel(&newChannel) - if mChannel == nil { - // Maybe it already exists? - if result := <-app.Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil { - l4g.Warn(utils.T("api.slackimport.slack_add_channels.import_failed.warn"), newChannel.DisplayName) - log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName})) - continue - } else { - mChannel = result.Data.(*model.Channel) - log.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName})) - } - } - addSlackUsersToChannel(sChannel.Members, users, mChannel, log) - log.WriteString(newChannel.DisplayName + "\r\n") - addedChannels[sChannel.Id] = mChannel - SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser) - } - - return addedChannels -} - -func SlackConvertUserMentions(users []SlackUser, posts map[string][]SlackPost) map[string][]SlackPost { - var regexes = make(map[string]*regexp.Regexp, len(users)) - for _, user := range users { - r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>") - if err != nil { - l4g.Warn(utils.T("api.slackimport.slack_convert_user_mentions.compile_regexp_failed.warn"), user.Id, user.Username) - continue - } - regexes["@"+user.Username] = r - } - - // Special cases. - regexes["@here"], _ = regexp.Compile(``) - regexes["@channel"], _ = regexp.Compile("") - regexes["@all"], _ = regexp.Compile("") - - for channelName, channelPosts := range posts { - for postIdx, post := range channelPosts { - for mention, r := range regexes { - post.Text = r.ReplaceAllString(post.Text, mention) - posts[channelName][postIdx] = post - } - } - } - - return posts -} - -func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]SlackPost) map[string][]SlackPost { - var regexes = make(map[string]*regexp.Regexp, len(channels)) - for _, channel := range channels { - r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>") - if err != nil { - l4g.Warn(utils.T("api.slackimport.slack_convert_channel_mentions.compile_regexp_failed.warn"), channel.Id, channel.Name) - continue - } - regexes["~"+channel.Name] = r - } - - for channelName, channelPosts := range posts { - for postIdx, post := range channelPosts { - for channelReplace, r := range regexes { - post.Text = r.ReplaceAllString(post.Text, channelReplace) - posts[channelName][postIdx] = post - } - } - } - - return posts -} - -func SlackConvertPostsMarkup(posts map[string][]SlackPost) map[string][]SlackPost { - // Convert URLs in Slack's format to Markdown format. - regex := regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`) - - for channelName, channelPosts := range posts { - for postIdx, post := range channelPosts { - posts[channelName][postIdx].Text = regex.ReplaceAllString(post.Text, "[$2]($1)") - } - } - - return posts -} - -func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { - // Create log file - log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log")) - - zipreader, err := zip.NewReader(fileData, fileSize) - if err != nil || zipreader.File == nil { - log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error")) - return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error()), log - } - - var channels []SlackChannel - var users []SlackUser - posts := make(map[string][]SlackPost) - uploads := make(map[string]*zip.File) - for _, file := range zipreader.File { - reader, err := file.Open() - if err != nil { - log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name})) - return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error()), log - } - if file.Name == "channels.json" { - channels, _ = SlackParseChannels(reader) - } else if file.Name == "users.json" { - users, _ = SlackParseUsers(reader) - } else { - spl := strings.Split(file.Name, "/") - if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") { - newposts, _ := SlackParsePosts(reader) - channel := spl[0] - if _, ok := posts[channel]; ok == false { - posts[channel] = newposts - } else { - posts[channel] = append(posts[channel], newposts...) - } - } else if len(spl) == 3 && spl[0] == "__uploads" { - uploads[spl[1]] = file - } - } - } - - posts = SlackConvertUserMentions(users, posts) - posts = SlackConvertChannelMentions(channels, posts) - posts = SlackConvertPostsMarkup(posts) - - addedUsers := SlackAddUsers(teamID, users, log) - botUser := SlackAddBotUser(teamID, log) - - SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log) - - if botUser != nil { - deactivateSlackBotUser(botUser) - } - - app.InvalidateAllCaches() - - log.WriteString(utils.T("api.slackimport.slack_import.notes")) - log.WriteString("=======\r\n\r\n") - - log.WriteString(utils.T("api.slackimport.slack_import.note1")) - log.WriteString(utils.T("api.slackimport.slack_import.note2")) - log.WriteString(utils.T("api.slackimport.slack_import.note3")) - - return nil, log -} diff --git a/api/slackimport_test.go b/api/slackimport_test.go deleted file mode 100644 index efe6e635f..000000000 --- a/api/slackimport_test.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package api - -import ( - "github.com/mattermost/platform/model" - "os" - "strings" - "testing" -) - -func TestSlackConvertTimeStamp(t *testing.T) { - - testTimeStamp := "1469785419.000033" - - result := SlackConvertTimeStamp(testTimeStamp) - - if result != 1469785419000 { - t.Fatalf("Unexpected timestamp value %v returned.", result) - } -} - -func TestSlackConvertChannelName(t *testing.T) { - var testData = []struct { - input string - output string - }{ - {"test-channel", "test-channel"}, - {"_test_channel_", "test_channel"}, - {"__test", "test"}, - {"-t", "slack-channel-t"}, - {"a", "slack-channel-a"}, - } - - for _, td := range testData { - if td.output != SlackConvertChannelName(td.input) { - t.Fatalf("Did not convert channel name correctly: %v", td.input) - } - } -} - -func TestSlackConvertUserMentions(t *testing.T) { - users := []SlackUser{ - {Id: "U00000A0A", Username: "firstuser"}, - {Id: "U00000B1B", Username: "seconduser"}, - } - - posts := map[string][]SlackPost{ - "test-channel": { - { - Text: ": Hi guys.", - }, - { - Text: "Calling .", - }, - { - Text: "Yo .", - }, - { - Text: "Regular user test <@U00000B1B|seconduser> and <@U00000A0A>.", - }, - }, - } - - expectedPosts := map[string][]SlackPost{ - "test-channel": { - { - Text: "@channel: Hi guys.", - }, - { - Text: "Calling @here.", - }, - { - Text: "Yo @all.", - }, - { - Text: "Regular user test @seconduser and @firstuser.", - }, - }, - } - - convertedPosts := SlackConvertUserMentions(users, posts) - - for channelName, channelPosts := range convertedPosts { - for postIdx, post := range channelPosts { - if post.Text != expectedPosts[channelName][postIdx].Text { - t.Fatalf("Converted post text not as expected: %v", post.Text) - } - } - } -} - -func TestSlackConvertChannelMentions(t *testing.T) { - channels := []SlackChannel{ - {Id: "C000AA00A", Name: "one"}, - {Id: "C000BB11B", Name: "two"}, - } - - posts := map[string][]SlackPost{ - "test-channel": { - { - Text: "Go to <#C000AA00A>.", - }, - { - User: "U00000A0A", - Text: "Try <#C000BB11B|two> for this.", - }, - }, - } - - expectedPosts := map[string][]SlackPost{ - "test-channel": { - { - Text: "Go to ~one.", - }, - { - Text: "Try ~two for this.", - }, - }, - } - - convertedPosts := SlackConvertChannelMentions(channels, posts) - - for channelName, channelPosts := range convertedPosts { - for postIdx, post := range channelPosts { - if post.Text != expectedPosts[channelName][postIdx].Text { - t.Fatalf("Converted post text not as expected: %v", post.Text) - } - } - } - -} - -func TestSlackParseChannels(t *testing.T) { - file, err := os.Open("../tests/slack-import-test-channels.json") - if err != nil { - t.Fatalf("Failed to open data file: %v", err) - } - - channels, err := SlackParseChannels(file) - if err != nil { - t.Fatalf("Error occurred parsing channels: %v", err) - } - - if len(channels) != 6 { - t.Fatalf("Unexpected number of channels: %v", len(channels)) - } -} - -func TestSlackParseUsers(t *testing.T) { - file, err := os.Open("../tests/slack-import-test-users.json") - if err != nil { - t.Fatalf("Failed to open data file: %v", err) - } - - users, err := SlackParseUsers(file) - if err != nil { - t.Fatalf("Error occurred parsing users: %v", err) - } - - if len(users) != 11 { - t.Fatalf("Unexpected number of users: %v", len(users)) - } -} - -func TestSlackParsePosts(t *testing.T) { - file, err := os.Open("../tests/slack-import-test-posts.json") - if err != nil { - t.Fatalf("Failed to open data file: %v", err) - } - - posts, err := SlackParsePosts(file) - if err != nil { - t.Fatalf("Error occurred parsing posts: %v", err) - } - - if len(posts) != 8 { - t.Fatalf("Unexpected number of posts: %v", len(posts)) - } -} - -func TestSlackSanitiseChannelProperties(t *testing.T) { - c1 := model.Channel{ - DisplayName: "display-name", - Name: "name", - Purpose: "The channel purpose", - Header: "The channel header", - } - - c1s := SlackSanitiseChannelProperties(c1) - if c1.DisplayName != c1s.DisplayName || c1.Name != c1s.Name || c1.Purpose != c1s.Purpose || c1.Header != c1s.Header { - t.Fatalf("Unexpected alterations to the channel properties.") - } - - c2 := model.Channel{ - DisplayName: strings.Repeat("abcdefghij", 7), - Name: strings.Repeat("abcdefghij", 7), - Purpose: strings.Repeat("0123456789", 30), - Header: strings.Repeat("0123456789", 120), - } - - c2s := SlackSanitiseChannelProperties(c2) - if c2s.DisplayName != strings.Repeat("abcdefghij", 6)+"abcd" { - t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.DisplayName) - } - - if c2s.Name != strings.Repeat("abcdefghij", 6)+"abcd" { - t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Name) - } - - if c2s.Purpose != strings.Repeat("0123456789", 25) { - t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Purpose) - } - - if c2s.Header != strings.Repeat("0123456789", 102)+"0123" { - t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Header) - } -} - -func TestSlackConvertPostsMarkup(t *testing.T) { - input := make(map[string][]SlackPost) - input["test"] = []SlackPost{ - { - Text: "This message contains a link to .", - }, - { - Text: "This message contains a mailto link to in it.", - }, - } - - output := SlackConvertPostsMarkup(input) - - if output["test"][0].Text != "This message contains a link to [Google](https://google.com)." { - t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text) - } - if output["test"][1].Text != "This message contains a mailto link to [me@example.com](mailto:me@example.com) in it." { - t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text) - } -} diff --git a/api/team.go b/api/team.go index 574c22de5..096e0a49f 100644 --- a/api/team.go +++ b/api/team.go @@ -516,7 +516,7 @@ func importTeam(c *Context, w http.ResponseWriter, r *http.Request) { switch importFrom { case "slack": var err *model.AppError - if err, log = SlackImport(fileData, fileSize, c.TeamId); err != nil { + if err, log = app.SlackImport(fileData, fileSize, c.TeamId); err != nil { c.Err = err c.Err.StatusCode = http.StatusBadRequest } diff --git a/app/file.go b/app/file.go index 93a286a14..a4419bde8 100644 --- a/app/file.go +++ b/app/file.go @@ -5,11 +5,17 @@ package app import ( "bytes" + _ "image/gif" "crypto/sha256" "encoding/base64" "fmt" + "image" + "image/color" + "image/draw" + "image/jpeg" "io" "io/ioutil" + "net/http" "net/url" "os" "path" @@ -20,8 +26,33 @@ import ( l4g "github.com/alecthomas/log4go" "github.com/mattermost/platform/model" "github.com/mattermost/platform/utils" - + "github.com/disintegration/imaging" s3 "github.com/minio/minio-go" + "github.com/rwcarlsen/goexif/exif" + _ "golang.org/x/image/bmp" +) + +const ( + /* + EXIF Image Orientations + 1 2 3 4 5 6 7 8 + + 888888 888888 88 88 8888888888 88 88 8888888888 + 88 88 88 88 88 88 88 88 88 88 88 88 + 8888 8888 8888 8888 88 8888888888 8888888888 88 + 88 88 88 88 + 88 88 888888 888888 + */ + Upright = 1 + UprightMirrored = 2 + UpsideDown = 3 + UpsideDownMirrored = 4 + RotatedCWMirrored = 5 + RotatedCCW = 6 + RotatedCCWMirrored = 7 + RotatedCW = 8 + + MaxImageSize = 6048 * 4032 // 24 megapixels, roughly 36MB as a raw image ) func ReadFile(path string) ([]byte, *model.AppError) { @@ -338,3 +369,161 @@ func GeneratePublicLinkHash(fileId, salt string) string { return base64.RawURLEncoding.EncodeToString(hash.Sum(nil)) } + +func DoUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { + filename := filepath.Base(rawFilename) + + info, err := model.GetInfoForBytes(filename, data) + if err != nil { + err.StatusCode = http.StatusBadRequest + return nil, err + } + + info.Id = model.NewId() + info.CreatorId = userId + + pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/" + info.Path = pathPrefix + filename + + if info.IsImage() { + // Check dimensions before loading the whole thing into memory later on + if info.Width*info.Height > MaxImageSize { + err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "") + err.StatusCode = http.StatusBadRequest + return nil, err + } + + nameWithoutExtension := filename[:strings.LastIndex(filename, ".")] + info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg" + info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" + } + + if err := WriteFile(data, info.Path); err != nil { + return nil, err + } + + if result := <-Srv.Store.FileInfo().Save(info); result.Err != nil { + return nil, result.Err + } + + return info, nil +} + +func HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) { + for i, data := range fileData { + go func(i int, data []byte) { + img, width, height := prepareImage(fileData[i]) + if img != nil { + go generateThumbnailImage(*img, thumbnailPathList[i], width, height) + go generatePreviewImage(*img, previewPathList[i], width) + } + }(i, data) + } +} + +func prepareImage(fileData []byte) (*image.Image, int, int) { + // Decode image bytes into Image object + img, imgType, err := image.Decode(bytes.NewReader(fileData)) + if err != nil { + l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err) + return nil, 0, 0 + } + + width := img.Bounds().Dx() + height := img.Bounds().Dy() + + // Fill in the background of a potentially-transparent png file as white + if imgType == "png" { + dst := image.NewRGBA(img.Bounds()) + draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src) + draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over) + img = dst + } + + // Flip the image to be upright + orientation, _ := getImageOrientation(fileData) + + switch orientation { + case UprightMirrored: + img = imaging.FlipH(img) + case UpsideDown: + img = imaging.Rotate180(img) + case UpsideDownMirrored: + img = imaging.FlipV(img) + case RotatedCWMirrored: + img = imaging.Transpose(img) + case RotatedCCW: + img = imaging.Rotate270(img) + case RotatedCCWMirrored: + img = imaging.Transverse(img) + case RotatedCW: + img = imaging.Rotate90(img) + } + + return &img, width, height +} + +func getImageOrientation(imageData []byte) (int, error) { + if exifData, err := exif.Decode(bytes.NewReader(imageData)); err != nil { + return Upright, err + } else { + if tag, err := exifData.Get("Orientation"); err != nil { + return Upright, err + } else { + orientation, err := tag.Int(0) + if err != nil { + return Upright, err + } else { + return orientation, nil + } + } + } +} + +func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) { + thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth) + thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight) + imgWidth := float64(width) + imgHeight := float64(height) + + var thumbnail image.Image + if imgHeight < thumbHeight && imgWidth < thumbWidth { + thumbnail = img + } else if imgHeight/imgWidth < thumbHeight/thumbWidth { + thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos) + } else { + thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos) + } + + buf := new(bytes.Buffer) + if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil { + l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err) + return + } + + if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil { + l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err) + return + } +} + +func generatePreviewImage(img image.Image, previewPath string, width int) { + var preview image.Image + if width > int(utils.Cfg.FileSettings.PreviewWidth) { + preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos) + } else { + preview = img + } + + buf := new(bytes.Buffer) + + if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil { + l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err) + return + } + + if err := WriteFile(buf.Bytes(), previewPath); err != nil { + l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err) + return + } +} diff --git a/app/import.go b/app/import.go new file mode 100644 index 000000000..8f2cf552e --- /dev/null +++ b/app/import.go @@ -0,0 +1,157 @@ +// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "bytes" + "io" + "regexp" + "unicode/utf8" + + l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" +) + +// +// Import functions are sutible for entering posts and users into the database without +// some of the usual checks. (IsValid is still run) +// + +func ImportPost(post *model.Post) { + // Workaround for empty messages, which may be the case if they are webhook posts. + firstIteration := true + for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) { + firstIteration = false + var remainder string + if messageRuneCount > model.POST_MESSAGE_MAX_RUNES { + remainder = string(([]rune(post.Message))[model.POST_MESSAGE_MAX_RUNES:]) + post.Message = truncateRunes(post.Message, model.POST_MESSAGE_MAX_RUNES) + } else { + remainder = "" + } + + post.Hashtags, _ = model.ParseHashtags(post.Message) + + if result := <-Srv.Store.Post().Save(post); result.Err != nil { + l4g.Debug(utils.T("api.import.import_post.saving.debug"), post.UserId, post.Message) + } + + for _, fileId := range post.FileIds { + if result := <-Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil { + l4g.Error(utils.T("api.import.import_post.attach_files.error"), post.Id, post.FileIds, result.Err) + } + } + + post.Id = "" + post.CreateAt++ + post.Message = remainder + } +} + +func ImportUser(team *model.Team, user *model.User) *model.User { + user.MakeNonNil() + + user.Roles = model.ROLE_SYSTEM_USER.Id + + if result := <-Srv.Store.User().Save(user); result.Err != nil { + l4g.Error(utils.T("api.import.import_user.saving.error"), result.Err) + return nil + } else { + ruser := result.Data.(*model.User) + + if cresult := <-Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil { + l4g.Error(utils.T("api.import.import_user.set_email.error"), cresult.Err) + } + + if err := JoinUserToTeam(team, user); err != nil { + l4g.Error(utils.T("api.import.import_user.join_team.error"), err) + } + + return ruser + } +} + +func ImportChannel(channel *model.Channel) *model.Channel { + if result := <-Srv.Store.Channel().Save(channel); result.Err != nil { + return nil + } else { + sc := result.Data.(*model.Channel) + + return sc + } +} + +func ImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { + buf := bytes.NewBuffer(nil) + io.Copy(buf, file) + data := buf.Bytes() + + fileInfo, err := DoUploadFile(teamId, channelId, userId, fileName, data) + if err != nil { + return nil, err + } + + img, width, height := prepareImage(data) + if img != nil { + generateThumbnailImage(*img, fileInfo.ThumbnailPath, width, height) + generatePreviewImage(*img, fileInfo.PreviewPath, width) + } + + return fileInfo, nil +} + +func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) { + linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) + post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})") + + post.AddProp("from_webhook", "true") + + if _, ok := props["override_username"]; !ok { + post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME) + } + + if len(props) > 0 { + for key, val := range props { + if key == "attachments" { + if list, success := val.([]interface{}); success { + // parse attachment links into Markdown format + for i, aInt := range list { + attachment := aInt.(map[string]interface{}) + if aText, ok := attachment["text"].(string); ok { + aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})") + attachment["text"] = aText + list[i] = attachment + } + if aText, ok := attachment["pretext"].(string); ok { + aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})") + attachment["pretext"] = aText + list[i] = attachment + } + if fVal, ok := attachment["fields"]; ok { + if fields, ok := fVal.([]interface{}); ok { + // parse attachment field links into Markdown format + for j, fInt := range fields { + field := fInt.(map[string]interface{}) + if fValue, ok := field["value"].(string); ok { + fValue = linkWithTextRegex.ReplaceAllString(fValue, "[${2}](${1})") + field["value"] = fValue + fields[j] = field + } + } + attachment["fields"] = fields + list[i] = attachment + } + } + } + post.AddProp(key, list) + } + } else if key != "from_webhook" { + post.AddProp(key, val) + } + } + } + + ImportPost(post) +} diff --git a/app/slackimport.go b/app/slackimport.go new file mode 100644 index 000000000..53f455069 --- /dev/null +++ b/app/slackimport.go @@ -0,0 +1,621 @@ +// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "archive/zip" + "bytes" + "encoding/json" + "io" + "mime/multipart" + "path/filepath" + "regexp" + "strconv" + "strings" + "unicode/utf8" + + l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" +) + +type SlackChannel struct { + Id string `json:"id"` + Name string `json:"name"` + Members []string `json:"members"` + Topic map[string]string `json:"topic"` + Purpose map[string]string `json:"purpose"` +} + +type SlackUser struct { + Id string `json:"id"` + Username string `json:"name"` + Profile map[string]string `json:"profile"` +} + +type SlackFile struct { + Id string `json:"id"` + Title string `json:"title"` +} + +type SlackPost struct { + User string `json:"user"` + BotId string `json:"bot_id"` + BotUsername string `json:"username"` + Text string `json:"text"` + TimeStamp string `json:"ts"` + Type string `json:"type"` + SubType string `json:"subtype"` + Comment *SlackComment `json:"comment"` + Upload bool `json:"upload"` + File *SlackFile `json:"file"` + Attachments []SlackAttachment `json:"attachments"` +} + +type SlackComment struct { + User string `json:"user"` + Comment string `json:"comment"` +} + +type SlackAttachment struct { + Id int `json:"id"` + Text string `json:"text"` + Pretext string `json:"pretext"` + Fields []map[string]interface{} `json:"fields"` +} + +func truncateRunes(s string, i int) string { + runes := []rune(s) + if len(runes) > i { + return string(runes[:i]) + } + return s +} + +func SlackConvertTimeStamp(ts string) int64 { + timeString := strings.SplitN(ts, ".", 2)[0] + + timeStamp, err := strconv.ParseInt(timeString, 10, 64) + if err != nil { + l4g.Warn(utils.T("api.slackimport.slack_convert_timestamp.bad.warn")) + return 1 + } + return timeStamp * 1000 // Convert to milliseconds +} + +func SlackConvertChannelName(channelName string) string { + newName := strings.Trim(channelName, "_-") + if len(newName) == 1 { + return "slack-channel-" + newName + } + + return newName +} + +func SlackParseChannels(data io.Reader) ([]SlackChannel, error) { + decoder := json.NewDecoder(data) + + var channels []SlackChannel + if err := decoder.Decode(&channels); err != nil { + l4g.Warn(utils.T("api.slackimport.slack_parse_channels.error")) + return channels, err + } + return channels, nil +} + +func SlackParseUsers(data io.Reader) ([]SlackUser, error) { + decoder := json.NewDecoder(data) + + var users []SlackUser + if err := decoder.Decode(&users); err != nil { + // This actually returns errors that are ignored. + // In this case it is erroring because of a null that Slack + // introduced. So we just return the users here. + return users, err + } + return users, nil +} + +func SlackParsePosts(data io.Reader) ([]SlackPost, error) { + decoder := json.NewDecoder(data) + + var posts []SlackPost + if err := decoder.Decode(&posts); err != nil { + l4g.Warn(utils.T("api.slackimport.slack_parse_posts.error")) + return posts, err + } + return posts, nil +} + +func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map[string]*model.User { + // Log header + log.WriteString(utils.T("api.slackimport.slack_add_users.created")) + log.WriteString("===============\r\n\r\n") + + addedUsers := make(map[string]*model.User) + + // Need the team + var team *model.Team + if result := <-Srv.Store.Team().Get(teamId); result.Err != nil { + log.WriteString(utils.T("api.slackimport.slack_import.team_fail")) + return addedUsers + } else { + team = result.Data.(*model.Team) + } + + for _, sUser := range slackusers { + firstName := "" + lastName := "" + if name, ok := sUser.Profile["first_name"]; ok { + firstName = name + } + if name, ok := sUser.Profile["last_name"]; ok { + lastName = name + } + + email := sUser.Profile["email"] + + password := model.NewId() + + // Check for email conflict and use existing user if found + if result := <-Srv.Store.User().GetByEmail(email); result.Err == nil { + existingUser := result.Data.(*model.User) + addedUsers[sUser.Id] = existingUser + if err := JoinUserToTeam(team, addedUsers[sUser.Id]); err != nil { + log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) + } else { + log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) + } + continue + } + + newUser := model.User{ + Username: sUser.Username, + FirstName: firstName, + LastName: lastName, + Email: email, + Password: password, + } + + if mUser := ImportUser(team, &newUser); mUser != nil { + addedUsers[sUser.Id] = mUser + log.WriteString(utils.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password})) + } else { + log.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username})) + } + } + + return addedUsers +} + +func SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User { + var team *model.Team + if result := <-Srv.Store.Team().Get(teamId); result.Err != nil { + log.WriteString(utils.T("api.slackimport.slack_import.team_fail")) + return nil + } else { + team = result.Data.(*model.Team) + } + + password := model.NewId() + username := "slackimportuser_" + model.NewId() + email := username + "@localhost" + + botUser := model.User{ + Username: username, + FirstName: "", + LastName: "", + Email: email, + Password: password, + } + + if mUser := ImportUser(team, &botUser); mUser != nil { + log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password})) + return mUser + } else { + log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username})) + return nil + } +} + +func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) { + for _, sPost := range posts { + switch { + case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"): + if sPost.User == "" { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug")) + continue + } else if users[sPost.User] == nil { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) + continue + } + newPost := model.Post{ + UserId: users[sPost.User].Id, + ChannelId: channel.Id, + Message: sPost.Text, + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + } + if sPost.Upload { + if fileInfo, ok := SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok == true { + newPost.FileIds = append(newPost.FileIds, fileInfo.Id) + newPost.Message = sPost.File.Title + } + } + ImportPost(&newPost) + for _, fileId := range newPost.FileIds { + if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil { + l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err) + } + } + + case sPost.Type == "message" && sPost.SubType == "file_comment": + if sPost.Comment == nil { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_comment.debug")) + continue + } else if sPost.Comment.User == "" { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) + continue + } else if users[sPost.Comment.User] == nil { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) + continue + } + newPost := model.Post{ + UserId: users[sPost.Comment.User].Id, + ChannelId: channel.Id, + Message: sPost.Comment.Comment, + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + } + ImportPost(&newPost) + case sPost.Type == "message" && sPost.SubType == "bot_message": + if botUser == nil { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn")) + continue + } else if sPost.BotId == "" { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.no_bot_id.warn")) + continue + } + + props := make(model.StringInterface) + props["override_username"] = sPost.BotUsername + if len(sPost.Attachments) > 0 { + var mAttachments []interface{} + for _, attachment := range sPost.Attachments { + mAttachments = append(mAttachments, map[string]interface{}{ + "text": attachment.Text, + "pretext": attachment.Pretext, + "fields": attachment.Fields, + }) + } + props["attachments"] = mAttachments + } + + post := &model.Post{ + UserId: botUser.Id, + ChannelId: channel.Id, + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + Message: sPost.Text, + Type: model.POST_SLACK_ATTACHMENT, + } + + ImportIncomingWebhookPost(post, props) + case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"): + if sPost.User == "" { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) + continue + } else if users[sPost.User] == nil { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) + continue + } + newPost := model.Post{ + UserId: users[sPost.User].Id, + ChannelId: channel.Id, + Message: sPost.Text, + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + Type: model.POST_JOIN_LEAVE, + } + ImportPost(&newPost) + case sPost.Type == "message" && sPost.SubType == "me_message": + if sPost.User == "" { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug")) + continue + } else if users[sPost.User] == nil { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) + continue + } + newPost := model.Post{ + UserId: users[sPost.User].Id, + ChannelId: channel.Id, + Message: "*" + sPost.Text + "*", + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + } + ImportPost(&newPost) + case sPost.Type == "message" && sPost.SubType == "channel_topic": + if sPost.User == "" { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) + continue + } else if users[sPost.User] == nil { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) + continue + } + newPost := model.Post{ + UserId: users[sPost.User].Id, + ChannelId: channel.Id, + Message: sPost.Text, + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + Type: model.POST_HEADER_CHANGE, + } + ImportPost(&newPost) + case sPost.Type == "message" && sPost.SubType == "channel_purpose": + if sPost.User == "" { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) + continue + } else if users[sPost.User] == nil { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) + continue + } + newPost := model.Post{ + UserId: users[sPost.User].Id, + ChannelId: channel.Id, + Message: sPost.Text, + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + Type: model.POST_PURPOSE_CHANGE, + } + ImportPost(&newPost) + case sPost.Type == "message" && sPost.SubType == "channel_name": + if sPost.User == "" { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) + continue + } else if users[sPost.User] == nil { + l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User) + continue + } + newPost := model.Post{ + UserId: users[sPost.User].Id, + ChannelId: channel.Id, + Message: sPost.Text, + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + Type: model.POST_DISPLAYNAME_CHANGE, + } + ImportPost(&newPost) + default: + l4g.Warn(utils.T("api.slackimport.slack_add_posts.unsupported.warn"), sPost.Type, sPost.SubType) + } + } +} + +func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId string, channelId string, userId string) (*model.FileInfo, bool) { + if sPost.File != nil { + if file, ok := uploads[sPost.File.Id]; ok == true { + openFile, err := file.Open() + if err != nil { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_open_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) + return nil, false + } + defer openFile.Close() + + uploadedFile, err := ImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name)) + if err != nil { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) + return nil, false + } + + return uploadedFile, true + } else { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_found.warn", map[string]interface{}{"FileId": sPost.File.Id})) + return nil, false + } + } else { + l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_in_json.warn")) + return nil, false + } +} + +func deactivateSlackBotUser(user *model.User) { + _, err := UpdateActive(user, false) + if err != nil { + l4g.Warn(utils.T("api.slackimport.slack_deactivate_bot_user.failed_to_deactivate", err)) + } +} + +func addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) { + for _, member := range members { + if user, ok := users[member]; !ok { + log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"})) + } else { + if _, err := AddUserToChannel(user, channel); err != nil { + log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username})) + } + } + } +} + +func SlackSanitiseChannelProperties(channel model.Channel) model.Channel { + if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES { + l4g.Warn("api.slackimport.slack_sanitise_channel_properties.display_name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}) + channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES) + } + + if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH { + l4g.Warn("api.slackimport.slack_sanitise_channel_properties.name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}) + channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH] + } + + if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES { + l4g.Warn("api.slackimport.slack_sanitise_channel_properties.purpose_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}) + channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES) + } + + if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES { + l4g.Warn("api.slackimport.slack_sanitise_channel_properties.header_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}) + channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES) + } + + return channel +} + +func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, log *bytes.Buffer) map[string]*model.Channel { + // Write Header + log.WriteString(utils.T("api.slackimport.slack_add_channels.added")) + log.WriteString("=================\r\n\r\n") + + addedChannels := make(map[string]*model.Channel) + for _, sChannel := range slackchannels { + newChannel := model.Channel{ + TeamId: teamId, + Type: model.CHANNEL_OPEN, + DisplayName: sChannel.Name, + Name: SlackConvertChannelName(sChannel.Name), + Purpose: sChannel.Purpose["value"], + Header: sChannel.Topic["value"], + } + newChannel = SlackSanitiseChannelProperties(newChannel) + mChannel := ImportChannel(&newChannel) + if mChannel == nil { + // Maybe it already exists? + if result := <-Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil { + l4g.Warn(utils.T("api.slackimport.slack_add_channels.import_failed.warn"), newChannel.DisplayName) + log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName})) + continue + } else { + mChannel = result.Data.(*model.Channel) + log.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName})) + } + } + addSlackUsersToChannel(sChannel.Members, users, mChannel, log) + log.WriteString(newChannel.DisplayName + "\r\n") + addedChannels[sChannel.Id] = mChannel + SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser) + } + + return addedChannels +} + +func SlackConvertUserMentions(users []SlackUser, posts map[string][]SlackPost) map[string][]SlackPost { + var regexes = make(map[string]*regexp.Regexp, len(users)) + for _, user := range users { + r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>") + if err != nil { + l4g.Warn(utils.T("api.slackimport.slack_convert_user_mentions.compile_regexp_failed.warn"), user.Id, user.Username) + continue + } + regexes["@"+user.Username] = r + } + + // Special cases. + regexes["@here"], _ = regexp.Compile(``) + regexes["@channel"], _ = regexp.Compile("") + regexes["@all"], _ = regexp.Compile("") + + for channelName, channelPosts := range posts { + for postIdx, post := range channelPosts { + for mention, r := range regexes { + post.Text = r.ReplaceAllString(post.Text, mention) + posts[channelName][postIdx] = post + } + } + } + + return posts +} + +func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]SlackPost) map[string][]SlackPost { + var regexes = make(map[string]*regexp.Regexp, len(channels)) + for _, channel := range channels { + r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>") + if err != nil { + l4g.Warn(utils.T("api.slackimport.slack_convert_channel_mentions.compile_regexp_failed.warn"), channel.Id, channel.Name) + continue + } + regexes["~"+channel.Name] = r + } + + for channelName, channelPosts := range posts { + for postIdx, post := range channelPosts { + for channelReplace, r := range regexes { + post.Text = r.ReplaceAllString(post.Text, channelReplace) + posts[channelName][postIdx] = post + } + } + } + + return posts +} + +func SlackConvertPostsMarkup(posts map[string][]SlackPost) map[string][]SlackPost { + // Convert URLs in Slack's format to Markdown format. + regex := regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`) + + for channelName, channelPosts := range posts { + for postIdx, post := range channelPosts { + posts[channelName][postIdx].Text = regex.ReplaceAllString(post.Text, "[$2]($1)") + } + } + + return posts +} + +func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { + // Create log file + log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log")) + + zipreader, err := zip.NewReader(fileData, fileSize) + if err != nil || zipreader.File == nil { + log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error")) + return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error()), log + } + + var channels []SlackChannel + var users []SlackUser + posts := make(map[string][]SlackPost) + uploads := make(map[string]*zip.File) + for _, file := range zipreader.File { + reader, err := file.Open() + if err != nil { + log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name})) + return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error()), log + } + if file.Name == "channels.json" { + channels, _ = SlackParseChannels(reader) + } else if file.Name == "users.json" { + users, _ = SlackParseUsers(reader) + } else { + spl := strings.Split(file.Name, "/") + if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") { + newposts, _ := SlackParsePosts(reader) + channel := spl[0] + if _, ok := posts[channel]; ok == false { + posts[channel] = newposts + } else { + posts[channel] = append(posts[channel], newposts...) + } + } else if len(spl) == 3 && spl[0] == "__uploads" { + uploads[spl[1]] = file + } + } + } + + posts = SlackConvertUserMentions(users, posts) + posts = SlackConvertChannelMentions(channels, posts) + posts = SlackConvertPostsMarkup(posts) + + addedUsers := SlackAddUsers(teamID, users, log) + botUser := SlackAddBotUser(teamID, log) + + SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log) + + if botUser != nil { + deactivateSlackBotUser(botUser) + } + + InvalidateAllCaches() + + log.WriteString(utils.T("api.slackimport.slack_import.notes")) + log.WriteString("=======\r\n\r\n") + + log.WriteString(utils.T("api.slackimport.slack_import.note1")) + log.WriteString(utils.T("api.slackimport.slack_import.note2")) + log.WriteString(utils.T("api.slackimport.slack_import.note3")) + + return nil, log +} diff --git a/app/slackimport_test.go b/app/slackimport_test.go new file mode 100644 index 000000000..3389c5217 --- /dev/null +++ b/app/slackimport_test.go @@ -0,0 +1,240 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "github.com/mattermost/platform/model" + "os" + "strings" + "testing" +) + +func TestSlackConvertTimeStamp(t *testing.T) { + + testTimeStamp := "1469785419.000033" + + result := SlackConvertTimeStamp(testTimeStamp) + + if result != 1469785419000 { + t.Fatalf("Unexpected timestamp value %v returned.", result) + } +} + +func TestSlackConvertChannelName(t *testing.T) { + var testData = []struct { + input string + output string + }{ + {"test-channel", "test-channel"}, + {"_test_channel_", "test_channel"}, + {"__test", "test"}, + {"-t", "slack-channel-t"}, + {"a", "slack-channel-a"}, + } + + for _, td := range testData { + if td.output != SlackConvertChannelName(td.input) { + t.Fatalf("Did not convert channel name correctly: %v", td.input) + } + } +} + +func TestSlackConvertUserMentions(t *testing.T) { + users := []SlackUser{ + {Id: "U00000A0A", Username: "firstuser"}, + {Id: "U00000B1B", Username: "seconduser"}, + } + + posts := map[string][]SlackPost{ + "test-channel": { + { + Text: ": Hi guys.", + }, + { + Text: "Calling .", + }, + { + Text: "Yo .", + }, + { + Text: "Regular user test <@U00000B1B|seconduser> and <@U00000A0A>.", + }, + }, + } + + expectedPosts := map[string][]SlackPost{ + "test-channel": { + { + Text: "@channel: Hi guys.", + }, + { + Text: "Calling @here.", + }, + { + Text: "Yo @all.", + }, + { + Text: "Regular user test @seconduser and @firstuser.", + }, + }, + } + + convertedPosts := SlackConvertUserMentions(users, posts) + + for channelName, channelPosts := range convertedPosts { + for postIdx, post := range channelPosts { + if post.Text != expectedPosts[channelName][postIdx].Text { + t.Fatalf("Converted post text not as expected: %v", post.Text) + } + } + } +} + +func TestSlackConvertChannelMentions(t *testing.T) { + channels := []SlackChannel{ + {Id: "C000AA00A", Name: "one"}, + {Id: "C000BB11B", Name: "two"}, + } + + posts := map[string][]SlackPost{ + "test-channel": { + { + Text: "Go to <#C000AA00A>.", + }, + { + User: "U00000A0A", + Text: "Try <#C000BB11B|two> for this.", + }, + }, + } + + expectedPosts := map[string][]SlackPost{ + "test-channel": { + { + Text: "Go to ~one.", + }, + { + Text: "Try ~two for this.", + }, + }, + } + + convertedPosts := SlackConvertChannelMentions(channels, posts) + + for channelName, channelPosts := range convertedPosts { + for postIdx, post := range channelPosts { + if post.Text != expectedPosts[channelName][postIdx].Text { + t.Fatalf("Converted post text not as expected: %v", post.Text) + } + } + } + +} + +func TestSlackParseChannels(t *testing.T) { + file, err := os.Open("../tests/slack-import-test-channels.json") + if err != nil { + t.Fatalf("Failed to open data file: %v", err) + } + + channels, err := SlackParseChannels(file) + if err != nil { + t.Fatalf("Error occurred parsing channels: %v", err) + } + + if len(channels) != 6 { + t.Fatalf("Unexpected number of channels: %v", len(channels)) + } +} + +func TestSlackParseUsers(t *testing.T) { + file, err := os.Open("../tests/slack-import-test-users.json") + if err != nil { + t.Fatalf("Failed to open data file: %v", err) + } + + users, err := SlackParseUsers(file) + if err != nil { + t.Fatalf("Error occurred parsing users: %v", err) + } + + if len(users) != 11 { + t.Fatalf("Unexpected number of users: %v", len(users)) + } +} + +func TestSlackParsePosts(t *testing.T) { + file, err := os.Open("../tests/slack-import-test-posts.json") + if err != nil { + t.Fatalf("Failed to open data file: %v", err) + } + + posts, err := SlackParsePosts(file) + if err != nil { + t.Fatalf("Error occurred parsing posts: %v", err) + } + + if len(posts) != 8 { + t.Fatalf("Unexpected number of posts: %v", len(posts)) + } +} + +func TestSlackSanitiseChannelProperties(t *testing.T) { + c1 := model.Channel{ + DisplayName: "display-name", + Name: "name", + Purpose: "The channel purpose", + Header: "The channel header", + } + + c1s := SlackSanitiseChannelProperties(c1) + if c1.DisplayName != c1s.DisplayName || c1.Name != c1s.Name || c1.Purpose != c1s.Purpose || c1.Header != c1s.Header { + t.Fatalf("Unexpected alterations to the channel properties.") + } + + c2 := model.Channel{ + DisplayName: strings.Repeat("abcdefghij", 7), + Name: strings.Repeat("abcdefghij", 7), + Purpose: strings.Repeat("0123456789", 30), + Header: strings.Repeat("0123456789", 120), + } + + c2s := SlackSanitiseChannelProperties(c2) + if c2s.DisplayName != strings.Repeat("abcdefghij", 6)+"abcd" { + t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.DisplayName) + } + + if c2s.Name != strings.Repeat("abcdefghij", 6)+"abcd" { + t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Name) + } + + if c2s.Purpose != strings.Repeat("0123456789", 25) { + t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Purpose) + } + + if c2s.Header != strings.Repeat("0123456789", 102)+"0123" { + t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Header) + } +} + +func TestSlackConvertPostsMarkup(t *testing.T) { + input := make(map[string][]SlackPost) + input["test"] = []SlackPost{ + { + Text: "This message contains a link to .", + }, + { + Text: "This message contains a mailto link to in it.", + }, + } + + output := SlackConvertPostsMarkup(input) + + if output["test"][0].Text != "This message contains a link to [Google](https://google.com)." { + t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text) + } + if output["test"][1].Text != "This message contains a mailto link to [me@example.com](mailto:me@example.com) in it." { + t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text) + } +} diff --git a/cmd/platform/import.go b/cmd/platform/import.go index b482cda7e..09b135354 100644 --- a/cmd/platform/import.go +++ b/cmd/platform/import.go @@ -6,7 +6,7 @@ import ( "errors" "os" - "github.com/mattermost/platform/api" + "github.com/mattermost/platform/app" "github.com/spf13/cobra" ) @@ -54,7 +54,7 @@ func slackImportCmdF(cmd *cobra.Command, args []string) error { CommandPrettyPrintln("Running Slack Import. This may take a long time for large teams or teams with many messages.") - api.SlackImport(fileReader, fileInfo.Size(), team.Id) + app.SlackImport(fileReader, fileInfo.Size(), team.Id) CommandPrettyPrintln("Finished Slack Import.") diff --git a/cmd/platform/oldcommands.go b/cmd/platform/oldcommands.go index 1ff65130c..ee7f66567 100644 --- a/cmd/platform/oldcommands.go +++ b/cmd/platform/oldcommands.go @@ -1075,7 +1075,7 @@ func cmdSlackImport() { fmt.Fprintln(os.Stdout, "Running Slack Import. This may take a long time for large teams or teams with many messages.") - api.SlackImport(fileReader, fileInfo.Size(), team.Id) + app.SlackImport(fileReader, fileInfo.Size(), team.Id) flushLogAndExit(0) } -- cgit v1.2.3-1-g7c22