From 8a0e649f989a824bb3bbfd1900a5b8e5383b47e1 Mon Sep 17 00:00:00 2001 From: Harrison Healey Date: Fri, 30 Sep 2016 11:06:30 -0400 Subject: PLT-3105 Files table migration (#4068) * Implemented initial changes for files table * Removed *_benchmark_test.go files * Re-implemented GetPublicFile and added support for old path * Localization for files table * Moved file system code into utils package * Finished server-side changes and added initial upgrade script * Added getPostFiles api * Re-add Extension and HasPreviewImage fields to FileInfo * Removed unused translation * Fixed merge conflicts left over after permissions changes * Forced FileInfo.extension to be lower case * Changed FileUploadResponse to contain the FileInfos instead of FileIds * Fixed permissions on getFile* calls * Fixed notifications for file uploads * Added initial version of client code for files changes * Permanently added FileIds field to Post object and removed Post.HasFiles * Updated PostStore.Update to be usable in more circumstances * Re-added Filenames field and switched file migration to be entirely lazy-loaded * Increased max listener count for FileStore * Removed unused fileInfoCache * Moved file system code back into api * Removed duplicate test case * Fixed unit test running on ports other than 8065 * Renamed HasPermissionToPostContext to HasPermissionToChannelByPostContext * Refactored handleImages to make it more easily understandable * Renamed getPostFiles to getFileInfosForPost * Re-added pre-FileIds posts to analytics * Changed files to be saved as their ids as opposed to id/filename.ext * Renamed FileInfo.UserId to FileInfo.CreatorId * Fixed detection of language in CodePreview * Fixed switching between threads in the RHS not loading new files * Add serverside protection against a rare bug where the client sends the same file twice for a single post * Refactored the important parts of uploadFile api call into a function that can be called without a web context --- api/post.go | 166 ++++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 106 insertions(+), 60 deletions(-) (limited to 'api/post.go') diff --git a/api/post.go b/api/post.go index 1286a23d9..498f5b363 100644 --- a/api/post.go +++ b/api/post.go @@ -49,6 +49,7 @@ func InitPost() { BaseRoutes.NeedPost.Handle("/delete", ApiUserRequired(deletePost)).Methods("POST") BaseRoutes.NeedPost.Handle("/before/{offset:[0-9]+}/{num_posts:[0-9]+}", ApiUserRequired(getPostsBefore)).Methods("GET") BaseRoutes.NeedPost.Handle("/after/{offset:[0-9]+}/{num_posts:[0-9]+}", ApiUserRequired(getPostsAfter)).Methods("GET") + BaseRoutes.NeedPost.Handle("/get_file_infos", ApiUserRequired(getFileInfosForPost)).Methods("GET") } func createPost(c *Context, w http.ResponseWriter, r *http.Request) { @@ -135,48 +136,26 @@ func CreatePost(c *Context, post *model.Post, triggerWebhooks bool) (*model.Post post.Hashtags, _ = model.ParseHashtags(post.Message) - if len(post.Filenames) > 0 { - doRemove := false - for i := len(post.Filenames) - 1; i >= 0; i-- { - path := post.Filenames[i] - - doRemove = false - if model.UrlRegex.MatchString(path) { - continue - } else if model.PartialUrlRegex.MatchString(path) { - matches := model.PartialUrlRegex.FindAllStringSubmatch(path, -1) - if len(matches) == 0 || len(matches[0]) < 4 { - doRemove = true - } - - channelId := matches[0][1] - if channelId != post.ChannelId { - doRemove = true - } - - userId := matches[0][2] - if userId != post.UserId { - doRemove = true - } - } else { - doRemove = true - } - if doRemove { - l4g.Error(utils.T("api.post.create_post.bad_filename.error"), path) - post.Filenames = append(post.Filenames[:i], post.Filenames[i+1:]...) - } - } - } - var rpost *model.Post if result := <-Srv.Store.Post().Save(post); result.Err != nil { return nil, result.Err } else { rpost = result.Data.(*model.Post) + } + + if len(post.FileIds) > 0 { + // There's a rare bug where the client sends up duplicate FileIds so protect against that + post.FileIds = utils.RemoveDuplicatesFromStringArray(post.FileIds) - go handlePostEvents(c, rpost, triggerWebhooks) + for _, fileId := range post.FileIds { + if result := <-Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil { + l4g.Error(utils.T("api.post.create_post.attach_files.error"), post.Id, post.FileIds, c.Session.UserId, result.Err) + } + } } + go handlePostEvents(c, rpost, triggerWebhooks) + return rpost, nil } @@ -566,6 +545,7 @@ func sendNotifications(c *Context, post *model.Post, team *model.Team, channel * pchan := Srv.Store.User().GetProfiles(c.TeamId) dpchan := Srv.Store.User().GetDirectProfiles(c.Session.UserId) mchan := Srv.Store.Channel().GetMembers(post.ChannelId) + fchan := Srv.Store.FileInfo().GetForPost(post.Id) var profileMap map[string]*model.User if result := <-pchan; result.Err != nil { @@ -785,12 +765,18 @@ func sendNotifications(c *Context, post *model.Post, team *model.Team, channel * message.Add("sender_name", senderName) message.Add("team_id", team.Id) - if len(post.Filenames) != 0 { + if len(post.FileIds) != 0 { message.Add("otherFile", "true") - for _, filename := range post.Filenames { - ext := filepath.Ext(filename) - if model.IsFileExtImage(ext) { + var infos []*model.FileInfo + if result := <-fchan; result.Err != nil { + l4g.Warn(utils.T("api.post.send_notifications.files.error"), post.Id, result.Err) + } else { + infos = result.Data.([]*model.FileInfo) + } + + for _, info := range infos { + if info.IsImage() { message.Add("image", "true") break } @@ -915,22 +901,29 @@ func sendNotificationEmail(c *Context, post *model.Post, user *model.User, chann } func getMessageForNotification(post *model.Post, translateFunc i18n.TranslateFunc) string { - if len(strings.TrimSpace(post.Message)) != 0 || len(post.Filenames) == 0 { + if len(strings.TrimSpace(post.Message)) != 0 || len(post.FileIds) == 0 { return post.Message } // extract the filenames from their paths and determine what type of files are attached - filenames := make([]string, len(post.Filenames)) + var infos []*model.FileInfo + if result := <-Srv.Store.FileInfo().GetForPost(post.Id); result.Err != nil { + l4g.Warn(utils.T("api.post.get_message_for_notification.get_files.error"), post.Id, result.Err) + } else { + infos = result.Data.([]*model.FileInfo) + } + + filenames := make([]string, len(infos)) onlyImages := true - for i, filename := range post.Filenames { - var err error - if filenames[i], err = url.QueryUnescape(filepath.Base(filename)); err != nil { + for i, info := range infos { + if escaped, err := url.QueryUnescape(filepath.Base(info.Name)); err != nil { // this should never error since filepath was escaped using url.QueryEscape - filenames[i] = filepath.Base(filename) + filenames[i] = escaped + } else { + filenames[i] = info.Name } - ext := filepath.Ext(filename) - onlyImages = onlyImages && model.IsFileExtImage(ext) + onlyImages = onlyImages && info.IsImage() } props := map[string]interface{}{"Filenames": strings.Join(filenames, ", ")} @@ -1099,9 +1092,6 @@ func SendEphemeralPost(teamId, userId string, post *model.Post) { if post.Props == nil { post.Props = model.StringInterface{} } - if post.Filenames == nil { - post.Filenames = []string{} - } message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_EPHEMERAL_MESSAGE, "", post.ChannelId, userId, nil) message.Add("post", post.ToJson()) @@ -1156,9 +1146,13 @@ func updatePost(c *Context, w http.ResponseWriter, r *http.Request) { } } - hashtags, _ := model.ParseHashtags(post.Message) + newPost := &model.Post{} + *newPost = *oldPost + + newPost.Message = post.Message + newPost.Hashtags, _ = model.ParseHashtags(post.Message) - if result := <-Srv.Store.Post().Update(oldPost, post.Message, hashtags); result.Err != nil { + if result := <-Srv.Store.Post().Update(newPost, oldPost); result.Err != nil { c.Err = result.Err return } else { @@ -1449,7 +1443,7 @@ func deletePost(c *Context, w http.ResponseWriter, r *http.Request) { message.Add("post", post.ToJson()) go Publish(message) - go DeletePostFiles(c.TeamId, post) + go DeletePostFiles(post) go DeleteFlaggedPost(c.Session.UserId, post) result := make(map[string]string) @@ -1465,17 +1459,13 @@ func DeleteFlaggedPost(userId string, post *model.Post) { } } -func DeletePostFiles(teamId string, post *model.Post) { - if len(post.Filenames) == 0 { +func DeletePostFiles(post *model.Post) { + if len(post.FileIds) != 0 { return } - prefix := "teams/" + teamId + "/channels/" + post.ChannelId + "/users/" + post.UserId + "/" - for _, filename := range post.Filenames { - splitUrl := strings.Split(filename, "/") - oldPath := prefix + splitUrl[len(splitUrl)-2] + "/" + splitUrl[len(splitUrl)-1] - newPath := prefix + splitUrl[len(splitUrl)-2] + "/deleted_" + splitUrl[len(splitUrl)-1] - MoveFile(oldPath, newPath) + if result := <-Srv.Store.FileInfo().DeleteForPost(post.Id); result.Err != nil { + l4g.Warn(utils.T("api.post.delete_post_files.app_error.warn"), post.Id, result.Err) } } @@ -1583,3 +1573,59 @@ func searchPosts(c *Context, w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") w.Write([]byte(posts.ToJson())) } + +func getFileInfosForPost(c *Context, w http.ResponseWriter, r *http.Request) { + params := mux.Vars(r) + + channelId := params["channel_id"] + if len(channelId) != 26 { + c.SetInvalidParam("getFileInfosForPost", "channelId") + return + } + + postId := params["post_id"] + if len(postId) != 26 { + c.SetInvalidParam("getFileInfosForPost", "postId") + return + } + + pchan := Srv.Store.Post().Get(postId) + fchan := Srv.Store.FileInfo().GetForPost(postId) + + if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_READ_CHANNEL) { + return + } + + var infos []*model.FileInfo + if result := <-fchan; result.Err != nil { + c.Err = result.Err + return + } else { + infos = result.Data.([]*model.FileInfo) + } + + if len(infos) == 0 { + // No FileInfos were returned so check if they need to be created for this post + var post *model.Post + if result := <-pchan; result.Err != nil { + c.Err = result.Err + return + } else { + post = result.Data.(*model.PostList).Posts[postId] + } + + if len(post.Filenames) > 0 { + // The post has Filenames that need to be replaced with FileInfos + infos = migrateFilenamesToFileInfos(post) + } + } + + etag := model.GetEtagForFileInfos(infos) + + if HandleEtag(etag, w, r) { + return + } else { + w.Header().Set(model.HEADER_ETAG_SERVER, etag) + w.Write([]byte(model.FileInfosToJson(infos))) + } +} -- cgit v1.2.3-1-g7c22