From 365b8b465e8a53ebb2da2bf3aef659ac81a2bc6a Mon Sep 17 00:00:00 2001 From: Joram Wilander Date: Wed, 19 Oct 2016 14:49:25 -0400 Subject: Merging performance branch into master (#4268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * improve performance on sendNotifications * Fix SQL queries * Remove get direct profiles, not needed anymore * Add raw data to error details if AppError fails to decode * men * Fix decode (#4052) * Fixing json decode * Adding unit test * Initial work for client scaling (#4051) * Begin adding paging to profiles API * Added more paging functionality * Finish hooking up admin console user lists * Add API for searching users and add searching to all user lists * Add lazy loading of profiles * Revert config.json * Fix unit tests and some style issues * Add GetProfilesFromList to Go driver and fix web unit test * Update etag for GetProfiles * Updating ui for filters and pagination (#4044) * Updating UI for pagination * Adjusting margins for filter row * Adjusting margin for specific modals * Adding relative padding to system console * Adjusting responsive view * Update client user tests * Minor fixes for direct messages modal (#4056) * Remove some unneeded initial load calls (#4057) * UX updates to user lists, added smart counts and bug fixes (#4059) * Improved getExplicitMentions and unit tests (#4064) * Refactor getting posts to lazy load profiles correctly (#4062) * Comment out SetActiveChannel test (#4066) * Profiler cpu, block, and memory profiler. (#4081) * Fix TestSetActiveChannel unit test (#4071) * Fixing build failure caused by dependancies updating (#4076) * Adding profiler * Fix admin_team_member_dropdown eslint errors * Bumping session cache size (#4077) * Bumping session cache size * Bumping status cache * Refactor how the client handles channel members to be large team friendly (#4106) * Refactor how the client handles channel members to be large team friendly * Change Id to ChannelId in ChannelStats model * Updated getChannelMember and getProfilesByIds routes to match proposal * Performance improvements (#4100) * Performance improvements * Fixing re-connect issue * Fixing error message * Some other minor perf tweaks * Some other minor perf tweaks * Fixing config file * Fixing buffer size * Fixing web socket send message * adding some error logging * fix getMe to be user required * Fix websocket event for new user * Fixing shutting down * Reverting web socket changes * Fixing logging lvl * Adding caching to GetMember * Adding some logging * Fixing caching * Fixing caching invalidate * Fixing direct message caching * Fixing caching * Fixing caching * Remove GetDirectProfiles from initial load * Adding logging and fixing websocket client * Adding back caching from bad merge. * Explicitly close go driver requests (#4162) * Refactored how the client handles team members to be more large team friendly (#4159) * Refactor getProfilesForDirectMessageList API into getAllProfiles API * Refactored how the client handles team members to be more large team friendly * Fix js error when receiving a notification * Fix JS error caused by current user being overwritten with sanitized version (#4165) * Adding error message to status failure (#4167) * Fix a few bugs caused by client scaling refactoring (#4170) * When there is no read replica, don't open a second set of connections to the master database (#4173) * Adding connection tacking to stats (#4174) * Reduce DB writes for statuses and other status related changes (#4175) * Fix bug preventing opening of DM channels from more modal (#4181) * Fixing socket timing error (#4183) * Fixing ping/pong handler * Fixing socket timing error * Commenting out status broadcasting * Removing user status changes * Removing user status changes * Removing user status changes * Removing user status changes * Adding DoPreComputeJson() * Performance improvements (#4194) * * Fix System Console Analytics queries * Add db.SetConnMaxLifetime to 15 minutes * Add "net/http/pprof" for profiling * Add FreeOSMemory() to manually release memory on reload config * Add flag to enable http profiler * Fix memory leak (#4197) * Fix memory leak * removed unneeded nil assignment * Fixing go routine leak (#4208) * Merge fixes * Merge fix * Refactored statuses to be queried by the client rather than broadcast by the server (#4212) * Refactored server code to reduce status broadcasts and to allow getting statuses by IDs * Refactor client code to periodically fetch statuses * Add store unit test for getting statuses by ids * Fix status unit test * Add getStatusesByIds REST API and move the client over to use that instead of the WebSocket * Adding multiple threads to websocket hub (#4230) * Adding multiple threads to websocket hub * Fixing unit tests * Fixing so websocket connections from the same user end up in the same… (#4240) * Fixing so websocket connections from the same user end up in the same list * Removing old comment * Refactor user autocomplete to query the server (#4239) * Add API for autocompleting users * Converted at mention autocomplete to query server * Converted user search autocomplete to query server * Switch autocomplete API naming to use term instead of username * Split autocomplete API into two, one for channels and for teams * Fix copy/paste error * Some final client scaling fixes (#4246) * Add lazy loading of profiles to integration pages * Add lazy loading of profiles to emoji page * Fix JS error when receiving post in select team menu and also clean up channel store --- api/user.go | 301 +++++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 259 insertions(+), 42 deletions(-) (limited to 'api/user.go') diff --git a/api/user.go b/api/user.go index 5f7e3ad10..12e57a33f 100644 --- a/api/user.go +++ b/api/user.go @@ -53,9 +53,15 @@ func InitUser() { BaseRoutes.Users.Handle("/newimage", ApiUserRequired(uploadProfileImage)).Methods("POST") BaseRoutes.Users.Handle("/me", ApiUserRequired(getMe)).Methods("GET") BaseRoutes.Users.Handle("/initial_load", ApiAppHandler(getInitialLoad)).Methods("GET") - BaseRoutes.Users.Handle("/direct_profiles", ApiUserRequired(getDirectProfiles)).Methods("GET") - BaseRoutes.Users.Handle("/profiles/{id:[A-Za-z0-9]+}", ApiUserRequired(getProfiles)).Methods("GET") - BaseRoutes.Users.Handle("/profiles_for_dm_list/{id:[A-Za-z0-9]+}", ApiUserRequired(getProfilesForDirectMessageList)).Methods("GET") + BaseRoutes.Users.Handle("/{offset:[0-9]+}/{limit:[0-9]+}", ApiUserRequired(getProfiles)).Methods("GET") + BaseRoutes.NeedTeam.Handle("/users/{offset:[0-9]+}/{limit:[0-9]+}", ApiUserRequired(getProfilesInTeam)).Methods("GET") + BaseRoutes.NeedChannel.Handle("/users/{offset:[0-9]+}/{limit:[0-9]+}", ApiUserRequired(getProfilesInChannel)).Methods("GET") + BaseRoutes.NeedChannel.Handle("/users/not_in_channel/{offset:[0-9]+}/{limit:[0-9]+}", ApiUserRequired(getProfilesNotInChannel)).Methods("GET") + BaseRoutes.Users.Handle("/search", ApiUserRequired(searchUsers)).Methods("POST") + BaseRoutes.Users.Handle("/ids", ApiUserRequired(getProfilesByIds)).Methods("POST") + + BaseRoutes.NeedTeam.Handle("/users/autocomplete", ApiUserRequired(autocompleteUsersInTeam)).Methods("GET") + BaseRoutes.NeedChannel.Handle("/users/autocomplete", ApiUserRequired(autocompleteUsersInChannel)).Methods("GET") BaseRoutes.Users.Handle("/mfa", ApiAppHandler(checkMfa)).Methods("POST") BaseRoutes.Users.Handle("/generate_mfa_qr", ApiUserRequiredTrustRequester(generateMfaQrCode)).Methods("GET") @@ -270,7 +276,9 @@ func CreateUser(user *model.User) (*model.User, *model.AppError) { ruser.Sanitize(map[string]bool{}) // This message goes to everyone, so the teamId, channelId and userId are irrelevant - go Publish(model.NewWebSocketEvent(model.WEBSOCKET_EVENT_NEW_USER, "", "", "", nil)) + message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_NEW_USER, "", "", "", nil) + message.Add("user_id", ruser.Id) + go Publish(message) return ruser, nil } @@ -379,7 +387,7 @@ func sendWelcomeEmail(c *Context, userId string, email string, siteURL string, v func addDirectChannels(teamId string, user *model.User) { var profiles map[string]*model.User - if result := <-Srv.Store.User().GetProfiles(teamId); result.Err != nil { + if result := <-Srv.Store.User().GetProfiles(teamId, 0, 100); result.Err != nil { l4g.Error(utils.T("api.user.add_direct_channels_and_forget.failed.error"), user.Id, teamId, result.Err.Error()) return } else { @@ -875,7 +883,6 @@ func getInitialLoad(c *Context, w http.ResponseWriter, r *http.Request) { uchan := Srv.Store.User().Get(c.Session.UserId) pchan := Srv.Store.Preference().GetAll(c.Session.UserId) tchan := Srv.Store.Team().GetTeamsByUserId(c.Session.UserId) - dpchan := Srv.Store.User().GetDirectProfiles(c.Session.UserId) il.TeamMembers = c.Session.TeamMembers @@ -904,19 +911,6 @@ func getInitialLoad(c *Context, w http.ResponseWriter, r *http.Request) { team.Sanitize() } } - - if dp := <-dpchan; dp.Err != nil { - c.Err = dp.Err - return - } else { - profiles := dp.Data.(map[string]*model.User) - - for k, p := range profiles { - profiles[k] = sanitizeProfile(c, p) - } - - il.DirectProfiles = profiles - } } if cchan != nil { @@ -960,25 +954,27 @@ func getUser(c *Context, w http.ResponseWriter, r *http.Request) { } } -func getProfilesForDirectMessageList(c *Context, w http.ResponseWriter, r *http.Request) { +func getProfiles(c *Context, w http.ResponseWriter, r *http.Request) { params := mux.Vars(r) - id := params["id"] - var pchan store.StoreChannel + offset, err := strconv.Atoi(params["offset"]) + if err != nil { + c.SetInvalidParam("getProfiles", "offset") + return + } - if *utils.Cfg.TeamSettings.RestrictDirectMessage == model.DIRECT_MESSAGE_TEAM { - if c.Session.GetTeamByTeamId(id) == nil { - if !HasPermissionToContext(c, model.PERMISSION_MANAGE_SYSTEM) { - return - } - } + limit, err := strconv.Atoi(params["limit"]) + if err != nil { + c.SetInvalidParam("getProfiles", "limit") + return + } - pchan = Srv.Store.User().GetProfiles(id) - } else { - pchan = Srv.Store.User().GetAllProfiles() + etag := (<-Srv.Store.User().GetEtagForAllProfiles()).Data.(string) + if HandleEtag(etag, w, r) { + return } - if result := <-pchan; result.Err != nil { + if result := <-Srv.Store.User().GetAllProfiles(offset, limit); result.Err != nil { c.Err = result.Err return } else { @@ -988,26 +984,39 @@ func getProfilesForDirectMessageList(c *Context, w http.ResponseWriter, r *http. profiles[k] = sanitizeProfile(c, p) } + w.Header().Set(model.HEADER_ETAG_SERVER, etag) w.Write([]byte(model.UserMapToJson(profiles))) } } -func getProfiles(c *Context, w http.ResponseWriter, r *http.Request) { +func getProfilesInTeam(c *Context, w http.ResponseWriter, r *http.Request) { params := mux.Vars(r) - id := params["id"] + teamId := params["team_id"] - if c.Session.GetTeamByTeamId(id) == nil { + if c.Session.GetTeamByTeamId(teamId) == nil { if !HasPermissionToContext(c, model.PERMISSION_MANAGE_SYSTEM) { return } } - etag := (<-Srv.Store.User().GetEtagForProfiles(id)).Data.(string) + offset, err := strconv.Atoi(params["offset"]) + if err != nil { + c.SetInvalidParam("getProfilesInTeam", "offset") + return + } + + limit, err := strconv.Atoi(params["limit"]) + if err != nil { + c.SetInvalidParam("getProfilesInTeam", "limit") + return + } + + etag := (<-Srv.Store.User().GetEtagForProfiles(teamId)).Data.(string) if HandleEtag(etag, w, r) { return } - if result := <-Srv.Store.User().GetProfiles(id); result.Err != nil { + if result := <-Srv.Store.User().GetProfiles(teamId, offset, limit); result.Err != nil { c.Err = result.Err return } else { @@ -1022,13 +1031,73 @@ func getProfiles(c *Context, w http.ResponseWriter, r *http.Request) { } } -func getDirectProfiles(c *Context, w http.ResponseWriter, r *http.Request) { - etag := (<-Srv.Store.User().GetEtagForDirectProfiles(c.Session.UserId)).Data.(string) - if HandleEtag(etag, w, r) { +func getProfilesInChannel(c *Context, w http.ResponseWriter, r *http.Request) { + params := mux.Vars(r) + channelId := params["channel_id"] + + if c.Session.GetTeamByTeamId(c.TeamId) == nil { + if !HasPermissionToContext(c, model.PERMISSION_MANAGE_SYSTEM) { + return + } + } + + if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_READ_CHANNEL) { + return + } + + offset, err := strconv.Atoi(params["offset"]) + if err != nil { + c.SetInvalidParam("getProfiles", "offset") return } - if result := <-Srv.Store.User().GetDirectProfiles(c.Session.UserId); result.Err != nil { + limit, err := strconv.Atoi(params["limit"]) + if err != nil { + c.SetInvalidParam("getProfiles", "limit") + return + } + + if result := <-Srv.Store.User().GetProfilesInChannel(channelId, offset, limit, false); result.Err != nil { + c.Err = result.Err + return + } else { + profiles := result.Data.(map[string]*model.User) + + for k, p := range profiles { + profiles[k] = sanitizeProfile(c, p) + } + + w.Write([]byte(model.UserMapToJson(profiles))) + } +} + +func getProfilesNotInChannel(c *Context, w http.ResponseWriter, r *http.Request) { + params := mux.Vars(r) + channelId := params["channel_id"] + + if c.Session.GetTeamByTeamId(c.TeamId) == nil { + if !HasPermissionToContext(c, model.PERMISSION_MANAGE_SYSTEM) { + return + } + } + + if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_READ_CHANNEL) { + return + } + + offset, err := strconv.Atoi(params["offset"]) + if err != nil { + c.SetInvalidParam("getProfiles", "offset") + return + } + + limit, err := strconv.Atoi(params["limit"]) + if err != nil { + c.SetInvalidParam("getProfiles", "limit") + return + } + + if result := <-Srv.Store.User().GetProfilesNotInChannel(c.TeamId, channelId, offset, limit); result.Err != nil { c.Err = result.Err return } else { @@ -1038,7 +1107,6 @@ func getDirectProfiles(c *Context, w http.ResponseWriter, r *http.Request) { profiles[k] = sanitizeProfile(c, p) } - w.Header().Set(model.HEADER_ETAG_SERVER, etag) w.Write([]byte(model.UserMapToJson(profiles))) } } @@ -2522,3 +2590,152 @@ func sanitizeProfile(c *Context, user *model.User) *model.User { return user } + +func searchUsers(c *Context, w http.ResponseWriter, r *http.Request) { + props := model.MapFromJson(r.Body) + + term := props["term"] + if len(term) == 0 { + c.SetInvalidParam("searchUsers", "term") + return + } + + teamId := props["team_id"] + inChannelId := props["in_channel"] + notInChannelId := props["not_in_channel"] + + if inChannelId != "" && !HasPermissionToChannelContext(c, inChannelId, model.PERMISSION_READ_CHANNEL) { + return + } + + if notInChannelId != "" && !HasPermissionToChannelContext(c, notInChannelId, model.PERMISSION_READ_CHANNEL) { + return + } + + var uchan store.StoreChannel + if inChannelId != "" { + uchan = Srv.Store.User().SearchInChannel(inChannelId, term, store.USER_SEARCH_TYPE_USERNAME) + } else if notInChannelId != "" { + uchan = Srv.Store.User().SearchNotInChannel(teamId, notInChannelId, term, store.USER_SEARCH_TYPE_USERNAME) + } else { + uchan = Srv.Store.User().Search(teamId, term, store.USER_SEARCH_TYPE_USERNAME) + } + + if result := <-uchan; result.Err != nil { + c.Err = result.Err + return + } else { + profiles := result.Data.([]*model.User) + + for _, p := range profiles { + sanitizeProfile(c, p) + } + + w.Write([]byte(model.UserListToJson(profiles))) + } +} + +func getProfilesByIds(c *Context, w http.ResponseWriter, r *http.Request) { + userIds := model.ArrayFromJson(r.Body) + + if len(userIds) == 0 { + c.SetInvalidParam("getProfilesByIds", "user_ids") + return + } + + if result := <-Srv.Store.User().GetProfileByIds(userIds); result.Err != nil { + c.Err = result.Err + return + } else { + profiles := result.Data.(map[string]*model.User) + + for _, p := range profiles { + sanitizeProfile(c, p) + } + + w.Write([]byte(model.UserMapToJson(profiles))) + } +} + +func autocompleteUsersInChannel(c *Context, w http.ResponseWriter, r *http.Request) { + params := mux.Vars(r) + channelId := params["channel_id"] + teamId := params["team_id"] + + term := r.URL.Query().Get("term") + + if c.Session.GetTeamByTeamId(teamId) == nil { + if !HasPermissionToContext(c, model.PERMISSION_MANAGE_SYSTEM) { + return + } + } + + if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_READ_CHANNEL) { + return + } + + uchan := Srv.Store.User().SearchInChannel(channelId, term, store.USER_SEARCH_TYPE_ALL) + nuchan := Srv.Store.User().SearchNotInChannel(teamId, channelId, term, store.USER_SEARCH_TYPE_ALL) + + autocomplete := &model.UserAutocompleteInChannel{} + + if result := <-uchan; result.Err != nil { + c.Err = result.Err + return + } else { + profiles := result.Data.([]*model.User) + + for _, p := range profiles { + sanitizeProfile(c, p) + } + + autocomplete.InChannel = profiles + } + + if result := <-nuchan; result.Err != nil { + c.Err = result.Err + return + } else { + profiles := result.Data.([]*model.User) + + for _, p := range profiles { + sanitizeProfile(c, p) + } + + autocomplete.OutOfChannel = profiles + } + + w.Write([]byte(autocomplete.ToJson())) +} + +func autocompleteUsersInTeam(c *Context, w http.ResponseWriter, r *http.Request) { + params := mux.Vars(r) + teamId := params["team_id"] + + term := r.URL.Query().Get("term") + + if c.Session.GetTeamByTeamId(teamId) == nil { + if !HasPermissionToContext(c, model.PERMISSION_MANAGE_SYSTEM) { + return + } + } + + uchan := Srv.Store.User().Search(teamId, term, store.USER_SEARCH_TYPE_ALL) + + autocomplete := &model.UserAutocompleteInTeam{} + + if result := <-uchan; result.Err != nil { + c.Err = result.Err + return + } else { + profiles := result.Data.([]*model.User) + + for _, p := range profiles { + sanitizeProfile(c, p) + } + + autocomplete.InTeam = profiles + } + + w.Write([]byte(autocomplete.ToJson())) +} -- cgit v1.2.3-1-g7c22