diff options
-rw-r--r-- | api/channel.go | 2 | ||||
-rw-r--r-- | api/import.go | 57 | ||||
-rw-r--r-- | api/slackimport.go | 246 | ||||
-rw-r--r-- | api/team.go | 71 | ||||
-rw-r--r-- | api/user.go | 3 | ||||
-rw-r--r-- | model/post.go | 4 | ||||
-rw-r--r-- | web/react/components/setting_upload.jsx | 79 | ||||
-rw-r--r-- | web/react/components/team_settings.jsx | 72 | ||||
-rw-r--r-- | web/react/components/team_settings_modal.jsx | 1 | ||||
-rw-r--r-- | web/react/utils/client.jsx | 16 | ||||
-rw-r--r-- | web/react/utils/utils.jsx | 9 |
11 files changed, 556 insertions, 4 deletions
diff --git a/api/channel.go b/api/channel.go index 151627623..de15b3610 100644 --- a/api/channel.go +++ b/api/channel.go @@ -397,7 +397,7 @@ func JoinChannel(c *Context, channelId string, role string) { } } -func JoinDefaultChannels(c *Context, user *model.User, channelRole string) *model.AppError { +func JoinDefaultChannels(user *model.User, channelRole string) *model.AppError { // We don't call JoinChannel here since c.Session is not populated on user creation var err *model.AppError = nil diff --git a/api/import.go b/api/import.go new file mode 100644 index 000000000..e3f314d20 --- /dev/null +++ b/api/import.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved. +// See License.txt for license information. + +package api + +import ( + l4g "code.google.com/p/log4go" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" +) + +// +// Import functions are sutible for entering posts and users into the database without +// some of the usual checks. (IsValid is still run) +// + +func ImportPost(post *model.Post) { + post.Hashtags, _ = model.ParseHashtags(post.Message) + + if result := <-Srv.Store.Post().Save(post); result.Err != nil { + l4g.Debug("Error saving post. user=" + post.UserId + ", message=" + post.Message) + } +} + +func ImportUser(user *model.User) *model.User { + user.MakeNonNil() + if len(user.Props["theme"]) == 0 { + user.AddProp("theme", utils.Cfg.TeamSettings.DefaultThemeColor) + } + + if result := <-Srv.Store.User().Save(user); result.Err != nil { + l4g.Error("Error saving user. err=%v", result.Err) + return nil + } else { + ruser := result.Data.(*model.User) + + if err := JoinDefaultChannels(ruser, ""); err != nil { + l4g.Error("Encountered an issue joining default channels user_id=%s, team_id=%s, err=%v", ruser.Id, ruser.TeamId, err) + } + + if cresult := <-Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil { + l4g.Error("Failed to set email verified err=%v", cresult.Err) + } + + return ruser + } +} + +func ImportChannel(channel *model.Channel) *model.Channel { + if result := <-Srv.Store.Channel().Save(channel); result.Err != nil { + return nil + } else { + sc := result.Data.(*model.Channel) + + return sc + } +} diff --git a/api/slackimport.go b/api/slackimport.go new file mode 100644 index 000000000..c662181cb --- /dev/null +++ b/api/slackimport.go @@ -0,0 +1,246 @@ +// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved. +// See License.txt for license information. + +package api + +import ( + "archive/zip" + "bytes" + l4g "code.google.com/p/log4go" + "encoding/json" + "github.com/mattermost/platform/model" + "io" + "mime/multipart" + "strconv" + "strings" +) + +type SlackChannel struct { + Id string `json:"id"` + Name string `json:"name"` + Members []string `json:"members"` + Topic map[string]string `json:"topic"` +} + +type SlackUser struct { + Id string `json:"id"` + UserName string `json:"name"` + Profile map[string]string `json:"profile"` +} + +type SlackPost struct { + User string `json:"user"` + BotId string `json:"bot_id"` + BotUsername string `json:"username"` + Text string `json:"text"` + TimeStamp string `json:"ts"` + Type string `json:"type"` + SubType string `json:"subtype"` + Comment map[string]string `json:"comment"` +} + +func SlackConvertTimeStamp(ts string) int64 { + timeString := strings.SplitN(ts, ".", 2)[0] + + timeStamp, err := strconv.ParseInt(timeString, 10, 64) + if err != nil { + l4g.Warn("Bad timestamp detected") + return 1 + } + return timeStamp * 1000 // Convert to milliseconds +} + +func SlackParseChannels(data io.Reader) []SlackChannel { + decoder := json.NewDecoder(data) + + var channels []SlackChannel + if err := decoder.Decode(&channels); err != nil { + return make([]SlackChannel, 0) + } + return channels +} + +func SlackParseUsers(data io.Reader) []SlackUser { + decoder := json.NewDecoder(data) + + var users []SlackUser + if err := decoder.Decode(&users); err != nil { + return make([]SlackUser, 0) + } + return users +} + +func SlackParsePosts(data io.Reader) []SlackPost { + decoder := json.NewDecoder(data) + + var posts []SlackPost + if err := decoder.Decode(&posts); err != nil { + return make([]SlackPost, 0) + } + return posts +} + +func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map[string]*model.User { + // Log header + log.WriteString("\n Users Created\n") + log.WriteString("===============\n\n") + + // Add users + addedUsers := make(map[string]*model.User) + for _, sUser := range slackusers { + firstName := "" + lastName := "" + if name, ok := sUser.Profile["first_name"]; ok { + firstName = name + } + if name, ok := sUser.Profile["last_name"]; ok { + lastName = name + } + + password := model.NewId() + + newUser := model.User{ + TeamId: teamId, + Username: sUser.UserName, + FirstName: firstName, + LastName: lastName, + Email: sUser.Profile["email"], + Password: password, + } + + if mUser := ImportUser(&newUser); mUser != nil { + addedUsers[sUser.Id] = mUser + log.WriteString("Email, Password: " + newUser.Email + ", " + password + "\n") + } else { + log.WriteString("Unable to import user: " + sUser.UserName) + } + } + + return addedUsers +} + +func SlackAddPosts(channel *model.Channel, posts []SlackPost, users map[string]*model.User) { + for _, sPost := range posts { + switch { + case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"): + if sPost.User == "" { + l4g.Debug("Message without user") + continue + } else if users[sPost.User] == nil { + l4g.Debug("User: " + sPost.User + " does not exist!") + continue + } + newPost := model.Post{ + UserId: users[sPost.User].Id, + ChannelId: channel.Id, + Message: sPost.Text, + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + } + ImportPost(&newPost) + case sPost.Type == "message" && sPost.SubType == "file_comment": + if sPost.Comment["user"] == "" { + l4g.Debug("Message without user") + continue + } else if users[sPost.Comment["user"]] == nil { + l4g.Debug("User: " + sPost.User + " does not exist!") + continue + } + newPost := model.Post{ + UserId: users[sPost.Comment["user"]].Id, + ChannelId: channel.Id, + Message: sPost.Comment["comment"], + CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + } + ImportPost(&newPost) + case sPost.Type == "message" && sPost.SubType == "bot_message": + // In the future this will use the "Action Post" spec to post + // a message without using a username. For now we just warn that we don't handle this case + l4g.Warn("Slack bot posts are not imported yet") + default: + l4g.Warn("Unsupported post type: " + sPost.Type + ", " + sPost.SubType) + } + } +} + +func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, log *bytes.Buffer) map[string]*model.Channel { + // Write Header + log.WriteString("\n Channels Added \n") + log.WriteString("=================\n\n") + + // Add channels + addedChannels := make(map[string]*model.Channel) + for _, sChannel := range slackchannels { + newChannel := model.Channel{ + TeamId: teamId, + Type: model.CHANNEL_OPEN, + DisplayName: sChannel.Name, + Name: sChannel.Name, + Description: sChannel.Topic["value"], + } + mChannel := ImportChannel(&newChannel) + if mChannel == nil { + // Maybe it already exists? + if result := <-Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil { + l4g.Debug("Failed to import: %s", newChannel.DisplayName) + log.WriteString("Failed to import: " + newChannel.DisplayName + "\n") + continue + } else { + mChannel = result.Data.(*model.Channel) + log.WriteString("Merged with existing channel: " + newChannel.DisplayName + "\n") + } + } + log.WriteString(newChannel.DisplayName + "\n") + addedChannels[sChannel.Id] = mChannel + SlackAddPosts(mChannel, posts[sChannel.Name], users) + } + + return addedChannels +} + +func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { + zipreader, err := zip.NewReader(fileData, fileSize) + if err != nil || zipreader.File == nil { + return model.NewAppError("SlackImport", "Unable to open zip file", err.Error()), nil + } + + // Create log file + log := bytes.NewBufferString("Mattermost Slack Import Log\n") + + var channels []SlackChannel + var users []SlackUser + posts := make(map[string][]SlackPost) + for _, file := range zipreader.File { + reader, err := file.Open() + if err != nil { + return model.NewAppError("SlackImport", "Unable to open: "+file.Name, err.Error()), log + } + if file.Name == "channels.json" { + channels = SlackParseChannels(reader) + } else if file.Name == "users.json" { + users = SlackParseUsers(reader) + } else { + spl := strings.Split(file.Name, "/") + if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") { + newposts := SlackParsePosts(reader) + channel := spl[0] + if _, ok := posts[channel]; ok == false { + posts[channel] = newposts + } else { + posts[channel] = append(posts[channel], newposts...) + } + } + + } + } + + addedUsers := SlackAddUsers(teamID, users, log) + SlackAddChannels(teamID, channels, posts, addedUsers, log) + + log.WriteString("\n Notes \n") + log.WriteString("=======\n\n") + + log.WriteString("- Some posts may not have been imported because they where not supported by this importer.\n") + log.WriteString("- Slack bot posts are currently not supported.\n") + + return nil, log +} diff --git a/api/team.go b/api/team.go index e6b8f4e14..12d087aea 100644 --- a/api/team.go +++ b/api/team.go @@ -4,6 +4,7 @@ package api import ( + "bytes" l4g "code.google.com/p/log4go" "fmt" "github.com/gorilla/mux" @@ -13,6 +14,7 @@ import ( "net/url" "strconv" "strings" + "time" ) func InitTeam(r *mux.Router) { @@ -29,6 +31,7 @@ func InitTeam(r *mux.Router) { sr.Handle("/update_name", ApiUserRequired(updateTeamDisplayName)).Methods("POST") sr.Handle("/update_valet_feature", ApiUserRequired(updateValetFeature)).Methods("POST") sr.Handle("/me", ApiUserRequired(getMyTeam)).Methods("GET") + sr.Handle("/import_team", ApiUserRequired(importTeam)).Methods("POST") } func signupTeam(c *Context, w http.ResponseWriter, r *http.Request) { @@ -489,3 +492,71 @@ func getMyTeam(c *Context, w http.ResponseWriter, r *http.Request) { return } } + +func importTeam(c *Context, w http.ResponseWriter, r *http.Request) { + + if !c.HasPermissionsToTeam(c.Session.TeamId, "import") { + c.Err = model.NewAppError("importTeam", "Only a team admin can import data.", "userId="+c.Session.UserId) + c.Err.StatusCode = http.StatusForbidden + return + } + + if err := r.ParseMultipartForm(10000000); err != nil { + c.Err = model.NewAppError("importTeam", "Could not parse multipart form", err.Error()) + return + } + + importFromArray, ok := r.MultipartForm.Value["importFrom"] + importFrom := importFromArray[0] + + fileSizeStr, ok := r.MultipartForm.Value["filesize"] + if !ok { + c.Err = model.NewAppError("importTeam", "Filesize unavilable", "") + c.Err.StatusCode = http.StatusBadRequest + return + } + + fileSize, err := strconv.ParseInt(fileSizeStr[0], 10, 64) + if err != nil { + c.Err = model.NewAppError("importTeam", "Filesize not an integer", "") + c.Err.StatusCode = http.StatusBadRequest + return + } + + fileInfoArray, ok := r.MultipartForm.File["file"] + if !ok { + c.Err = model.NewAppError("importTeam", "No file under 'file' in request", "") + c.Err.StatusCode = http.StatusBadRequest + return + } + + if len(fileInfoArray) <= 0 { + c.Err = model.NewAppError("importTeam", "Empty array under 'file' in request", "") + c.Err.StatusCode = http.StatusBadRequest + return + } + + fileInfo := fileInfoArray[0] + + fileData, err := fileInfo.Open() + defer fileData.Close() + if err != nil { + c.Err = model.NewAppError("importTeam", "Could not open file", err.Error()) + c.Err.StatusCode = http.StatusBadRequest + return + } + + var log *bytes.Buffer + switch importFrom { + case "slack": + var err *model.AppError + if err, log = SlackImport(fileData, fileSize, c.Session.TeamId); err != nil { + c.Err = err + c.Err.StatusCode = http.StatusBadRequest + } + } + + w.Header().Set("Content-Disposition", "attachment; filename=MattermostImportLog.txt") + w.Header().Set("Content-Type", "application/octet-stream") + http.ServeContent(w, r, "MattermostImportLog.txt", time.Now(), bytes.NewReader(log.Bytes())) +} diff --git a/api/user.go b/api/user.go index 2e71ddfc6..05ccd03e8 100644 --- a/api/user.go +++ b/api/user.go @@ -181,12 +181,13 @@ func CreateUser(c *Context, team *model.Team, user *model.User) *model.User { if result := <-Srv.Store.User().Save(user); result.Err != nil { c.Err = result.Err + l4g.Error("Filae err=%v", result.Err) return nil } else { ruser := result.Data.(*model.User) // Soft error if there is an issue joining the default channels - if err := JoinDefaultChannels(c, ruser, channelRole); err != nil { + if err := JoinDefaultChannels(ruser, channelRole); err != nil { l4g.Error("Encountered an issue joining default channels user_id=%s, team_id=%s, err=%v", ruser.Id, ruser.TeamId, err) } diff --git a/model/post.go b/model/post.go index 0c035d4e7..f8a3032a3 100644 --- a/model/post.go +++ b/model/post.go @@ -120,7 +120,9 @@ func (o *Post) PreSave() { o.OriginalId = "" - o.CreateAt = GetMillis() + if o.CreateAt <= 0 { + o.CreateAt = GetMillis() + } o.UpdateAt = o.CreateAt if o.Props == nil { diff --git a/web/react/components/setting_upload.jsx b/web/react/components/setting_upload.jsx new file mode 100644 index 000000000..870710850 --- /dev/null +++ b/web/react/components/setting_upload.jsx @@ -0,0 +1,79 @@ +// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved. +// See License.txt for license information. + +module.exports = React.createClass({ + displayName: 'Setting Upload', + propTypes: { + title: React.PropTypes.string.isRequired, + submit: React.PropTypes.func.isRequired, + fileTypesAccepted: React.PropTypes.string.isRequired, + clientError: React.PropTypes.string, + serverError: React.PropTypes.string + }, + getInitialState: function() { + return { + clientError: this.props.clientError, + serverError: this.props.serverError + }; + }, + componentWillReceiveProps: function() { + this.setState({ + clientError: this.props.clientError, + serverError: this.props.serverError + }); + }, + doFileSelect: function(e) { + e.preventDefault(); + this.setState({ + clientError: '', + serverError: '' + }); + }, + doSubmit: function(e) { + e.preventDefault(); + var inputnode = this.refs.uploadinput.getDOMNode(); + if (inputnode.files && inputnode.files[0]) { + this.props.submit(inputnode.files[0]); + } else { + this.setState({clientError: 'No file selected.'}); + } + }, + doCancel: function(e) { + e.preventDefault(); + this.refs.uploadinput.getDOMNode().value = ''; + this.setState({ + clientError: '', + serverError: '' + }); + }, + render: function() { + var clientError = null; + if (this.state.clientError) { + clientError = ( + <div className='form-group has-error'><label className='control-label'>{this.state.clientError}</label></div> + ); + } + var serverError = null; + if (this.state.serverError) { + serverError = ( + <div className='form-group has-error'><label className='control-label'>{this.state.serverError}</label></div> + ); + } + return ( + <ul className='section-max'> + <li className='col-xs-12 section-title'>{this.props.title}</li> + <li className='col-xs-offset-3 col-xs-8'> + <ul className='setting-list'> + <li className='setting-list-item'> + {serverError} + {clientError} + <span className='btn btn-sm btn-primary btn-file sel-btn'>SelectFile<input ref='uploadinput' accept={this.props.fileTypesAccepted} type='file' onChange={this.onFileSelect}/></span> + <a className={'btn btn-sm btn-primary'} onClick={this.doSubmit}>Import</a> + <a className='btn btn-sm theme' href='#' onClick={this.doCancel}>Cancel</a> + </li> + </ul> + </li> + </ul> + ); + } +}); diff --git a/web/react/components/team_settings.jsx b/web/react/components/team_settings.jsx index 3bbb5e892..433ec9177 100644 --- a/web/react/components/team_settings.jsx +++ b/web/react/components/team_settings.jsx @@ -6,6 +6,7 @@ var TeamStore = require('../stores/team_store.jsx'); var SettingItemMin = require('./setting_item_min.jsx'); var SettingItemMax = require('./setting_item_max.jsx'); var SettingPicture = require('./setting_picture.jsx'); +var SettingUpload = require('./setting_upload.jsx'); var utils = require('../utils/utils.jsx'); var client = require('../utils/client.jsx'); @@ -126,6 +127,69 @@ var FeatureTab = React.createClass({ } }); +var ImportTab = React.createClass({ + getInitialState: function() { + return {status: 'ready'}; + }, + onImportFailure: function() { + this.setState({status: 'fail'}); + }, + onImportSuccess: function(data) { + this.setState({status: 'done'}); + }, + doImportSlack: function(file) { + this.setState({status: 'in-progress'}); + utils.importSlack(file, this.onImportSuccess, this.onImportFailure); + }, + render: function() { + + uploadSection = ( + <SettingUpload + title="Import from Slack" + submit={this.doImportSlack} + fileTypesAccepted='.zip'/> + ); + + var messageSection; + switch (this.state.status) { + case 'ready': + messageSection = ''; + break; + case 'in-progress': + messageSection = ( + <p>Importing...</p> + ); + break; + case 'done': + messageSection = ( + <p>Import sucessfull: <a href={this.state.link} download='MattermostImportSummery.txt'>View Summery</a></p> + ); + break; + case 'fail': + messageSection = ( + <p>Import failure: <a href={this.state.link} download='MattermostImportSummery.txt'>View Summery</a></p> + ); + break; + } + + return ( + <div> + <div className="modal-header"> + <button type="button" className="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> + <h4 className="modal-title" ref="title"><i className="modal-back"></i>Import</h4> + </div> + <div ref="wrapper" className="user-settings"> + <h3 className="tab-header">Import</h3> + <div className="divider-dark first"/> + {uploadSection} + {messageSection} + <div className="divider-dark"/> + </div> + </div> + ); + } +}); + module.exports = React.createClass({ componentDidMount: function() { TeamStore.addChangeListener(this._onChange); @@ -154,7 +218,13 @@ module.exports = React.createClass({ <FeatureTab team={this.state.team} activeSection={this.props.activeSection} updateSection={this.props.updateSection} /> </div> ); - } else { + } else if (this.props.activeTab === 'import') { + return ( + <div> + <ImportTab team={this.state.team} activeSection={this.props.activeSection} updateSection={this.props.updateSection} /> + </div> + ); + } else { return <div/>; } } diff --git a/web/react/components/team_settings_modal.jsx b/web/react/components/team_settings_modal.jsx index b1c38fd16..f935daf82 100644 --- a/web/react/components/team_settings_modal.jsx +++ b/web/react/components/team_settings_modal.jsx @@ -27,6 +27,7 @@ module.exports = React.createClass({ render: function() { var tabs = []; tabs.push({name: "feature", ui_name: "Features", icon: "glyphicon glyphicon-wrench"}); + tabs.push({name: "import", ui_name: "Import", icon: "glyphicon glyphicon-upload"}); return ( <div className="modal fade" ref="modal" id="team_settings" role="dialog" tabIndex="-1" aria-hidden="true"> diff --git a/web/react/utils/client.jsx b/web/react/utils/client.jsx index da0b74081..f4c72ba46 100644 --- a/web/react/utils/client.jsx +++ b/web/react/utils/client.jsx @@ -862,6 +862,22 @@ module.exports.uploadProfileImage = function(imageData, success, error) { }); }; +module.exports.importSlack = function(fileData, success, error) { + $.ajax({ + url: "/api/v1/teams/import_team", + type: 'POST', + data: fileData, + cache: false, + contentType: false, + processData: false, + success: success, + error: function(xhr, status, err) { + e = handleError("importTeam", xhr, status, err); + error(e); + } + }); +} + module.exports.getStatuses = function(success, error) { $.ajax({ url: "/api/v1/users/status", diff --git a/web/react/utils/utils.jsx b/web/react/utils/utils.jsx index 618cc1557..f0a343513 100644 --- a/web/react/utils/utils.jsx +++ b/web/react/utils/utils.jsx @@ -982,3 +982,12 @@ module.exports.getUserIdFromChannelName = function(channel) { return otherUserId; }; + +module.exports.importSlack = function(file, success, error) { + formData = new FormData(); + formData.append('file', file, file.name); + formData.append('filesize', file.size); + formData.append('importFrom', 'slack'); + + client.importSlack(formData, success, error); +}; |