diff options
-rw-r--r-- | cmd/mattermost/commands/message_export.go | 74 | ||||
-rw-r--r-- | cmd/mattermost/commands/message_export_test.go | 8 | ||||
-rw-r--r-- | cmd/mattermost/commands/sampledata.go | 2 | ||||
-rw-r--r-- | einterfaces/message_export.go | 1 | ||||
-rw-r--r-- | i18n/en.json | 26 | ||||
-rw-r--r-- | model/config.go | 3 | ||||
-rw-r--r-- | model/message_export.go | 16 | ||||
-rw-r--r-- | store/sqlstore/compliance_store.go | 8 | ||||
-rw-r--r-- | utils/file_backend.go | 1 | ||||
-rw-r--r-- | utils/file_backend_local.go | 8 | ||||
-rw-r--r-- | utils/file_backend_s3.go | 12 |
11 files changed, 129 insertions, 30 deletions
diff --git a/cmd/mattermost/commands/message_export.go b/cmd/mattermost/commands/message_export.go index 41b4fd289..ee1a7ef7f 100644 --- a/cmd/mattermost/commands/message_export.go +++ b/cmd/mattermost/commands/message_export.go @@ -15,21 +15,48 @@ import ( ) var MessageExportCmd = &cobra.Command{ - Use: "export", - Short: "Export data from Mattermost", - Long: "Export data from Mattermost in a format suitable for import into a third-party application", - Example: "export --format=actiance --exportFrom=12345", - RunE: messageExportCmdF, + Use: "export", + Short: "Export data from Mattermost", + Long: "Export data from Mattermost in a format suitable for import into a third-party application", +} + +var ScheduleExportCmd = &cobra.Command{ + Use: "schedule", + Short: "Schedule an export data job in Mattermost", + Long: "Schedule an export data job in Mattermost (this will run asynchronously via a background worker)", + Example: "export schedule --format=actiance --exportFrom=12345 --timeoutSeconds=12345", + RunE: scheduleExportCmdF, +} + +var CsvExportCmd = &cobra.Command{ + Use: "csv", + Short: "Export data from Mattermost in CSV format", + Long: "Export data from Mattermost in CSV format", + Example: "export csv --exportFrom=12345", + RunE: buildExportCmdF("csv"), +} + +var ActianceExportCmd = &cobra.Command{ + Use: "actiance", + Short: "Export data from Mattermost in Actiance format", + Long: "Export data from Mattermost in Actiance format", + Example: "export actiance --exportFrom=12345", + RunE: buildExportCmdF("actiance"), } func init() { - MessageExportCmd.Flags().String("format", "actiance", "The format to export data in") - MessageExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") - MessageExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.") + ScheduleExportCmd.Flags().String("format", "actiance", "The format to export data") + ScheduleExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") + ScheduleExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.") + CsvExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") + ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") + MessageExportCmd.AddCommand(ScheduleExportCmd) + MessageExportCmd.AddCommand(CsvExportCmd) + MessageExportCmd.AddCommand(ActianceExportCmd) RootCmd.AddCommand(MessageExportCmd) } -func messageExportCmdF(command *cobra.Command, args []string) error { +func scheduleExportCmdF(command *cobra.Command, args []string) error { a, err := InitDBCommandContextCobra(command) if err != nil { return err @@ -79,3 +106,32 @@ func messageExportCmdF(command *cobra.Command, args []string) error { return nil } + +func buildExportCmdF(format string) func(command *cobra.Command, args []string) error { + return func(command *cobra.Command, args []string) error { + a, err := InitDBCommandContextCobra(command) + if err != nil { + return err + } + defer a.Shutdown() + + startTime, err := command.Flags().GetInt64("exportFrom") + if err != nil { + return errors.New("exportFrom flag error") + } else if startTime < 0 { + return errors.New("exportFrom must be a positive integer") + } + + if a.MessageExport == nil { + CommandPrettyPrintln("MessageExport feature not available") + } + + err2 := a.MessageExport.RunExport(format, startTime) + if err2 != nil { + return err2 + } + CommandPrettyPrintln("SUCCESS: Your data was exported.") + + return nil + } +} diff --git a/cmd/mattermost/commands/message_export_test.go b/cmd/mattermost/commands/message_export_test.go index 7572d8b48..89ef45a6a 100644 --- a/cmd/mattermost/commands/message_export_test.go +++ b/cmd/mattermost/commands/message_export_test.go @@ -24,7 +24,7 @@ func TestMessageExportNotEnabled(t *testing.T) { defer os.RemoveAll(filepath.Dir(configPath)) // should fail fast because the feature isn't enabled - require.Error(t, RunCommand(t, "--config", configPath, "export")) + require.Error(t, RunCommand(t, "--config", configPath, "export", "schedule")) } func TestMessageExportInvalidFormat(t *testing.T) { @@ -32,7 +32,7 @@ func TestMessageExportInvalidFormat(t *testing.T) { defer os.RemoveAll(filepath.Dir(configPath)) // should fail fast because format isn't supported - require.Error(t, RunCommand(t, "--config", configPath, "--format", "not_actiance", "export")) + require.Error(t, RunCommand(t, "--config", configPath, "--format", "not_actiance", "export", "schedule")) } func TestMessageExportNegativeExportFrom(t *testing.T) { @@ -40,7 +40,7 @@ func TestMessageExportNegativeExportFrom(t *testing.T) { defer os.RemoveAll(filepath.Dir(configPath)) // should fail fast because export from must be a valid timestamp - require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export")) + require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export", "schedule")) } func TestMessageExportNegativeTimeoutSeconds(t *testing.T) { @@ -48,7 +48,7 @@ func TestMessageExportNegativeTimeoutSeconds(t *testing.T) { defer os.RemoveAll(filepath.Dir(configPath)) // should fail fast because timeout seconds must be a positive int - require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export")) + require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export", "schedule")) } func writeTempConfig(t *testing.T, isMessageExportEnabled bool) string { diff --git a/cmd/mattermost/commands/sampledata.go b/cmd/mattermost/commands/sampledata.go index 0051679eb..0983ab0df 100644 --- a/cmd/mattermost/commands/sampledata.go +++ b/cmd/mattermost/commands/sampledata.go @@ -56,7 +56,7 @@ func sliceIncludes(vs []string, t string) bool { func randomPastTime(seconds int) int64 { now := time.Now() today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.FixedZone("UTC", 0)) - return today.Unix() - int64(rand.Intn(seconds*1000)) + return (today.Unix() * 1000) - int64(rand.Intn(seconds*1000)) } func randomEmoji() string { diff --git a/einterfaces/message_export.go b/einterfaces/message_export.go index ba498cdfb..8fde65070 100644 --- a/einterfaces/message_export.go +++ b/einterfaces/message_export.go @@ -11,4 +11,5 @@ import ( type MessageExportInterface interface { StartSynchronizeJob(ctx context.Context, exportFromTimestamp int64) (*model.Job, *model.AppError) + RunExport(format string, since int64) *model.AppError } diff --git a/i18n/en.json b/i18n/en.json index 3639aaeea..b4daa6f58 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1046,11 +1046,19 @@ }, { "id": "api.file.read_file.reading_local.app_error", - "translation": "Encountered an error reading from local server storage" + "translation": "Encountered an error reading from local server file storage" }, { "id": "api.file.read_file.s3.app_error", - "translation": "" + "translation": "Encountered an error reading from S3 storage" + }, + { + "id": "api.file.reader.reading_local.app_error", + "translation": "Encountered an error opening a reader from local server file storage" + }, + { + "id": "api.file.reader.s3.app_error", + "translation": "Encountered an error opening a reader from S3 storage" }, { "id": "api.file.test_connection.local.connection.app_error", @@ -3580,27 +3588,27 @@ }, { "id": "ent.compliance.bad_export_type.appError", - "translation": "" + "translation": "Unknown output format {{.ExportType}}" }, { "id": "ent.compliance.csv.attachment.copy.appError", - "translation": "" + "translation": "Unable to copy the attachment into the zip file." }, { "id": "ent.compliance.csv.attachment.export.appError", - "translation": "" + "translation": "Unable to add attachment to the CSV export." }, { "id": "ent.compliance.csv.file.creation.appError", - "translation": "" + "translation": "Unable to create temporary CSV export file." }, { "id": "ent.compliance.csv.header.export.appError", - "translation": "" + "translation": "Unable to add header to the CSV export." }, { "id": "ent.compliance.csv.metadata.export.appError", - "translation": "" + "translation": "Unable to add metadata file to the zip file." }, { "id": "ent.compliance.csv.metadata.json.marshalling.appError", @@ -4444,7 +4452,7 @@ }, { "id": "model.config.is_valid.message_export.export_type.app_error", - "translation": "Message export job ExportFormat must be one of either 'actiance' or 'globalrelay'" + "translation": "Message export job ExportFormat must be one of 'actiance', 'csv' or 'globalrelay'" }, { "id": "model.config.is_valid.message_export.global_relay.config_missing.app_error", diff --git a/model/config.go b/model/config.go index 0931cff87..868bb01d5 100644 --- a/model/config.go +++ b/model/config.go @@ -156,6 +156,7 @@ const ( TIMEZONE_SETTINGS_DEFAULT_SUPPORTED_TIMEZONES_PATH = "timezones.json" + COMPLIANCE_EXPORT_TYPE_CSV = "csv" COMPLIANCE_EXPORT_TYPE_ACTIANCE = "actiance" COMPLIANCE_EXPORT_TYPE_GLOBALRELAY = "globalrelay" GLOBALRELAY_CUSTOMER_TYPE_A9 = "A9" @@ -2366,7 +2367,7 @@ func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError { return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest) } else if mes.BatchSize == nil || *mes.BatchSize < 0 { return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest) - } else if mes.ExportFormat == nil || (*mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_ACTIANCE && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_GLOBALRELAY) { + } else if mes.ExportFormat == nil || (*mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_ACTIANCE && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_GLOBALRELAY && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_CSV) { return NewAppError("Config.IsValid", "model.config.is_valid.message_export.export_type.app_error", nil, "", http.StatusBadRequest) } diff --git a/model/message_export.go b/model/message_export.go index 6efb8c6a4..7ac50db25 100644 --- a/model/message_export.go +++ b/model/message_export.go @@ -4,7 +4,12 @@ package model type MessageExport struct { + TeamId *string + TeamName *string + TeamDisplayName *string + ChannelId *string + ChannelName *string ChannelDisplayName *string ChannelType *string @@ -12,9 +17,10 @@ type MessageExport struct { UserEmail *string Username *string - PostId *string - PostCreateAt *int64 - PostMessage *string - PostType *string - PostFileIds StringArray + PostId *string + PostCreateAt *int64 + PostMessage *string + PostType *string + PostOriginalId *string + PostFileIds StringArray } diff --git a/store/sqlstore/compliance_store.go b/store/sqlstore/compliance_store.go index c3c75581e..52bdee693 100644 --- a/store/sqlstore/compliance_store.go +++ b/store/sqlstore/compliance_store.go @@ -223,13 +223,18 @@ func (s SqlComplianceStore) MessageExport(after int64, limit int) store.StoreCha Posts.CreateAt AS PostCreateAt, Posts.Message AS PostMessage, Posts.Type AS PostType, + Posts.OriginalId AS PostOriginalId, Posts.FileIds AS PostFileIds, + Teams.Id AS TeamId, + Teams.Name AS TeamName, + Teams.DisplayName AS TeamDisplayName, Channels.Id AS ChannelId, - CASE + CASE WHEN Channels.Type = 'D' THEN 'Direct Message' WHEN Channels.Type = 'G' THEN 'Group Message' ELSE Channels.DisplayName END AS ChannelDisplayName, + Channels.Name AS ChannelName, Channels.Type AS ChannelType, Users.Id AS UserId, Users.Email AS UserEmail, @@ -237,6 +242,7 @@ func (s SqlComplianceStore) MessageExport(after int64, limit int) store.StoreCha FROM Posts LEFT OUTER JOIN Channels ON Posts.ChannelId = Channels.Id + LEFT OUTER JOIN Teams ON Channels.TeamId = Teams.Id LEFT OUTER JOIN Users ON Posts.UserId = Users.Id WHERE Posts.CreateAt > :StartTime AND diff --git a/utils/file_backend.go b/utils/file_backend.go index 60c90960d..9ed564592 100644 --- a/utils/file_backend.go +++ b/utils/file_backend.go @@ -13,6 +13,7 @@ import ( type FileBackend interface { TestConnection() *model.AppError + Reader(path string) (io.ReadCloser, *model.AppError) ReadFile(path string) ([]byte, *model.AppError) CopyFile(oldPath, newPath string) *model.AppError MoveFile(oldPath, newPath string) *model.AppError diff --git a/utils/file_backend_local.go b/utils/file_backend_local.go index a2d311f83..ec0c657a7 100644 --- a/utils/file_backend_local.go +++ b/utils/file_backend_local.go @@ -33,6 +33,14 @@ func (b *LocalFileBackend) TestConnection() *model.AppError { return nil } +func (b *LocalFileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { + if f, err := os.Open(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("Reader", "api.file.reader.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) { if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil { return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) diff --git a/utils/file_backend_s3.go b/utils/file_backend_s3.go index 6f1fa9ab0..a0c46e5d3 100644 --- a/utils/file_backend_s3.go +++ b/utils/file_backend_s3.go @@ -82,6 +82,18 @@ func (b *S3FileBackend) TestConnection() *model.AppError { return nil } +func (b *S3FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) + if err != nil { + return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return minioObject, nil +} + func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { s3Clnt, err := b.s3New() if err != nil { |