diff options
author | Jonathan <jonfritz@gmail.com> | 2017-11-30 09:07:04 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-11-30 09:07:04 -0500 |
commit | 375c0632fab03e3fb54865e320585888499c076d (patch) | |
tree | ce6cba679337a82370d5a730c428bef21a9964bf /model/config.go | |
parent | d0d9ba4a7e43301697d1c9f495930e0c0179fdc3 (diff) | |
download | chat-375c0632fab03e3fb54865e320585888499c076d.tar.gz chat-375c0632fab03e3fb54865e320585888499c076d.tar.bz2 chat-375c0632fab03e3fb54865e320585888499c076d.zip |
PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612)
* Created message export scheduled task
* Added CLI command to immediately kick off an export job
* Added email addresses for users joining and leaving the channel to the export
* Added support for both MySQL and PostgreSQL
* Fixing gofmt error
* Added a new ChannelMemberHistory store and associated tests
* Updating the ChannelMemberHistory channel as users create/join/leave channels
* Added user email to the message export object so it can be included in the actiance export xml
* Don't fail to log a leave event if a corresponding join event wasn't logged
* Adding copyright notices
* Adding message export settings to daily diagnostics report
* Added System Console integration for message export
* Cleaned up TODOs
* Made batch size configurable
* Added export from timestamp to CLI command
* Made ChannelMemberHistory table updates best effort
* Added a context-based timeout option to the message export CLI
* Minor PR updates/improvements
* Removed unnecessary fields from MessageExport object to reduce query overhead
* Removed JSON functions from the message export query in an effort to optimize performance
* Changed the way that channel member history queries and purges work to better account for edge cases
* Fixing a test I missed with the last refactor
* Added file copy functionality to file backend, improved config validation, added default config values
* Fixed file copy tests
* More concise use of the testing libraries
* Fixed context leak error
* Changed default export path to correctly place an 'export' directory under the 'data' directory
* Can't delete records from a read replica
* Fixed copy file tests
* Start job workers when license is applied, if configured to do so
* Suggestions from the PR
* Moar unit tests
* Fixed test imports
Diffstat (limited to 'model/config.go')
-rw-r--r-- | model/config.go | 66 |
1 files changed, 66 insertions, 0 deletions
diff --git a/model/config.go b/model/config.go index e2f05d72e..1f56eb4f5 100644 --- a/model/config.go +++ b/model/config.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "path/filepath" "strings" "time" ) @@ -1508,6 +1509,36 @@ func (s *PluginSettings) SetDefaults() { } } +type MessageExportSettings struct { + EnableExport *bool + DailyRunTime *string + ExportFromTimestamp *int64 + FileLocation *string + BatchSize *int +} + +func (s *MessageExportSettings) SetDefaults() { + if s.EnableExport == nil { + s.EnableExport = NewBool(false) + } + + if s.FileLocation == nil { + s.FileLocation = NewString("export") + } + + if s.DailyRunTime == nil { + s.DailyRunTime = NewString("01:00") + } + + if s.ExportFromTimestamp == nil { + s.ExportFromTimestamp = NewInt64(0) + } + + if s.BatchSize == nil { + s.BatchSize = NewInt(10000) + } +} + type ConfigFunc func() *Config type Config struct { @@ -1538,6 +1569,7 @@ type Config struct { WebrtcSettings WebrtcSettings ElasticsearchSettings ElasticsearchSettings DataRetentionSettings DataRetentionSettings + MessageExportSettings MessageExportSettings JobSettings JobSettings PluginSettings PluginSettings } @@ -1617,6 +1649,7 @@ func (o *Config) SetDefaults() { o.LogSettings.SetDefaults() o.JobSettings.SetDefaults() o.WebrtcSettings.SetDefaults() + o.MessageExportSettings.SetDefaults() } func (o *Config) IsValid() *AppError { @@ -1680,6 +1713,10 @@ func (o *Config) IsValid() *AppError { return err } + if err := o.MessageExportSettings.isValid(o.FileSettings); err != nil { + return err + } + return nil } @@ -1998,6 +2035,35 @@ func (ls *LocalizationSettings) isValid() *AppError { return nil } +func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError { + if mes.EnableExport == nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.enable.app_error", nil, "", http.StatusBadRequest) + } + if *mes.EnableExport { + if mes.ExportFromTimestamp == nil || *mes.ExportFromTimestamp < 0 || *mes.ExportFromTimestamp > time.Now().Unix() { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.export_from.app_error", nil, "", http.StatusBadRequest) + } else if mes.DailyRunTime == nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, "", http.StatusBadRequest) + } else if _, err := time.Parse("15:04", *mes.DailyRunTime); err != nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest) + } else if mes.FileLocation == nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.app_error", nil, "", http.StatusBadRequest) + } else if mes.BatchSize == nil || *mes.BatchSize < 0 { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest) + } else if *fs.DriverName != IMAGE_DRIVER_LOCAL { + if absFileDir, err := filepath.Abs(fs.Directory); err != nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest) + } else if absMessageExportDir, err := filepath.Abs(*mes.FileLocation); err != nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest) + } else if !strings.HasPrefix(absMessageExportDir, absFileDir) { + // configured export directory must be relative to data directory + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, "", http.StatusBadRequest) + } + } + } + return nil +} + func (o *Config) GetSanitizeOptions() map[string]bool { options := map[string]bool{} options["fullname"] = o.PrivacySettings.ShowFullName |