diff options
author | George Goldberg <george@gberg.me> | 2017-08-17 15:05:17 +0100 |
---|---|---|
committer | Harrison Healey <harrisonmhealey@gmail.com> | 2017-08-17 10:05:17 -0400 |
commit | 22459ee17a3ba0b4487f975b6ebe630cab2d9feb (patch) | |
tree | 275d8b68561d1c5046416d04264efc35806b8342 /model | |
parent | 4e92d1801733410c47bdde29c4bda4d52210d4e7 (diff) | |
download | chat-22459ee17a3ba0b4487f975b6ebe630cab2d9feb.tar.gz chat-22459ee17a3ba0b4487f975b6ebe630cab2d9feb.tar.bz2 chat-22459ee17a3ba0b4487f975b6ebe630cab2d9feb.zip |
PLT-7302: Aggregate Elasticsearch indexes over a certain age. (#7224)
* PLT-7302: Aggregate Elasticsearch indexes over a certain age.
This is done by a scheduled daily job, in order to keep the shard count
to a sensible level in Elasticsearch.
* Use map[string]string instead of StringMap
Diffstat (limited to 'model')
-rw-r--r-- | model/config.go | 49 | ||||
-rw-r--r-- | model/job.go | 24 |
2 files changed, 49 insertions, 24 deletions
diff --git a/model/config.go b/model/config.go index 933c643f2..9906723e7 100644 --- a/model/config.go +++ b/model/config.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "time" ) const ( @@ -123,11 +124,13 @@ const ( ANNOUNCEMENT_SETTINGS_DEFAULT_BANNER_COLOR = "#f2a93b" ANNOUNCEMENT_SETTINGS_DEFAULT_BANNER_TEXT_COLOR = "#333333" - ELASTICSEARCH_SETTINGS_DEFAULT_CONNECTION_URL = "" - ELASTICSEARCH_SETTINGS_DEFAULT_USERNAME = "" - ELASTICSEARCH_SETTINGS_DEFAULT_PASSWORD = "" - ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_REPLICAS = 1 - ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_SHARDS = 1 + ELASTICSEARCH_SETTINGS_DEFAULT_CONNECTION_URL = "" + ELASTICSEARCH_SETTINGS_DEFAULT_USERNAME = "" + ELASTICSEARCH_SETTINGS_DEFAULT_PASSWORD = "" + ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_REPLICAS = 1 + ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_SHARDS = 1 + ELASTICSEARCH_SETTINGS_DEFAULT_AGGREGATE_POSTS_AFTER_DAYS = 365 + ELASTICSEARCH_SETTINGS_DEFAULT_POSTS_AGGREGATOR_JOB_START_TIME = "03:00" ) type ServiceSettings struct { @@ -441,14 +444,16 @@ type WebrtcSettings struct { } type ElasticsearchSettings struct { - ConnectionUrl *string - Username *string - Password *string - EnableIndexing *bool - EnableSearching *bool - Sniff *bool - PostIndexReplicas *int - PostIndexShards *int + ConnectionUrl *string + Username *string + Password *string + EnableIndexing *bool + EnableSearching *bool + Sniff *bool + PostIndexReplicas *int + PostIndexShards *int + AggregatePostsAfterDays *int + PostsAggregatorJobStartTime *string } type DataRetentionSettings struct { @@ -1452,6 +1457,16 @@ func (o *Config) SetDefaults() { *o.ElasticsearchSettings.PostIndexShards = ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_SHARDS } + if o.ElasticsearchSettings.AggregatePostsAfterDays == nil { + o.ElasticsearchSettings.AggregatePostsAfterDays = new(int) + *o.ElasticsearchSettings.AggregatePostsAfterDays = ELASTICSEARCH_SETTINGS_DEFAULT_AGGREGATE_POSTS_AFTER_DAYS + } + + if o.ElasticsearchSettings.PostsAggregatorJobStartTime == nil { + o.ElasticsearchSettings.PostsAggregatorJobStartTime = new(string) + *o.ElasticsearchSettings.PostsAggregatorJobStartTime = ELASTICSEARCH_SETTINGS_DEFAULT_POSTS_AGGREGATOR_JOB_START_TIME + } + if o.DataRetentionSettings.Enable == nil { o.DataRetentionSettings.Enable = new(bool) *o.DataRetentionSettings.Enable = false @@ -1700,6 +1715,14 @@ func (o *Config) IsValid() *AppError { return NewLocAppError("Config.IsValid", "model.config.is_valid.elastic_search.enable_searching.app_error", nil, "") } + if *o.ElasticsearchSettings.AggregatePostsAfterDays < 1 { + return NewAppError("Config.IsValid", "model.config.is_valid.elastic_search.aggregate_posts_after_days.app_error", nil, "", http.StatusBadRequest) + } + + if _, err := time.Parse("03:04", *o.ElasticsearchSettings.PostsAggregatorJobStartTime); err != nil { + return NewAppError("Config.IsValid", "model.config.is_valid.elastic_search.posts_aggregator_job_start_time.app_error", nil, err.Error(), http.StatusBadRequest) + } + return nil } diff --git a/model/job.go b/model/job.go index 004331a1f..258fa2bd3 100644 --- a/model/job.go +++ b/model/job.go @@ -10,8 +10,9 @@ import ( ) const ( - JOB_TYPE_DATA_RETENTION = "data_retention" - JOB_TYPE_ELASTICSEARCH_POST_INDEXING = "elasticsearch_post_indexing" + JOB_TYPE_DATA_RETENTION = "data_retention" + JOB_TYPE_ELASTICSEARCH_POST_INDEXING = "elasticsearch_post_indexing" + JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION = "elasticsearch_post_aggregation" JOB_STATUS_PENDING = "pending" JOB_STATUS_IN_PROGRESS = "in_progress" @@ -22,15 +23,15 @@ const ( ) type Job struct { - Id string `json:"id"` - Type string `json:"type"` - Priority int64 `json:"priority"` - CreateAt int64 `json:"create_at"` - StartAt int64 `json:"start_at"` - LastActivityAt int64 `json:"last_activity_at"` - Status string `json:"status"` - Progress int64 `json:"progress"` - Data map[string]interface{} `json:"data"` + Id string `json:"id"` + Type string `json:"type"` + Priority int64 `json:"priority"` + CreateAt int64 `json:"create_at"` + StartAt int64 `json:"start_at"` + LastActivityAt int64 `json:"last_activity_at"` + Status string `json:"status"` + Progress int64 `json:"progress"` + Data map[string]string `json:"data"` } func (j *Job) IsValid() *AppError { @@ -45,6 +46,7 @@ func (j *Job) IsValid() *AppError { switch j.Type { case JOB_TYPE_DATA_RETENTION: case JOB_TYPE_ELASTICSEARCH_POST_INDEXING: + case JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION: default: return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest) } |