diff --git a/db-connector.go b/db-connector.go index 75bbeb1b..865ee55d 100755 --- a/db-connector.go +++ b/db-connector.go @@ -7,7 +7,7 @@ import ( "crypto/sha1" "crypto/tls" "encoding/hex" - + "encoding/json" "errors" @@ -22,11 +22,11 @@ import ( "crypto/sha256" "math" "math/rand" + "regexp" "sort" "strings" "sync" "time" - "regexp" runtimeDebug "runtime/debug" @@ -77,36 +77,6 @@ type ShuffleStorage struct { var maxCacheKeyLength = 250 -// Create ElasticSearch/OpenSearch index prefix -// It is used where a single cluster of ElasticSearch/OpenSearch utilized by several -// Shuffle instance -// E.g. Instance1_Workflowapp -func GetESIndexPrefix(index string) string { - prefix := os.Getenv("SHUFFLE_OPENSEARCH_INDEX_PREFIX") - if len(prefix) > 0 { - return fmt.Sprintf("%s_%s", prefix, index) - } - - return index -} - -func GetOpensearchBaseIndexes() []string { - return []string{ - "workflowexecution", - "datastore_ngram", - "org_cache", - "org_cache_revisions", - "notifications", - "shuffle_logs", - "environments", - "org_statistics", - "workflowapp", - "workflow", - "workflow_revisions", - "datastore_category", - } -} - func SetOrgStatistics(ctx context.Context, stats ExecutionInfo, id string) error { nameKey := "org_statistics" @@ -381,7 +351,6 @@ func SetCache(ctx context.Context, name string, data []byte, expiration int32, u } } - // Splitting into multiple cache items //if project.Environment == "cloud" || len(memcached) > 0 { if len(memcached) > 0 { @@ -583,8 +552,6 @@ func SetWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id st return nil } - - func GetEsConfig(defaultCreds bool) *opensearchapi.Client { esUrl := os.Getenv("SHUFFLE_OPENSEARCH_URL") if len(esUrl) == 0 { @@ -920,54 +887,39 @@ func IncrementCacheDump(ctx context.Context, orgId, dataType string, amount ...i // Get it from opensearch (may be prone to more issues at scale (thousands/second) due to no transactional locking) id := strings.ToLower(orgId) - resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{ - Index: strings.ToLower(GetESIndexPrefix(nameKey)), - DocumentID: id, - }) - - if err != nil { - if debug { - log.Printf("[WARNING] Error in org STATS get: %s", err) - } - //return err + data, found, getErr := getDocumentByID(ctx, strings.ToLower(GetESIndexPrefix(nameKey)), id, "last_cleared") + if getErr != nil { + log.Printf("[WARNING] Failed getting org STATS body: %s", getErr) + return getErr } - res := resp.Inspect().Response - defer res.Body.Close() - respBody, bodyErr := ioutil.ReadAll(res.Body) - if err != nil || bodyErr != nil || res.StatusCode >= 300 { - log.Printf("[WARNING] Failed getting org STATS body: %s. Resp: %d. Body err: %s", err, res.StatusCode, bodyErr) - + if !found { // Init the org stats if it doesn't exist - if res.StatusCode == 404 { - orgStatistics.OrgId = orgId - orgStatistics = HandleIncrement(dataType, orgStatistics, dbDumpInterval) - orgStatistics = handleDailyCacheUpdate(orgStatistics) + orgStatistics.OrgId = orgId + orgStatistics = HandleIncrement(dataType, orgStatistics, dbDumpInterval) + orgStatistics = handleDailyCacheUpdate(orgStatistics) - marshalledData, err := json.Marshal(orgStatistics) - if err != nil { - log.Printf("[ERROR] Failed marshalling org STATS body: %s", err) + marshalledData, marshalErr := json.Marshal(orgStatistics) + if marshalErr != nil { + log.Printf("[ERROR] Failed marshalling org STATS body: %s", marshalErr) + } else { + if indexErr := indexEs(ctx, nameKey, id, marshalledData); indexErr != nil { + log.Printf("[ERROR] Failed indexing org STATS body: %s", indexErr) } else { - err := indexEs(ctx, nameKey, id, marshalledData) - if err != nil { - log.Printf("[ERROR] Failed indexing org STATS body: %s", err) - } else { - log.Printf("[DEBUG] Indexed org STATS body for %s", orgId) - } + log.Printf("[DEBUG] Indexed org STATS body for %s", orgId) } } - return err + return nil } - orgStatsWrapper := &ExecutionInfoWrapper{} - err = json.Unmarshal(respBody, &orgStatsWrapper) - if err != nil { - log.Printf("[ERROR] Failed unmarshalling org STATS body: %s", err) - return err + source := ExecutionInfo{} + if unmarshalErr := json.Unmarshal(data, &source); unmarshalErr != nil { + log.Printf("[ERROR] Failed unmarshalling org STATS body: %s", unmarshalErr) + return unmarshalErr } - orgStatistics = &orgStatsWrapper.Source + orgStatistics = &source if orgStatistics.OrgName == "" || orgStatistics.OrgName == orgStatistics.OrgId { org, err := GetOrg(ctx, orgId) if err == nil { @@ -987,9 +939,8 @@ func IncrementCacheDump(ctx context.Context, orgId, dataType string, amount ...i return err } - err = indexEs(ctx, nameKey, id, marshalledData) - if err != nil { - log.Printf("[ERROR] Failed indexing org STATS body (2): %s", err) + if indexErr := indexEs(ctx, nameKey, id, marshalledData); indexErr != nil { + log.Printf("[ERROR] Failed indexing org STATS body (2): %s", indexErr) } //log.Printf("[DEBUG] Incremented org stats for %s", orgId) @@ -1531,11 +1482,11 @@ func getExecutionFileValue(ctx context.Context, workflowExecution WorkflowExecut obj := bucket.Object(fullParsedPath) fileReader, err := obj.NewReader(ctx) if err != nil { - if debug { + if debug { log.Printf("[DEBUG] Failed reading file '%s' from bucket %s: %s. Will try with alternative solution.", fullParsedPath, bucketName, err) } - // Cache sip for the minute + // Cache sip for the minute SetCache(ctx, cacheKey, []byte{}, 1) if projectName != "shuffler" { @@ -1545,7 +1496,7 @@ func getExecutionFileValue(ctx context.Context, workflowExecution WorkflowExecut fileReader, err = obj.NewReader(ctx) if err != nil { //log.Printf("[ERROR] Failed reading file '%s' again from bucket %s: %s", fullParsedPath, bucketName, err) - + return "", err } } else { @@ -2449,11 +2400,11 @@ func GetEnvironment(ctx context.Context, id, orgId string) (*Environment, error) if err == nil { timenow := time.Now().Unix() - if env.SensorGroup { + if env.SensorGroup { for sensorIndex, _ := range env.SensorHosts { sensor := env.SensorHosts[sensorIndex] - - env.SensorHosts[sensorIndex].Active = false + + env.SensorHosts[sensorIndex].Active = false if sensor.Checkin > 0 && timenow-sensor.Checkin < 300 { env.SensorHosts[sensorIndex].Active = true } @@ -2590,11 +2541,11 @@ func GetEnvironment(ctx context.Context, id, orgId string) (*Environment, error) } timenow := time.Now().Unix() - if env.SensorGroup { + if env.SensorGroup { for sensorIndex, _ := range env.SensorHosts { sensor := env.SensorHosts[sensorIndex] - - env.SensorHosts[sensorIndex].Active = false + + env.SensorHosts[sensorIndex].Active = false if sensor.Checkin > 0 && timenow-sensor.Checkin < 300 { env.SensorHosts[sensorIndex].Active = true } @@ -2640,6 +2591,10 @@ func GetWorkflowRunCount(ctx context.Context, id string, start int64, end int64) if project.DbType == "opensearch" { // count WorkflowExecution where workflowId = id + // Uses a cardinality aggregation on execution_id instead of + // track_total_hits so a duplicate _id across live+archive (the narrow + // crash/sweep-race window described in execution_lifecycle.go) is + // counted once, not twice. query := map[string]interface{}{ "size": 0, "query": map[string]interface{}{ @@ -2661,6 +2616,13 @@ func GetWorkflowRunCount(ctx context.Context, id string, start int64, end int64) }, }, }, + "aggs": map[string]interface{}{ + "unique_executions": map[string]interface{}{ + "cardinality": map[string]interface{}{ + "field": "execution_id", + }, + }, + }, } var buf bytes.Buffer @@ -2670,10 +2632,11 @@ func GetWorkflowRunCount(ctx context.Context, id string, start int64, end int64) } resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{ - Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))}, + Indices: executionSearchIndices(), Body: &buf, Params: opensearchapi.SearchParams{ - TrackTotalHits: true, + AllowNoIndices: opensearchapi.ToPointer(true), + IgnoreUnavailable: opensearchapi.ToPointer(true), }, }) @@ -2722,7 +2685,7 @@ func GetWorkflowRunCount(ctx context.Context, id string, start int64, end int64) return 0, err } - count = wrapped.Hits.Total.Value + count = wrapped.Aggregations.UniqueExecutions.Value } else { // count WorkflowExecution where workflowId = id //query := datastore.NewQuery(nameKey).Filter("workflow_id =", strings.ToLower(id)) @@ -3237,43 +3200,20 @@ func GetOrgStatistics(ctx context.Context, orgId string) (*ExecutionInfo, error) if project.DbType == "opensearch" { shouldInitializeStats := false - resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{ - Index: strings.ToLower(GetESIndexPrefix(nameKey)), - DocumentID: orgId, - }) - - if err != nil && !strings.Contains(err.Error(), "status: 404") { + data, found, err := getDocumentByID(ctx, strings.ToLower(GetESIndexPrefix(nameKey)), orgId, "last_cleared") + if err != nil { log.Printf("[WARNING] Error for %s: %s", cacheKey, err) return stats, err } - if err != nil && strings.Contains(err.Error(), "status: 404") { + if !found { shouldInitializeStats = true - } - - if !shouldInitializeStats { - res := resp.Inspect().Response - defer res.Body.Close() - if res.StatusCode == 404 { - shouldInitializeStats = true - } else { - respBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return stats, err - } - - wrapped := ExecutionInfoWrapper{} - err = json.Unmarshal(respBody, &wrapped) - if err != nil { - return stats, err - } - - if !wrapped.Found { - shouldInitializeStats = true - } else { - stats = &wrapped.Source - } + } else { + source := ExecutionInfo{} + if unmarshalErr := json.Unmarshal(data, &source); unmarshalErr != nil { + return stats, unmarshalErr } + stats = &source } if shouldInitializeStats { @@ -3605,7 +3545,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso _, err = it.Next(&innerWorkflow) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") { - if debug { + if debug { //log.Printf("[DEBUG] Workflow load iterator issue: %s", err) } @@ -3619,7 +3559,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso } if innerWorkflow.Public { - //if debug { + //if debug { // log.Printf("[DEBUG] Skipping public workflow %s (%s) for org %s", innerWorkflow.Name, innerWorkflow.ID, user.ActiveOrg.Id) //} @@ -3627,7 +3567,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso } if innerWorkflow.Hidden { - //if debug { + //if debug { // log.Printf("[DEBUG] Skipping HIDDEN workflow %s (%s) for org %s", innerWorkflow.Name, innerWorkflow.ID, user.ActiveOrg.Id) //} @@ -3651,12 +3591,12 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso } } - // Fallback for when the iterator fails due to a datastore issue + // Fallback for when the iterator fails due to a datastore issue // (e.g. "cannot load field" error) and similar if err != iterator.Done { log.Printf("[WARNING] Failed fetching workflow results for org %s: %v", user.ActiveOrg.Id, err) - // Check if query contains edited or not + // Check if query contains edited or not if strings.Contains(fmt.Sprintf("%s", err), "FailedPrecondition desc") && strings.Contains(fmt.Sprintf("%s", query), "edited") { log.Printf("[ERROR] Retrying workflow query without Edited sort due to error: %s", err) @@ -3692,7 +3632,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso }) if len(workflows) > maxAmount { - if debug { + if debug { log.Printf("[WARNING] Found %d workflows for user %s (%s) in org %s, but limiting to %d", len(workflows), user.Username, user.Id, user.ActiveOrg.Id, maxAmount) } @@ -6454,11 +6394,11 @@ func GetEnvironments(ctx context.Context, orgId string) ([]Environment, error) { timenow := time.Now().Unix() for envIndex, env := range environments { - if env.SensorGroup { + if env.SensorGroup { for sensorIndex, _ := range env.SensorHosts { sensor := env.SensorHosts[sensorIndex] - - environments[envIndex].SensorHosts[sensorIndex].Active = false + + environments[envIndex].SensorHosts[sensorIndex].Active = false if sensor.Checkin > 0 && timenow-sensor.Checkin < 300 { environments[envIndex].SensorHosts[sensorIndex].Active = true } @@ -6637,11 +6577,11 @@ func GetEnvironments(ctx context.Context, orgId string) ([]Environment, error) { timenow := time.Now().Unix() for envIndex, env := range environments { - if env.SensorGroup { + if env.SensorGroup { for sensorIndex, _ := range env.SensorHosts { sensor := env.SensorHosts[sensorIndex] - - environments[envIndex].SensorHosts[sensorIndex].Active = false + + environments[envIndex].SensorHosts[sensorIndex].Active = false if sensor.Checkin > 0 && timenow-sensor.Checkin < 300 { environments[envIndex].SensorHosts[sensorIndex].Active = true } @@ -10119,7 +10059,7 @@ func GetApikey(ctx context.Context, apikey string) (User, error) { } if debug { - log.Printf("[DEBUG] API key cache miss; looking up user") + log.Printf("[DEBUG] API key cache miss; looking up user") } if project.DbType == "opensearch" { @@ -10438,34 +10378,20 @@ func GetNotification(ctx context.Context, id string) (*Notification, error) { cacheKey := fmt.Sprintf("%s_%s", nameKey, id) curFile := &Notification{} if project.DbType == "opensearch" { - //log.Printf("GETTING ES USER %s", - resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{ - Index: strings.ToLower(GetESIndexPrefix(nameKey)), - DocumentID: id, - }) + data, found, err := getDocumentByID(ctx, strings.ToLower(GetESIndexPrefix(nameKey)), id, "updated_at") if err != nil { log.Printf("[WARNING] Error for %s: %s", cacheKey, err) return &Notification{}, err } - - res := resp.Inspect().Response - defer res.Body.Close() - if res.StatusCode == 404 { + if !found { return &Notification{}, errors.New("Notification with that ID doesn't exist") } - respBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return &Notification{}, err + source := Notification{} + if unmarshalErr := json.Unmarshal(data, &source); unmarshalErr != nil { + return &Notification{}, unmarshalErr } - - wrapped := NotificationWrapper{} - err = json.Unmarshal(respBody, &wrapped) - if err != nil { - return &Notification{}, err - } - - curFile = &wrapped.Source + curFile = &source } else { key := datastore.NameKey(nameKey, id, nil) if err := project.Dbclient.Get(ctx, key, curFile); err != nil { @@ -10855,7 +10781,7 @@ func GetOrgNotifications(ctx context.Context, orgId string) ([]Notification, err "size": 1000, "sort": map[string]interface{}{ "updated_at": map[string]interface{}{ - "order": "desc", + "order": "desc", "unmapped_type": "long", }, }, @@ -12037,7 +11963,8 @@ func GetUnfinishedExecutions(ctx context.Context, workflowId string) ([]Workflow "size": 1000, "sort": map[string]interface{}{ "started_at": map[string]interface{}{ - "order": "desc", + "order": "desc", + "unmapped_type": "long", }, }, "query": map[string]interface{}{ @@ -12065,18 +11992,15 @@ func GetUnfinishedExecutions(ctx context.Context, workflowId string) ([]Workflow // Perform the search request. resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{ - Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))}, + Indices: executionSearchIndices(), Body: &buf, Params: opensearchapi.SearchParams{ - TrackTotalHits: true, + TrackTotalHits: true, + AllowNoIndices: opensearchapi.ToPointer(true), + IgnoreUnavailable: opensearchapi.ToPointer(true), }, }) if err != nil { - if strings.Contains(err.Error(), "index_not_found_exception") { - return executions, nil - } - - log.Printf("[ERROR] Error getting response from Opensearch (get workflow executions): %s", err) return executions, err } @@ -12120,6 +12044,7 @@ func GetUnfinishedExecutions(ctx context.Context, workflowId string) ([]Workflow for _, hit := range wrapped.Hits.Hits { executions = append(executions, hit.Source) } + executions = dedupExecutionsByID(executions) return executions, nil } else { @@ -12252,7 +12177,8 @@ func GetAllWorkflowExecutionsV2(ctx context.Context, workflowId string, amount i }, "sort": map[string]interface{}{ "started_at": map[string]interface{}{ - "order": "desc", + "order": "desc", + "unmapped_type": "long", }, }, } @@ -12263,10 +12189,12 @@ func GetAllWorkflowExecutionsV2(ctx context.Context, workflowId string, amount i // Perform the search request. resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{ - Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))}, + Indices: executionSearchIndices(), Body: &buf, Params: opensearchapi.SearchParams{ - TrackTotalHits: true, + TrackTotalHits: true, + AllowNoIndices: opensearchapi.ToPointer(true), + IgnoreUnavailable: opensearchapi.ToPointer(true), }, }) if err != nil { @@ -12320,6 +12248,7 @@ func GetAllWorkflowExecutionsV2(ctx context.Context, workflowId string, amount i executions = append(executions, hit.Source) } } + executions = dedupExecutionsByID(executions) } else { query := datastore.NewQuery(nameKey).Filter("workflow_id =", workflowId).Order("-started_at").Limit(5) @@ -12641,7 +12570,8 @@ func GetAllWorkflowExecutions(ctx context.Context, workflowId string, amount int }, "sort": map[string]interface{}{ "started_at": map[string]interface{}{ - "order": "desc", + "order": "desc", + "unmapped_type": "long", }, }, } @@ -12652,10 +12582,12 @@ func GetAllWorkflowExecutions(ctx context.Context, workflowId string, amount int // Perform the search request. resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{ - Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))}, + Indices: executionSearchIndices(), Body: &buf, Params: opensearchapi.SearchParams{ - TrackTotalHits: true, + TrackTotalHits: true, + AllowNoIndices: opensearchapi.ToPointer(true), + IgnoreUnavailable: opensearchapi.ToPointer(true), }, }) if err != nil { @@ -12709,6 +12641,7 @@ func GetAllWorkflowExecutions(ctx context.Context, workflowId string, amount int executions = append(executions, hit.Source) } } + executions = dedupExecutionsByID(executions) //return executions, nil } else { @@ -13808,7 +13741,7 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto oldDoc := config.Value newDoc := cacheData.Value - if debug { + if debug { log.Printf("\n\nOLD: %s\n\nNEW: %s\n\n", oldDoc, newDoc) } @@ -13833,9 +13766,9 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto break } - // This NEVER triggers. RLS just returns the merged JSON - // and we trust it. If we don't trust it, we can set - // ruleValid to false above. + // This NEVER triggers. RLS just returns the merged JSON + // and we trust it. If we don't trust it, we can set + // ruleValid to false above. if !ruleValid { // Break out if debug { @@ -13844,8 +13777,8 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto keyUpdated = false - cacheData.Existed = true - cacheData.Changed = keyUpdated + cacheData.Existed = true + cacheData.Changed = keyUpdated datastoreKeys <- *datastore.NameKey(nameKey, datastoreId, nil) cacheKeys <- cacheData return @@ -14190,7 +14123,7 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto } if len(newArray) > 0 { - if debug { + if debug { log.Printf("[INFO] SetDatastoreKeyBulk: Successfully set %d key(s) in category %s for org %s", len(newArray), mainCategory, orgId) } } @@ -15114,6 +15047,130 @@ func getCacheKeyByAliasSearch(ctx context.Context, aliasName, id string) (*Cache return &item, nil } +type esDocGetWrapper struct { + Found bool `json:"found"` + Source json.RawMessage `json:"_source"` +} + +type esSearchDocWrapper struct { + Hits struct { + Hits []struct { + Source json.RawMessage `json:"_source"` + } `json:"hits"` + } `json:"hits"` +} + +// buildAliasSearchBody builds an ids _search query for a single document. When +// sortField is non-empty the result is ordered desc on that field so the most +// recent generation wins when an alias spans multiple backing indices. +func buildAliasSearchBody(sortField, id string) ([]byte, error) { + query := map[string]interface{}{ + "size": 1, + "query": map[string]interface{}{ + "ids": map[string]interface{}{ + "values": []string{id}, + }, + }, + } + + if sortField != "" { + query["sort"] = []map[string]interface{}{ + { + sortField: map[string]interface{}{ + "order": "desc", + "unmapped_type": "long", + }, + }, + } + } + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(query); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// getDocumentByID fetches a document by id from an OpenSearch index/alias and +// returns its raw _source. If the target is a rollover alias spanning more than +// one backing index (which fails for a single-document get), it falls back to an +// ids _search across the alias (optionally sorted so the newest generation wins). +// found=false with err=nil means the document does not exist. +func getDocumentByID(ctx context.Context, indexName, id, sortField string) (doc []byte, found bool, err error) { + resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{ + Index: indexName, + DocumentID: id, + }) + + if err != nil { + if strings.Contains(err.Error(), "status: 404") { + return nil, false, nil + } + + if !strings.Contains(err.Error(), "has more than one index associated with it") { + return nil, false, err + } + + // Alias spans multiple backing indices: fall back to an ids search. + body, buildErr := buildAliasSearchBody(sortField, id) + if buildErr != nil { + return nil, false, buildErr + } + + searchResp, searchErr := project.Es.Search(ctx, &opensearchapi.SearchReq{ + Indices: []string{indexName}, + Body: bytes.NewReader(body), + Params: opensearchapi.SearchParams{TrackTotalHits: true}, + }) + if searchErr != nil { + return nil, false, searchErr + } + + searchRes := searchResp.Inspect().Response + defer searchRes.Body.Close() + + respBody, readErr := ioutil.ReadAll(searchRes.Body) + if readErr != nil { + return nil, false, readErr + } + if searchRes.StatusCode != 200 && searchRes.StatusCode != 201 { + return nil, false, fmt.Errorf("failed alias fallback lookup. status=%d body=%s", searchRes.StatusCode, string(respBody)) + } + + wrapped := esSearchDocWrapper{} + if unmarshalErr := json.Unmarshal(respBody, &wrapped); unmarshalErr != nil { + return nil, false, unmarshalErr + } + if len(wrapped.Hits.Hits) == 0 { + return nil, false, nil + } + + return wrapped.Hits.Hits[0].Source, true, nil + } + + res := resp.Inspect().Response + defer res.Body.Close() + if res.StatusCode >= 300 { + return nil, false, fmt.Errorf("failed document get. status=%d", res.StatusCode) + } + + respBody, readErr := ioutil.ReadAll(res.Body) + if readErr != nil { + return nil, false, readErr + } + + wrapped := esDocGetWrapper{} + if unmarshalErr := json.Unmarshal(respBody, &wrapped); unmarshalErr != nil { + return nil, false, unmarshalErr + } + if !wrapped.Found { + return nil, false, nil + } + + return wrapped.Source, true, nil +} + var retryCount int func RunInit(dbclient datastore.Client, storageClient storage.Client, gceProject, environment string, cacheDb bool, dbType string, defaultCreds bool, count int) (ShuffleStorage, error) { @@ -16299,37 +16356,37 @@ func GetAllCacheKeys(ctx context.Context, orgId string, category string, max int if parentOrgDepth >= 3 { log.Printf("[ERROR] Reached maximum parent org lookup depth (%d) for org %s. Skipping parent org cache lookup to prevent infinite recursion.", parentOrgDepth, orgId) } else { - parentOrg, err := GetOrg(ctx, foundOrg.CreatorOrg) - if err != nil { + parentOrg, err := GetOrg(ctx, foundOrg.CreatorOrg) + if err != nil { if debug { log.Printf("[DEBUG] Could not find parent org %s for org %s (possibly in different region): %s", foundOrg.CreatorOrg, orgId, err) } - } else { + } else { parentOrgCache, _, err := GetAllCacheKeys(ctx, parentOrg.Id, "", max, inputcursor, cleanupDepth, parentOrgDepth+1) - if err != nil { + if err != nil { if debug { log.Printf("[DEBUG] Failed getting parent org cache keys for org %s: %s", parentOrg.Id, err) } - } else { - if debug { - //log.Printf("[DEBUG] Loaded %d parent org cache keys for org %s. Validating if child org %s should get the keys", len(parentOrgCache), parentOrg.Id, orgId) - } + } else { + if debug { + //log.Printf("[DEBUG] Loaded %d parent org cache keys for org %s. Validating if child org %s should get the keys", len(parentOrgCache), parentOrg.Id, orgId) + } - for _, parentCache := range parentOrgCache { - /* - if debug && len(parentCache.SuborgDistribution) > 0 { - log.Printf("[DEBUG] Parent org %s keys: %#v", parentOrg.Id, parentCache.SuborgDistribution) - } - */ + for _, parentCache := range parentOrgCache { + /* + if debug && len(parentCache.SuborgDistribution) > 0 { + log.Printf("[DEBUG] Parent org %s keys: %#v", parentOrg.Id, parentCache.SuborgDistribution) + } + */ - if !ArrayContains(parentCache.SuborgDistribution, orgId) { - continue - } + if !ArrayContains(parentCache.SuborgDistribution, orgId) { + continue + } - // Clean up just in case - parentCache.PublicAuthorization = "" - parentCache.SuborgDistribution = []string{orgId} - cacheKeys = append(cacheKeys, parentCache) + // Clean up just in case + parentCache.PublicAuthorization = "" + parentCache.SuborgDistribution = []string{orgId} + cacheKeys = append(cacheKeys, parentCache) } } } @@ -17653,7 +17710,8 @@ func GetWorkflowRunsBySearch(ctx context.Context, orgId string, search WorkflowS }, "sort": map[string]interface{}{ "started_at": map[string]interface{}{ - "order": "desc", + "order": "desc", + "unmapped_type": "long", }, }, } @@ -17754,10 +17812,12 @@ func GetWorkflowRunsBySearch(ctx context.Context, orgId string, search WorkflowS // Perform the search request. resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{ - Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))}, + Indices: executionSearchIndices(), Body: &buf, Params: opensearchapi.SearchParams{ - TrackTotalHits: true, + TrackTotalHits: true, + AllowNoIndices: opensearchapi.ToPointer(true), + IgnoreUnavailable: opensearchapi.ToPointer(true), }, }) @@ -17792,6 +17852,10 @@ func GetWorkflowRunsBySearch(ctx context.Context, orgId string, search WorkflowS for _, hit := range wrapped.Hits.Hits { executions = append(executions, hit.Source) } + // Searching both workflowexecution_live and workflowexecution (archive) + // can surface the same execution_id twice in the narrow window between + // an unarchive write and its archive-side delete; dedupe defensively. + executions = dedupExecutionsByID(executions) //return executions, "", errors.New("Not implemented yet") } else { @@ -18683,33 +18747,20 @@ func GetDatastoreNGramItem(ctx context.Context, key string) (*NGramItem, error) } if project.DbType == "opensearch" { - resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{ - Index: strings.ToLower(GetESIndexPrefix(nameKey)), - DocumentID: key, - }) + data, found, err := getDocumentByID(ctx, strings.ToLower(GetESIndexPrefix(nameKey)), key, "") if err != nil { log.Printf("[WARNING] Error for %s: %s", cacheKey, err) return ngramItem, err } - - res := resp.Inspect().Response - defer res.Body.Close() - if res.StatusCode == 404 { + if !found { return ngramItem, errors.New("Item doesn't exist") } - respBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return ngramItem, err - } - - wrapped := NgramItemWrapper{} - err = json.Unmarshal(respBody, &wrapped) - if err != nil { - return ngramItem, err + source := NGramItem{} + if unmarshalErr := json.Unmarshal(data, &source); unmarshalErr != nil { + return ngramItem, unmarshalErr } - - ngramItem = &wrapped.Source + ngramItem = &source } else { // Get the ngram item from the datastore getNgramKey := datastore.NameKey(nameKey, key, nil) @@ -18764,448 +18815,26 @@ func HealthCheckHandler(resp http.ResponseWriter, request *http.Request) { //fmt.Fprint(res, "OK") } -func InitOpensearchIndexes() { +// StartExecutionLifecycleJobs starts the background jobs that keep +// workflowexecution_live bounded. +func StartExecutionLifecycleJobs(ctx context.Context) { if project.DbType != "opensearch" { return } - if os.Getenv("SHUFFLE_SKIP_OPENSEARCH_INDEX_INIT") == "true" { - return - } - - // Check if the "workflowexecution" index exists and configuring rollovers if possible - log.Printf("[INFO] Configuring Opensearch indexes for scaling") - - ctx := context.Background() - opensearchUrl := strings.TrimRight(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "/") - if len(opensearchUrl) == 0 { - opensearchUrl = "https://shuffle-opensearch:9200" - } - - relevantScaleIndexes := []string{} - for _, baseIndex := range GetOpensearchBaseIndexes() { - relevantScaleIndexes = append(relevantScaleIndexes, GetESIndexPrefix(baseIndex)) - } - - customConfig := os.Getenv("OPENSEARCH_INDEX_CONFIG") - if len(customConfig) > 0 { - checkValidJson := map[string]interface{}{} - if err := json.Unmarshal([]byte(customConfig), &checkValidJson); err != nil { - log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_CONFIG: %s", err) - customConfig = "" - } - - log.Printf("[DEBUG] Using custom index config for relevant scale indexes: %s", customConfig) - } - - customRollover := os.Getenv("OPENSEARCH_INDEX_ROLLOVER") - if len(customRollover) > 0 { - checkValidJson := map[string]interface{}{} - if err := json.Unmarshal([]byte(customRollover), &checkValidJson); err != nil { - log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_ROLLOVER: %s", err) - customRollover = "" - } - - log.Printf("[DEBUG] Using custom rollover config for relevant scale indexes: %s", customRollover) - } - - rolloverConfig := []byte(fmt.Sprintf(`{ - "conditions": { - "max_age": "90d", - "max_size": "40gb", - "max_docs": 1000000 - } - }`)) - - if len(customRollover) > 0 { - rolloverConfig = []byte(customRollover) - } - - ismEnabled := strings.ToLower(strings.TrimSpace(os.Getenv("OPENSEARCH_USE_ISM_ROLLOVER"))) != "false" - ismPolicyName := strings.TrimSpace(os.Getenv("OPENSEARCH_ISM_POLICY_NAME")) - if ismPolicyName == "" { - ismPolicyName = "shuffle-rollover" - } - - ismReady := false - if ismEnabled { - var err error - ismReady, err = ensureOpensearchISMRolloverPolicy(ctx, opensearchUrl, relevantScaleIndexes, rolloverConfig, ismPolicyName) - if err != nil { - log.Printf("[WARNING] Failed ensuring ISM rollover policy '%s': %s", ismPolicyName, err) - } - } - - if fixResult, fixErr := FixOpensearchIndexPrefix(ctx); fixErr != nil { - log.Printf("[WARNING] Prefix repair before init failed: %s", fixErr) - } else if !fixResult.Success { - log.Printf("[WARNING] Prefix repair before init completed with verification warnings: %s", fixResult.Reason) - } else { - log.Printf("[INFO] Prefix repair before init: expected aliases=%d found=%d", fixResult.ExpectedAliases, fixResult.FoundAliases) - } - - for _, index := range relevantScaleIndexes { - indexConfig := []byte(fmt.Sprintf(`{ - "aliases": { - "%s": { - "is_write_index": true - } - }, - "settings": { - "number_of_shards": 3, - "number_of_replicas": 1, - "refresh_interval": "30s" - }, - "mappings": { - "dynamic_templates": [ - { - "strings_as_keywords": { - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } - } - } - ] - } - }`, index)) - - if len(customConfig) > 0 { - indexConfig = []byte(customConfig) - - // Check if alias is in the index or not, otherwise inject it - unmarshalled := map[string]interface{}{} - if err := json.Unmarshal(indexConfig, &unmarshalled); err != nil { - log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_CONFIG (2): %s", err) - } else { - if _, ok := unmarshalled["aliases"]; !ok { - // Inject it - aliasPart := map[string]interface{}{ - index: map[string]bool{ - "is_write_index": true, - }, - } - unmarshalled["aliases"] = aliasPart - newConfig, err := json.Marshal(unmarshalled) - if err != nil { - log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_CONFIG (3): %s", err) - } else { - indexConfig = newConfig - log.Printf("[INFO] Injected alias into OPENSEARCH_INDEX_CONFIG for index %s", index) - } - } - } - - } - - index = strings.ToLower(index) - initialIndexName := fmt.Sprintf("%s-000001", index) - indexConfig = ensureOpensearchIndexRolloverAlias(indexConfig, index) - // Directly try to force create it. Opensearch throws a 400 if it fails. - - resp, err := project.Es.Indices.Create(ctx, opensearchapi.IndicesCreateReq{ - Index: initialIndexName, - Body: bytes.NewReader(indexConfig), - }) - - res := resp.Inspect().Response - defer res.Body.Close() - if err != nil { - if !strings.Contains(fmt.Sprintf("%s", err), "serverless mode") && !strings.Contains(fmt.Sprintf("%s", err), "resource_already_exists_exception") { - log.Printf("[WARNING] Error creating index %s: %s", index, err) - } - - // Make sure if the resource exist it is part of correct alias - if strings.Contains(fmt.Sprintf("%s", err), "resource_already_exists_exception") { - body := fmt.Sprintf(`{ - "actions": [ - { - "add": { - "index": "%s", - "alias": "%s", - "is_write_index": true - } - } - ] - }`, initialIndexName, index) - - aliasResp, aerr := project.Es.Aliases(ctx, opensearchapi.AliasesReq{ - Body: strings.NewReader(body), - }) - if aerr != nil { - log.Printf("[WARNING] Failed to ensure alias %s for index %s: %s", index, initialIndexName, aerr) - return - } - - res := aliasResp.Inspect().Response - defer res.Body.Close() - - if res.StatusCode >= 300 { - log.Printf("[WARNING] Alias enforcement failed: %s", res.String()) - return - } - } - } else { - if res.IsError() { - if !strings.Contains(res.String(), "resource_already_exists_exception") { - log.Printf("[DEBUG] Error creating index %s with custom config: %s", index, res.String()) - } - - } else { - log.Printf("[DEBUG] Successfully created index %s with custom config", index) - } - } - - if ismReady { - if err := ensureOpensearchIndexRolloverAliasSetting(ctx, opensearchUrl, initialIndexName, index); err != nil { - log.Printf("[WARNING] Failed ensuring rollover_alias on index %s: %s", initialIndexName, err) - } - - if err := ensureOpensearchIndexISMPolicy(ctx, opensearchUrl, initialIndexName, ismPolicyName); err != nil { - log.Printf("[WARNING] Failed attaching ISM policy '%s' to %s: %s", ismPolicyName, initialIndexName, err) - } - - continue - } - - rolloverResp, err := project.Es.Indices.Rollover(ctx, opensearchapi.IndicesRolloverReq{ - Alias: index, - Body: bytes.NewReader(rolloverConfig), - }) - - if err != nil { - if !strings.Contains(fmt.Sprintf("%s", err), "serverless mode") && !strings.Contains(fmt.Sprintf("%s", err), "status: 404") { - log.Printf("[WARNING] Problem during rollover config for %s: %s", index, err) - } - - continue - } - - rolloverRes := rolloverResp.Inspect().Response - defer rolloverRes.Body.Close() - if rolloverRes.IsError() { - log.Printf("[ERROR] Rollover config failed for %s: %s", index, rolloverRes.String()) - } else { - log.Printf("[INFO] Rollover executed successfully for %s", index) - } - + if strings.ToLower(os.Getenv("SHUFFLE_SKIP_EXECUTION_LIVE_MIGRATION")) == "true" { + log.Printf("[INFO] Skipping in-flight execution migration to workflowexecution_live (SHUFFLE_SKIP_EXECUTION_LIVE_MIGRATION=true)") + } else if err := migrateInFlightExecutionsToLive(ctx); err != nil { + log.Printf("[WARNING] Failed migrating in-flight executions to live index: %s", err) } - if fixResult, fixErr := FixOpensearchIndexPrefix(ctx); fixErr != nil { - log.Printf("[WARNING] Alias verification after init failed: %s", fixErr) - } else if !fixResult.Success { - log.Printf("[WARNING] Alias verification after init completed with warnings: %s", fixResult.Reason) + if strings.ToLower(os.Getenv("SHUFFLE_SKIP_EXECUTION_ARCHIVAL_SWEEP")) == "true" { + log.Printf("[WARNING] Execution archival sweep disabled (SHUFFLE_SKIP_EXECUTION_ARCHIVAL_SWEEP=true) - workflowexecution_live will grow unbounded") } else { - log.Printf("[INFO] Alias verification after init passed: expected aliases=%d found=%d", fixResult.ExpectedAliases, fixResult.FoundAliases) - } - -} - -func ensureOpensearchIndexRolloverAlias(indexConfig []byte, alias string) []byte { - unmarshalled := map[string]interface{}{} - if err := json.Unmarshal(indexConfig, &unmarshalled); err != nil { - return indexConfig + go StartExecutionArchivalSweeper(context.Background()) } - settings, ok := unmarshalled["settings"].(map[string]interface{}) - if !ok || settings == nil { - settings = map[string]interface{}{} - } - - settings["plugins.index_state_management.rollover_alias"] = alias - unmarshalled["settings"] = settings - - updated, err := json.Marshal(unmarshalled) - if err != nil { - return indexConfig - } - - return updated -} - -func getOpensearchISMRolloverConditions(rolloverConfig []byte) map[string]interface{} { - defaultConditions := map[string]interface{}{ - "min_index_age": "90d", - "min_size": "40gb", - "min_doc_count": 1000000, - } - - parsed := struct { - Conditions map[string]interface{} `json:"conditions"` - }{} - - if err := json.Unmarshal(rolloverConfig, &parsed); err != nil { - return defaultConditions - } - - if len(parsed.Conditions) == 0 { - return defaultConditions - } - - conditions := map[string]interface{}{} - if value, ok := parsed.Conditions["min_index_age"]; ok { - conditions["min_index_age"] = value - } else if value, ok := parsed.Conditions["max_age"]; ok { - conditions["min_index_age"] = value - } - - if value, ok := parsed.Conditions["min_size"]; ok { - conditions["min_size"] = value - } else if value, ok := parsed.Conditions["max_size"]; ok { - conditions["min_size"] = value - } - - if value, ok := parsed.Conditions["min_doc_count"]; ok { - conditions["min_doc_count"] = value - } else if value, ok := parsed.Conditions["max_docs"]; ok { - conditions["min_doc_count"] = value - } - - if len(conditions) == 0 { - return defaultConditions - } - - return conditions -} - -func ensureOpensearchISMRolloverPolicy(ctx context.Context, opensearchUrl string, aliases []string, rolloverConfig []byte, policyName string) (bool, error) { - conditions := getOpensearchISMRolloverConditions(rolloverConfig) - - patterns := []string{} - for _, alias := range aliases { - patterns = append(patterns, fmt.Sprintf("%s-*", alias)) - } - - policyBody := map[string]interface{}{ - "policy": map[string]interface{}{ - "description": "Shuffle rollover policy", - "default_state": "hot", - "states": []map[string]interface{}{ - { - "name": "hot", - "actions": []map[string]interface{}{ - { - "rollover": conditions, - }, - }, - "transitions": []interface{}{}, - }, - }, - "ism_template": []map[string]interface{}{ - { - "index_patterns": patterns, - "priority": 100, - }, - }, - }, - } - - policyData, err := json.Marshal(policyBody) - if err != nil { - return false, err - } - - req, err := http.NewRequestWithContext(ctx, "PUT", fmt.Sprintf("%s/_plugins/_ism/policies/%s", opensearchUrl, policyName), bytes.NewReader(policyData)) - if err != nil { - return false, err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := project.Es.Client.Transport.Perform(req) - if err != nil { - return false, err - } - defer resp.Body.Close() - - body, _ := ioutil.ReadAll(resp.Body) - if resp.StatusCode >= 300 { - if resp.StatusCode == 404 || resp.StatusCode == 400 { - if strings.Contains(strings.ToLower(string(body)), "_plugins/_ism") || strings.Contains(strings.ToLower(string(body)), "no handler found") { - log.Printf("[INFO] ISM plugin not available. Falling back to direct rollover") - return false, nil - } - } - - return false, fmt.Errorf("status: %d, body: %s", resp.StatusCode, string(body)) - } - - log.Printf("[INFO] Ensured ISM rollover policy '%s' for %d index patterns", policyName, len(patterns)) - return true, nil -} - -func ensureOpensearchIndexRolloverAliasSetting(ctx context.Context, opensearchUrl, indexName, alias string) error { - settingsBody := map[string]interface{}{ - "index": map[string]interface{}{ - "plugins.index_state_management.rollover_alias": alias, - }, - } - - body, err := json.Marshal(settingsBody) - if err != nil { - return err - } - - req, err := http.NewRequestWithContext(ctx, "PUT", fmt.Sprintf("%s/%s/_settings", opensearchUrl, indexName), bytes.NewReader(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := project.Es.Client.Transport.Perform(req) - if err != nil { - return err - } - defer resp.Body.Close() - - respBody, _ := ioutil.ReadAll(resp.Body) - if resp.StatusCode >= 300 { - if resp.StatusCode == 404 && strings.Contains(strings.ToLower(string(respBody)), "index_not_found_exception") { - return nil - } - - return fmt.Errorf("status: %d, body: %s", resp.StatusCode, string(respBody)) - } - - return nil -} - -func ensureOpensearchIndexISMPolicy(ctx context.Context, opensearchUrl, indexName, policyName string) error { - policyBody := map[string]interface{}{ - "policy_id": policyName, - } - - body, err := json.Marshal(policyBody) - if err != nil { - return err - } - - req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/_plugins/_ism/add/%s", opensearchUrl, indexName), bytes.NewReader(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := project.Es.Client.Transport.Perform(req) - if err != nil { - return err - } - defer resp.Body.Close() - - respBody, _ := ioutil.ReadAll(resp.Body) - if resp.StatusCode >= 300 { - lowerResp := strings.ToLower(string(respBody)) - if strings.Contains(lowerResp, "already has a policy") { - return nil - } - - if resp.StatusCode == 404 && strings.Contains(lowerResp, "index_not_found_exception") { - return nil - } - - return fmt.Errorf("status: %d, body: %s", resp.StatusCode, string(respBody)) - } - - return nil + go StartNotificationRetentionSweeper(context.Background()) } func ListVulnerabilities(ctx context.Context, ecosystem string, inputcursor string) ([]OSVVulnerability, string, error) { diff --git a/db_connector_opensearch_test.go b/db_connector_opensearch_test.go new file mode 100644 index 00000000..36b05f94 --- /dev/null +++ b/db_connector_opensearch_test.go @@ -0,0 +1,186 @@ +package shuffle + +import ( + "encoding/json" + "sort" + "testing" +) + +func TestBuildAliasSearchBody(t *testing.T) { + body, err := buildAliasSearchBody("last_cleared", "abc-123") + if err != nil { + t.Fatalf("buildAliasSearchBody: %v", err) + } + + var doc map[string]interface{} + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if doc["size"] != float64(1) { + t.Fatalf("expected size 1, got %v", doc["size"]) + } + + query := doc["query"].(map[string]interface{}) + ids := query["ids"].(map[string]interface{})["values"].([]interface{}) + if len(ids) != 1 || ids[0] != "abc-123" { + t.Fatalf("unexpected ids values: %v", ids) + } + + sortArr := doc["sort"].([]interface{}) + first := sortArr[0].(map[string]interface{})["last_cleared"].(map[string]interface{}) + if first["order"] != "desc" || first["unmapped_type"] != "long" { + t.Fatalf("unexpected sort: %v", first) + } +} + +func TestBuildAliasSearchBodyNoSort(t *testing.T) { + body, err := buildAliasSearchBody("", "x") + if err != nil { + t.Fatalf("buildAliasSearchBody: %v", err) + } + + var doc map[string]interface{} + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if _, ok := doc["sort"]; ok { + t.Fatalf("expected no sort when sortField empty") + } +} + +func TestCollapseGenerationOrdering(t *testing.T) { + gens := []string{"org_stats-000001", "org_stats-000004", "org_stats-000002"} + sort.Slice(gens, func(i, j int) bool { + return getOpensearchGeneration(gens[i]) > getOpensearchGeneration(gens[j]) + }) + + want := []string{"org_stats-000004", "org_stats-000002", "org_stats-000001"} + for i := range want { + if gens[i] != want[i] { + t.Fatalf("expected %v, got %v", want, gens) + } + } +} + +func TestResolveAppendIndexCreationTarget(t *testing.T) { + tests := []struct { + name string + existingIndices []string + index string + wantTarget string + wantAlreadyExists bool + }{ + { + name: "fresh index with no existing generations", + existingIndices: []string{"some_other_index-000001"}, + index: "workflowexecution", + wantTarget: "workflowexecution-000001", + wantAlreadyExists: false, + }, + { + name: "already-collapsed archive sitting at generation 3", + existingIndices: []string{"workflowexecution-000003", "shuffle_logs-000001"}, + index: "workflowexecution", + wantTarget: "workflowexecution-000003", + wantAlreadyExists: true, + }, + { + name: "multiple existing generations picks the highest", + existingIndices: []string{"workflowexecution-000001", "workflowexecution-000002", "workflowexecution-000005"}, + index: "workflowexecution", + wantTarget: "workflowexecution-000005", + wantAlreadyExists: true, + }, + { + name: "does not match a different index with a similar prefix", + existingIndices: []string{"workflowexecution_live-000001"}, + index: "workflowexecution", + wantTarget: "workflowexecution-000001", + wantAlreadyExists: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + target, alreadyExists := resolveAppendIndexCreationTarget(tt.existingIndices, tt.index) + if target != tt.wantTarget || alreadyExists != tt.wantAlreadyExists { + t.Fatalf("expected (%s, %v), got (%s, %v)", tt.wantTarget, tt.wantAlreadyExists, target, alreadyExists) + } + }) + } +} + +func TestOpensearchMappingsDiffer(t *testing.T) { + kw := func() map[string]interface{} { return map[string]interface{}{"type": "keyword"} } + + // datastore_ngram desired: key, org_id keyword; amount long + if !opensearchMappingsDiffer("datastore_ngram", nil) { + t.Fatalf("expected differ when no properties present") + } + + if opensearchMappingsDiffer("datastore_ngram", map[string]interface{}{ + "key": kw(), + "org_id": kw(), + "amount": map[string]interface{}{"type": "long"}, + }) { + t.Fatalf("expected no differ when datastore_ngram fields match") + } + + // amount wrong type (keyword instead of long) -> differs + if !opensearchMappingsDiffer("datastore_ngram", map[string]interface{}{ + "key": kw(), + "org_id": kw(), + "amount": kw(), + }) { + t.Fatalf("expected differ on wrong type") + } + + // notifications.image must be index:false; live index:true -> differs + if !opensearchMappingsDiffer("notifications", map[string]interface{}{ + "id": kw(), + "org_id": kw(), + "user_id": kw(), + "created_at": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "updated_at": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "amount": map[string]interface{}{"type": "long"}, + "image": map[string]interface{}{"type": "keyword", "index": true}, + "read": map[string]interface{}{"type": "boolean"}, + "ignored": map[string]interface{}{"type": "boolean"}, + "dismissable": map[string]interface{}{"type": "boolean"}, + "personal": map[string]interface{}{"type": "boolean"}, + }) { + t.Fatalf("expected differ when image is indexed") + } + + // notifications with image index:false matches + if opensearchMappingsDiffer("notifications", map[string]interface{}{ + "id": kw(), + "org_id": kw(), + "user_id": kw(), + "execution_id": kw(), + "workflow_id": kw(), + "org_notification_id": kw(), + "created_at": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "updated_at": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "amount": map[string]interface{}{"type": "long"}, + "image": map[string]interface{}{"type": "keyword", "index": false}, + "read": map[string]interface{}{"type": "boolean"}, + "ignored": map[string]interface{}{"type": "boolean"}, + "dismissable": map[string]interface{}{"type": "boolean"}, + "personal": map[string]interface{}{"type": "boolean"}, + }) { + t.Fatalf("expected no differ when image is not indexed") + } +} +func TestWorkflowExecutionLiveReusesExecutionMapping(t *testing.T) { + liveMapping := opensearchMappingsFor("workflowexecution_live") + archiveMapping := opensearchMappingsFor("workflowexecution") + + liveProps, _ := liveMapping["properties"].(map[string]interface{}) + archiveProps, _ := archiveMapping["properties"].(map[string]interface{}) + if len(liveProps) == 0 || len(liveProps) != len(archiveProps) { + t.Fatalf("expected workflowexecution_live mapping to match workflowexecution mapping, got %d vs %d fields", len(liveProps), len(archiveProps)) + } +} diff --git a/execution_lifecycle.go b/execution_lifecycle.go new file mode 100644 index 00000000..010c105a --- /dev/null +++ b/execution_lifecycle.go @@ -0,0 +1,613 @@ +package shuffle + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + "time" + + "github.com/shuffle/opensearch-go/v4/opensearchapi" +) + +// ErrExecutionArchived is returned by SetWorkflowExecution when a write +// targets an execution_id that has already been moved to the OpenSearch +// archive. Archived executions are immutable; callers (e.g. the /rerun and force-continue HTTP handlers) +// should turn this into a clean, non-500 response instead of retrying or failing loudly. +var ErrExecutionArchived = errors.New("execution has been archived and can no longer be modified") + +const ( + defaultExecutionGracePeriod = time.Hour + defaultExecutionArchiveSweepPeriod = 30 * time.Minute +) + +// isTerminalExecutionStatus reports whether a WorkflowExecution.Status value +// represents a finished run. Terminal executions are eligible for archival +// once they've sat in the live index for longer than getExecutionGracePeriod(). +func isTerminalExecutionStatus(status string) bool { + switch status { + case "FINISHED", "ABORTED", "FAILURE": + return true + default: + return false + } +} + +// liveExecutionBaseIndex is the base index name (before GetESIndexPrefix) for +// the small, non-rolling, keyed index that holds in-flight and +// recently-terminal executions. All execution writes target this index. +func liveExecutionBaseIndex() string { + return "workflowexecution_live" +} + +// archiveExecutionBaseIndex is the base index name for the existing +// "workflowexecution" alias, repurposed as an append-only, rollover-managed +// archive for confirmed-terminal executions. +func archiveExecutionBaseIndex() string { + return "workflowexecution" +} + +// getExecutionGracePeriod returns how long a terminal execution stays in the +// live index before becoming eligible for archival. Configurable via +// OPENSEARCH_EXECUTION_GRACE_PERIOD (Go duration string, e.g. "1h", "90m"). +// Falls back to the 1h default on missing or invalid input. +func getExecutionGracePeriod() time.Duration { + raw := strings.TrimSpace(os.Getenv("OPENSEARCH_EXECUTION_GRACE_PERIOD")) + if raw == "" { + return defaultExecutionGracePeriod + } + + parsed, err := time.ParseDuration(raw) + if err != nil { + log.Printf("[WARNING] Invalid OPENSEARCH_EXECUTION_GRACE_PERIOD %q, using default %s: %s", raw, defaultExecutionGracePeriod, err) + return defaultExecutionGracePeriod + } + + return parsed +} + +// getExecutionArchiveSweepInterval returns how often the background archival +// sweep runs. Configurable via OPENSEARCH_EXECUTION_ARCHIVE_SWEEP_INTERVAL. +// Falls back to the 30m default on missing or invalid input. +func getExecutionArchiveSweepInterval() time.Duration { + raw := strings.TrimSpace(os.Getenv("OPENSEARCH_EXECUTION_ARCHIVE_SWEEP_INTERVAL")) + if raw == "" { + return defaultExecutionArchiveSweepPeriod + } + + parsed, err := time.ParseDuration(raw) + if err != nil { + log.Printf("[WARNING] Invalid OPENSEARCH_EXECUTION_ARCHIVE_SWEEP_INTERVAL %q, using default %s: %s", raw, defaultExecutionArchiveSweepPeriod, err) + return defaultExecutionArchiveSweepPeriod + } + + return parsed +} + +func init() { + if mapping, ok := opensearchCoreMappings["workflowexecution"]; ok { + opensearchCoreMappings["workflowexecution_live"] = mapping + } +} + +// resolveExecutionWriteTarget decides which base index a write for a given +// execution should target, and whether that write must also clean up an +// existing archive copy (unarchive). archiveStatusLookup returns +// (status, true) if the execution already exists in the archive, or +// ("", false) if it doesn't. +// +// Every new/live execution writes to live. An execution already archived +// only blocks the write if BOTH the archive copy and the incoming write are +// terminal (a duplicate/late re-affirmation of an already-finished +// execution) - that's rejected with ErrExecutionArchived. If the archive +// copy is terminal but the incoming write is non-terminal, this is a +// legitimate reopen (async decision-fixup race, or a slower recovery/ +// failover path that fires after the grace window) - unarchive is +// signaled so the caller moves the doc back to live instead of leaving two +// diverging copies or writing into the (supposed to be append-only) archive. +func resolveExecutionWriteTarget(incomingStatus string, archiveStatusLookup func() (status string, found bool)) (targetIndex string, unarchive bool, err error) { + _, found := archiveStatusLookup() + if !found { + return liveExecutionBaseIndex(), false, nil + } + + if isTerminalExecutionStatus(incomingStatus) { + return "", false, ErrExecutionArchived + } + + return liveExecutionBaseIndex(), true, nil +} + +// findExecutionInArchive searches the archive alias for a doc with this +// execution_id, returning the concrete backing index and status of the newest +// matching generation if found. +func findExecutionInArchive(ctx context.Context, aliasName, executionId string) (index string, status string, found bool) { + var buf bytes.Buffer + query := map[string]interface{}{ + "size": 1, + "query": map[string]interface{}{ + "ids": map[string]interface{}{ + "values": []string{executionId}, + }, + }, + "sort": []map[string]interface{}{ + { + "edited": map[string]interface{}{ + "order": "desc", + "unmapped_type": "long", + }, + }, + { + "created": map[string]interface{}{ + "order": "desc", + "unmapped_type": "long", + }, + }, + }, + } + if err := json.NewEncoder(&buf).Encode(query); err != nil { + return "", "", false + } + + resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{ + Indices: []string{aliasName}, + Body: &buf, + Params: opensearchapi.SearchParams{ + TrackTotalHits: true, + }, + }) + if err != nil { + return "", "", false + } + + res := resp.Inspect().Response + defer res.Body.Close() + if res.StatusCode == 404 { + return "", "", false + } + + respBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", "", false + } + + wrapped := ExecutionSearchWrapper{} + if err := json.Unmarshal(respBody, &wrapped); err != nil || len(wrapped.Hits.Hits) == 0 { + return "", "", false + } + + top := wrapped.Hits.Hits[0] + return top.Index, top.Source.Status, true +} + +// deleteExecutionFromArchiveIndex removes a single execution document from a +// concrete archive backing index (not the alias). +func deleteExecutionFromArchiveIndex(ctx context.Context, concreteIndex, executionId string) error { + resp, err := project.Es.Document.Delete(ctx, opensearchapi.DocumentDeleteReq{ + Index: concreteIndex, + DocumentID: executionId, + }) + if err != nil { + return err + } + + res := resp.Inspect().Response + defer res.Body.Close() + if res.StatusCode != 200 && res.StatusCode != 404 { + respBody, _ := ioutil.ReadAll(res.Body) + return fmt.Errorf("failed deleting %s from archive index %s: status=%d body=%s", executionId, concreteIndex, res.StatusCode, string(respBody)) + } + + return nil +} + +// writeExecutionDocument is the single choke point for persisting a +// WorkflowExecution document to OpenSearch. It always targets the live index +// unless the execution is already archived and the incoming write is itself +// terminal (rejected with ErrExecutionArchived); if the execution is +// archived but the incoming write is non-terminal, the doc is unarchived +// (written to live, then removed from the archive) before returning. +func writeExecutionDocument(ctx context.Context, executionId string, incomingStatus string, data []byte) error { + archiveAlias := strings.ToLower(GetESIndexPrefix(archiveExecutionBaseIndex())) + archiveIndex, archiveStatus, archiveFound := findExecutionInArchive(ctx, archiveAlias, executionId) + + target, unarchive, err := resolveExecutionWriteTarget(incomingStatus, func() (string, bool) { + return archiveStatus, archiveFound + }) + if err != nil { + return err + } + + if err := indexEs(ctx, target, executionId, data); err != nil { + return err + } + + if unarchive { + if delErr := deleteExecutionFromArchiveIndex(ctx, archiveIndex, executionId); delErr != nil { + log.Printf("[WARNING][%s] Unarchived execution to live but failed to remove stale archive copy from %s: %s", executionId, archiveIndex, delErr) + } + } + + return nil +} + +// getExecutionDocumentWithLookups tries liveLookup first; if it errors, falls +// back to archiveLookup. Factored out from getExecutionDocument so the +// try-live-then-archive control flow is unit-testable without a live +// OpenSearch cluster. +func getExecutionDocumentWithLookups(liveLookup, archiveLookup func() (*WorkflowExecution, error)) (*WorkflowExecution, error) { + exec, err := liveLookup() + if err == nil { + return exec, nil + } + + return archiveLookup() +} + +// getExecutionDocument fetches a single WorkflowExecution by id, checking the +// live index first and falling back to the archive alias for executions that +// have already been archived. +func getExecutionDocument(ctx context.Context, executionId string) (*WorkflowExecution, error) { + liveLookup := func() (*WorkflowExecution, error) { + resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{ + Index: strings.ToLower(GetESIndexPrefix(liveExecutionBaseIndex())), + DocumentID: executionId, + }) + if err != nil { + return nil, err + } + + res := resp.Inspect().Response + defer res.Body.Close() + if res.StatusCode == 404 { + return nil, errors.New("execution doesn't exist in live index") + } + + respBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + wrapped := ExecWrapper{} + if err := json.Unmarshal(respBody, &wrapped); err != nil || !wrapped.Found { + return nil, errors.New("execution not found in live index") + } + + return &wrapped.Source, nil + } + + archiveLookup := func() (*WorkflowExecution, error) { + resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{ + Index: strings.ToLower(GetESIndexPrefix(archiveExecutionBaseIndex())), + DocumentID: executionId, + }) + if err != nil { + if strings.Contains(err.Error(), "has more than one index associated with it") { + return getWorkflowExecutionByAliasSearch(ctx, strings.ToLower(GetESIndexPrefix(archiveExecutionBaseIndex())), executionId) + } + return nil, err + } + + res := resp.Inspect().Response + defer res.Body.Close() + if res.StatusCode == 404 { + return nil, errors.New("execution doesn't exist") + } + + respBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + wrapped := ExecWrapper{} + if err := json.Unmarshal(respBody, &wrapped); err != nil || !wrapped.Found { + return nil, errors.New("execution not found in archive") + } + + return &wrapped.Source, nil + } + + return getExecutionDocumentWithLookups(liveLookup, archiveLookup) +} + +// executionSearchIndices returns the list of concrete index/alias names that +// execution list/history queries should search across. +func executionSearchIndices() []string { + return []string{ + strings.ToLower(GetESIndexPrefix(liveExecutionBaseIndex())), + strings.ToLower(GetESIndexPrefix(archiveExecutionBaseIndex())), + } +} + +// dedupExecutionsByID collapses a result set that may contain more than one +// doc for the same execution_id down to one entry per execution_id, keeping +// whichever looks newest. +func dedupExecutionsByID(executions []WorkflowExecution) []WorkflowExecution { + newest := map[string]WorkflowExecution{} + order := []string{} + + for _, exec := range executions { + existing, found := newest[exec.ExecutionId] + if !found { + newest[exec.ExecutionId] = exec + order = append(order, exec.ExecutionId) + continue + } + + existingTs := existing.CompletedAt + if existingTs == 0 { + existingTs = existing.StartedAt + } + candidateTs := exec.CompletedAt + if candidateTs == 0 { + candidateTs = exec.StartedAt + } + + if candidateTs > existingTs { + newest[exec.ExecutionId] = exec + } + } + + deduped := make([]WorkflowExecution, 0, len(order)) + for _, id := range order { + deduped = append(deduped, newest[id]) + } + + return deduped +} + +// buildArchivalSweepQuery builds the OpenSearch query body for finding +// executions in the live index that are eligible for archival. +func buildArchivalSweepQuery(now time.Time, grace time.Duration) map[string]interface{} { + cutoff := now.Add(-grace).Unix() + + return map[string]interface{}{ + "size": 1000, + "query": map[string]interface{}{ + "bool": map[string]interface{}{ + "must": []map[string]interface{}{ + { + "terms": map[string]interface{}{ + "status": []string{"FINISHED", "ABORTED", "FAILURE"}, + }, + }, + { + "range": map[string]interface{}{ + "completed_at": map[string]interface{}{ + "lt": cutoff, + }, + }, + }, + }, + }, + }, + } +} + +// resolveArchiveWriteTarget decides where an archive write for a given +// execution_id should go. +func resolveArchiveWriteTarget(aliasName string, existingLookup func() (index string, found bool)) (target string, isAlias bool) { + if existingIndex, found := existingLookup(); found { + return existingIndex, false + } + + return aliasName, true +} + +// archiveExecutionDocument writes an execution document into the archive, +// idempotently. +func archiveExecutionDocument(ctx context.Context, executionId string, data []byte) error { + aliasName := strings.ToLower(GetESIndexPrefix(archiveExecutionBaseIndex())) + + target, isAlias := resolveArchiveWriteTarget(aliasName, func() (string, bool) { + index, _, found := findExecutionInArchive(ctx, aliasName, executionId) + return index, found + }) + + if isAlias { + return indexEs(ctx, archiveExecutionBaseIndex(), executionId, data) + } + + _, err := project.Es.Index(ctx, opensearchapi.IndexReq{ + Index: target, + DocumentID: executionId, + Body: bytes.NewReader(data), + Params: opensearchapi.IndexParams{ + Refresh: "true", + }, + }) + return err +} + +// sweepArchivableExecutions finds executions in the live index that have +// been terminal for longer than getExecutionGracePeriod(), copies each into +// the archive, and deletes it from live. +func sweepArchivableExecutions(ctx context.Context) error { + const lockKey = "opensearch_execution_sweep_lock" + if _, err := GetCache(ctx, lockKey); err == nil { + log.Printf("[DEBUG] Execution archival sweep already in progress elsewhere, skipping this pass") + return nil + } + _ = SetCache(ctx, lockKey, []byte("1"), 300) + + query := buildArchivalSweepQuery(time.Now(), getExecutionGracePeriod()) + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(query); err != nil { + return err + } + + resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{ + Indices: []string{strings.ToLower(GetESIndexPrefix(liveExecutionBaseIndex()))}, + Body: &buf, + }) + if err != nil { + if strings.Contains(err.Error(), "index_not_found_exception") { + return nil + } + return err + } + + res := resp.Inspect().Response + defer res.Body.Close() + if res.StatusCode == 404 { + return nil + } + + respBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + + wrapped := ExecutionSearchWrapper{} + if err := json.Unmarshal(respBody, &wrapped); err != nil { + return err + } + + archived := 0 + for _, hit := range wrapped.Hits.Hits { + data, err := json.Marshal(hit.Source) + if err != nil { + log.Printf("[WARNING] Failed marshalling execution %s for archival: %s", hit.Source.ExecutionId, err) + continue + } + + if err := archiveExecutionDocument(ctx, hit.Source.ExecutionId, data); err != nil { + log.Printf("[WARNING] Failed archiving execution %s: %s", hit.Source.ExecutionId, err) + continue + } + + if err := DeleteKey(ctx, liveExecutionBaseIndex(), hit.Source.ExecutionId); err != nil { + log.Printf("[WARNING] Archived execution %s but failed deleting it from live index: %s", hit.Source.ExecutionId, err) + continue + } + + archived++ + } + + if archived > 0 { + log.Printf("[INFO] Archived %d terminal executions from live to archive index", archived) + } + + return nil +} + +// StartExecutionArchivalSweeper runs sweepArchivableExecutions on a ticker at +// getExecutionArchiveSweepInterval(). +func StartExecutionArchivalSweeper(ctx context.Context) { + if project.DbType != "opensearch" { + return + } + + interval := getExecutionArchiveSweepInterval() + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := sweepArchivableExecutions(ctx); err != nil { + log.Printf("[WARNING] Execution archival sweep failed: %s", err) + } + } + } +} + +// buildInFlightExecutionsQuery builds the query used by the one-time startup +// migration to find non-terminal executions in the legacy/archive index that +// need to move to the new live index. +func buildInFlightExecutionsQuery() map[string]interface{} { + return map[string]interface{}{ + "size": 1000, + "query": map[string]interface{}{ + "bool": map[string]interface{}{ + "must": []map[string]interface{}{ + { + "terms": map[string]interface{}{ + "status": []string{"EXECUTING", "WAITING"}, + }, + }, + }, + }, + }, + } +} + +// migrateInFlightExecutionsToLive is a one-time startup migration for +// existing deployments. +func migrateInFlightExecutionsToLive(ctx context.Context) error { + const lockKey = "opensearch_execution_migration_lock" + if _, err := GetCache(ctx, lockKey); err == nil { + log.Printf("[DEBUG] In-flight execution migration already in progress elsewhere, skipping") + return nil + } + _ = SetCache(ctx, lockKey, []byte("1"), 600) + + query := buildInFlightExecutionsQuery() + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(query); err != nil { + return err + } + + resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{ + Indices: []string{strings.ToLower(GetESIndexPrefix(archiveExecutionBaseIndex()))}, + Body: &buf, + }) + if err != nil { + if strings.Contains(err.Error(), "index_not_found_exception") { + return nil + } + return err + } + + res := resp.Inspect().Response + defer res.Body.Close() + if res.StatusCode == 404 { + return nil + } + + respBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + + wrapped := ExecutionSearchWrapper{} + if err := json.Unmarshal(respBody, &wrapped); err != nil { + return err + } + + migrated := 0 + for _, hit := range wrapped.Hits.Hits { + data, err := json.Marshal(hit.Source) + if err != nil { + log.Printf("[WARNING] Failed marshalling execution %s for live migration: %s", hit.Source.ExecutionId, err) + continue + } + + if err := indexEs(ctx, liveExecutionBaseIndex(), hit.Source.ExecutionId, data); err != nil { + log.Printf("[WARNING] Failed migrating execution %s to live index: %s", hit.Source.ExecutionId, err) + continue + } + + if err := deleteExecutionFromArchiveIndex(ctx, hit.Index, hit.Source.ExecutionId); err != nil { + log.Printf("[WARNING] Migrated execution %s to live but failed deleting legacy copy from %s: %s", hit.Source.ExecutionId, hit.Index, err) + continue + } + + migrated++ + } + + if migrated > 0 { + log.Printf("[INFO] Migrated %d in-flight executions from legacy index to workflowexecution_live", migrated) + } + + return nil +} diff --git a/execution_lifecycle_test.go b/execution_lifecycle_test.go new file mode 100644 index 00000000..b1482a29 --- /dev/null +++ b/execution_lifecycle_test.go @@ -0,0 +1,336 @@ +package shuffle + +import ( + "errors" + "os" + "strings" + "testing" + "time" +) + +func TestIsTerminalExecutionStatus(t *testing.T) { + terminal := []string{"FINISHED", "ABORTED", "FAILURE"} + for _, status := range terminal { + if !isTerminalExecutionStatus(status) { + t.Fatalf("expected %s to be terminal", status) + } + } + + nonTerminal := []string{"EXECUTING", "WAITING", ""} + for _, status := range nonTerminal { + if isTerminalExecutionStatus(status) { + t.Fatalf("expected %s to be non-terminal", status) + } + } +} + +func TestExecutionIndexNames(t *testing.T) { + if liveExecutionBaseIndex() != "workflowexecution_live" { + t.Fatalf("expected workflowexecution_live, got %s", liveExecutionBaseIndex()) + } + if archiveExecutionBaseIndex() != "workflowexecution" { + t.Fatalf("expected workflowexecution, got %s", archiveExecutionBaseIndex()) + } +} + +func TestGetExecutionGracePeriodDefault(t *testing.T) { + os.Unsetenv("OPENSEARCH_EXECUTION_GRACE_PERIOD") + if got := getExecutionGracePeriod(); got != time.Hour { + t.Fatalf("expected 1h default, got %s", got) + } +} + +func TestGetExecutionGracePeriodOverride(t *testing.T) { + os.Setenv("OPENSEARCH_EXECUTION_GRACE_PERIOD", "90m") + defer os.Unsetenv("OPENSEARCH_EXECUTION_GRACE_PERIOD") + + if got := getExecutionGracePeriod(); got != 90*time.Minute { + t.Fatalf("expected 90m override, got %s", got) + } +} + +func TestGetExecutionGracePeriodInvalidFallsBackToDefault(t *testing.T) { + os.Setenv("OPENSEARCH_EXECUTION_GRACE_PERIOD", "not-a-duration") + defer os.Unsetenv("OPENSEARCH_EXECUTION_GRACE_PERIOD") + + if got := getExecutionGracePeriod(); got != time.Hour { + t.Fatalf("expected fallback to 1h default on invalid input, got %s", got) + } +} + +func TestGetExecutionArchiveSweepIntervalDefault(t *testing.T) { + os.Unsetenv("OPENSEARCH_EXECUTION_ARCHIVE_SWEEP_INTERVAL") + if got := getExecutionArchiveSweepInterval(); got != 30*time.Minute { + t.Fatalf("expected 30m default, got %s", got) + } +} + +func TestResolveExecutionWriteTarget(t *testing.T) { + tests := []struct { + name string + incomingStatus string + archiveHasStatus string + wantIndex string + wantUnarchive bool + wantErr error + }{ + { + name: "new execution not in archive writes to live", + incomingStatus: "EXECUTING", + archiveHasStatus: "", + wantIndex: "workflowexecution_live", + wantUnarchive: false, + wantErr: nil, + }, + { + name: "terminal write against already-archived terminal execution is rejected", + incomingStatus: "FINISHED", + archiveHasStatus: "FINISHED", + wantIndex: "", + wantUnarchive: false, + wantErr: ErrExecutionArchived, + }, + { + name: "non-terminal write against an already-archived execution triggers unarchive", + incomingStatus: "EXECUTING", + archiveHasStatus: "FINISHED", + wantIndex: "workflowexecution_live", + wantUnarchive: true, + wantErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lookup := func() (string, bool) { + if tt.archiveHasStatus == "" { + return "", false + } + return tt.archiveHasStatus, true + } + + index, unarchive, err := resolveExecutionWriteTarget(tt.incomingStatus, lookup) + if err != tt.wantErr { + t.Fatalf("expected err %v, got %v", tt.wantErr, err) + } + if index != tt.wantIndex { + t.Fatalf("expected index %s, got %s", tt.wantIndex, index) + } + if unarchive != tt.wantUnarchive { + t.Fatalf("expected unarchive %v, got %v", tt.wantUnarchive, unarchive) + } + }) + } +} + +func TestGetExecutionDocumentFallsBackToArchive(t *testing.T) { + liveCalled := false + archiveCalled := false + + liveLookup := func() (*WorkflowExecution, error) { + liveCalled = true + return nil, errors.New("execution doesn't exist") + } + archiveLookup := func() (*WorkflowExecution, error) { + archiveCalled = true + return &WorkflowExecution{ExecutionId: "abc"}, nil + } + + exec, err := getExecutionDocumentWithLookups(liveLookup, archiveLookup) + if err != nil { + t.Fatalf("expected no error, got %s", err) + } + if !liveCalled || !archiveCalled { + t.Fatalf("expected both live (tried first) and archive (fallback) to be called: live=%v archive=%v", liveCalled, archiveCalled) + } + if exec.ExecutionId != "abc" { + t.Fatalf("expected execution abc, got %s", exec.ExecutionId) + } +} + +func TestGetExecutionDocumentSkipsArchiveWhenLiveFound(t *testing.T) { + archiveCalled := false + + liveLookup := func() (*WorkflowExecution, error) { + return &WorkflowExecution{ExecutionId: "live-one"}, nil + } + archiveLookup := func() (*WorkflowExecution, error) { + archiveCalled = true + return nil, errors.New("should not be called") + } + + exec, err := getExecutionDocumentWithLookups(liveLookup, archiveLookup) + if err != nil { + t.Fatalf("expected no error, got %s", err) + } + if archiveCalled { + t.Fatalf("archive lookup should not be called when live lookup succeeds") + } + if exec.ExecutionId != "live-one" { + t.Fatalf("expected live-one, got %s", exec.ExecutionId) + } +} + +func TestExecutionSearchIndices(t *testing.T) { + indices := executionSearchIndices() + want := []string{ + strings.ToLower(GetESIndexPrefix(liveExecutionBaseIndex())), + strings.ToLower(GetESIndexPrefix(archiveExecutionBaseIndex())), + } + + if len(indices) != len(want) { + t.Fatalf("expected %d indices, got %d (%v)", len(want), len(indices), indices) + } + for i := range want { + if indices[i] != want[i] { + t.Fatalf("expected index[%d]=%s, got %s", i, want[i], indices[i]) + } + } +} + +func TestDedupExecutionsByID(t *testing.T) { + executions := []WorkflowExecution{ + {ExecutionId: "exec-1", CompletedAt: 100}, + {ExecutionId: "exec-2", CompletedAt: 50}, + {ExecutionId: "exec-1", CompletedAt: 200}, + } + + deduped := dedupExecutionsByID(executions) + if len(deduped) != 2 { + t.Fatalf("expected 2 unique executions, got %d", len(deduped)) + } + + byId := map[string]WorkflowExecution{} + for _, e := range deduped { + byId[e.ExecutionId] = e + } + + if byId["exec-1"].CompletedAt != 200 { + t.Fatalf("expected exec-1 to keep the newest copy (CompletedAt=200), got %d", byId["exec-1"].CompletedAt) + } + if byId["exec-2"].CompletedAt != 50 { + t.Fatalf("expected exec-2 unchanged, got %d", byId["exec-2"].CompletedAt) + } +} + +func TestBuildArchivalSweepQuery(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + grace := time.Hour + + query := buildArchivalSweepQuery(now, grace) + + boolQuery, ok := query["query"].(map[string]interface{})["bool"].(map[string]interface{}) + if !ok { + t.Fatalf("expected bool query, got %v", query) + } + + must, ok := boolQuery["must"].([]map[string]interface{}) + if !ok || len(must) != 2 { + t.Fatalf("expected 2 must clauses (status terms + completed_at range), got %v", must) + } + + terms, ok := must[0]["terms"].(map[string]interface{}) + if !ok { + t.Fatalf("expected first clause to be a terms query on status, got %v", must[0]) + } + statuses, ok := terms["status"].([]string) + if !ok || len(statuses) != 3 { + t.Fatalf("expected 3 terminal statuses, got %v", terms["status"]) + } + + rangeQuery, ok := must[1]["range"].(map[string]interface{}) + if !ok { + t.Fatalf("expected second clause to be a range query on completed_at, got %v", must[1]) + } + completedAtRange, ok := rangeQuery["completed_at"].(map[string]interface{}) + if !ok { + t.Fatalf("expected completed_at range, got %v", rangeQuery) + } + + wantCutoff := now.Add(-grace).Unix() + if completedAtRange["lt"] != wantCutoff { + t.Fatalf("expected cutoff %d, got %v", wantCutoff, completedAtRange["lt"]) + } +} + +func TestResolveArchiveWriteTarget(t *testing.T) { + tests := []struct { + name string + existingIndex string + wantAlias bool + }{ + { + name: "not yet archived writes via alias", + existingIndex: "", + wantAlias: true, + }, + { + name: "already archived in an older generation writes to that concrete index", + existingIndex: "shuffle_workflowexecution_000002", + wantAlias: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lookup := func() (string, bool) { + if tt.existingIndex == "" { + return "", false + } + return tt.existingIndex, true + } + + target, isAlias := resolveArchiveWriteTarget("shuffle_workflowexecution", lookup) + if isAlias != tt.wantAlias { + t.Fatalf("expected isAlias=%v, got %v (target=%s)", tt.wantAlias, isAlias, target) + } + if !tt.wantAlias && target != tt.existingIndex { + t.Fatalf("expected target %s, got %s", tt.existingIndex, target) + } + }) + } +} + +func TestBuildInFlightExecutionsQuery(t *testing.T) { + query := buildInFlightExecutionsQuery() + + boolQuery, ok := query["query"].(map[string]interface{})["bool"].(map[string]interface{}) + if !ok { + t.Fatalf("expected bool query, got %v", query) + } + + must, ok := boolQuery["must"].([]map[string]interface{}) + if !ok || len(must) != 1 { + t.Fatalf("expected 1 must clause (status terms), got %v", must) + } + + terms, ok := must[0]["terms"].(map[string]interface{}) + if !ok { + t.Fatalf("expected terms query on status, got %v", must[0]) + } + statuses, ok := terms["status"].([]string) + if !ok || len(statuses) != 2 { + t.Fatalf("expected 2 non-terminal statuses (EXECUTING, WAITING), got %v", terms["status"]) + } + for _, s := range statuses { + if s == "FINISHED" || s == "ABORTED" || s == "FAILURE" { + t.Fatalf("migration query must never select terminal statuses, got %s", s) + } + } +} + +func TestGetOpensearchRetentionDaysIncludesWorkflowExecution(t *testing.T) { + os.Unsetenv("OPENSEARCH_INDEX_RETENTION_DAYS") + if got := getOpensearchRetentionDays("workflowexecution"); got != "365d" { + t.Fatalf("expected 365d default archive retention, got %q", got) + } +} + +func TestGetOpensearchRetentionDaysOverride(t *testing.T) { + os.Setenv("OPENSEARCH_INDEX_RETENTION_DAYS", `{"workflowexecution": 180}`) + defer os.Unsetenv("OPENSEARCH_INDEX_RETENTION_DAYS") + + if got := getOpensearchRetentionDays("workflowexecution"); got != "180d" { + t.Fatalf("expected 180d override via existing OPENSEARCH_INDEX_RETENTION_DAYS mechanism, got %q", got) + } +} diff --git a/executions.go b/executions.go index 82574ca3..5ddcd3fa 100644 --- a/executions.go +++ b/executions.go @@ -2,25 +2,23 @@ package shuffle import ( //"github.com/goccy/go-json" - "encoding/json" "context" + "encoding/json" + "errors" "fmt" "log" - "time" - "strconv" - "errors" - "sort" + "math/rand" "os" + "sort" + "strconv" "strings" - "math/rand" - "io/ioutil" + "time" "cloud.google.com/go/datastore" - "github.com/shuffle/opensearch-go/v4/opensearchapi" ) // A file built single-handedly for optimising executions. Functions: -// - Fixexecution +// - Fixexecution // - Setexecution // - Getexecution @@ -50,7 +48,7 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor // Very weird edgecase handling for agent cleanup // This is for auto-correctiveness of executions - if len(workflowExecution.Workflow.Actions) == 1 && action.Name == "agent" && innerresult.Action.Name == "agent" && innerresult.Action.ID == "" { + if len(workflowExecution.Workflow.Actions) == 1 && action.Name == "agent" && innerresult.Action.Name == "agent" && innerresult.Action.ID == "" { innerresult.Action.ID = action.ID innerresult.Action.AppName = "AI Agent" } @@ -76,7 +74,7 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor // Special cleanup for agents if innerresult.Action.AppName == "AI Agent" || innerresult.Action.AppName == "Shuffle Agent" { - if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" { + if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "ABORTED" { //if workflowExecution.Status == "FINISHED" { // log.Printf("[DEBUG][%s] Fixexecution: Agent execution is finished, skipping agent result %s", workflowExecution.ExecutionId, innerresult.Action.ID) //} @@ -125,34 +123,33 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor } } - // Overwrites missing statuses - setFinished := mappedOutput.Status == "FINISHED" && mappedOutput.CompletedAt > 0 + setFinished := mappedOutput.Status == "FINISHED" && mappedOutput.CompletedAt > 0 finishFound := false - for decisionIndex, decision := range mappedOutput.Decisions { - if setFinished && decision.RunDetails.Status == "" { + for decisionIndex, decision := range mappedOutput.Decisions { + if setFinished && decision.RunDetails.Status == "" { mappedOutput.Decisions[decisionIndex].RunDetails.Status = "IGNORED" mappedOutput.Decisions[decisionIndex].RunDetails.CompletedAt = time.Now().UnixMilli() decisionsUpdated = true } - if decision.Action == "finish" || decision.Category == "finish" { + if decision.Action == "finish" || decision.Category == "finish" { - if decision.RunDetails.Status != "FINISHED" { - if mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt == 0 { - mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt = time.Now().UnixMilli() + if decision.RunDetails.Status != "FINISHED" { + if mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt == 0 { + mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt = time.Now().UnixMilli() } - mappedOutput.Decisions[decisionIndex].RunDetails.CompletedAt = time.Now().UnixMilli() + mappedOutput.Decisions[decisionIndex].RunDetails.CompletedAt = time.Now().UnixMilli() mappedOutput.Decisions[decisionIndex].RunDetails.Status = "FINISHED" decisionsUpdated = true } - + finishFound = true } } - if finishFound { + if finishFound { mappedOutput.Status = "FINISHED" result.Status = "SUCCESS" @@ -162,7 +159,7 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor break } - if !finishFound && (innerresult.Status == "WAITING" || innerresult.Status == "SUCCESS") || decisionsUpdated { + if !finishFound && (innerresult.Status == "WAITING" || innerresult.Status == "SUCCESS") || decisionsUpdated { if workflowExecution.Results[resultIndex].StartedAt == 0 { workflowExecution.Results[resultIndex].StartedAt = time.Now().UnixMilli() } @@ -185,14 +182,14 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor if decision.Action == "finish" { finishDecisionFound = true - if decision.RunDetails.Status == "" { + if decision.RunDetails.Status == "" { decision.RunDetails.Status = "FINISHED" mappedOutput.Decisions[decisionIndex].RunDetails.Status = "FINISHED" } } parsedDelay, err := strconv.Atoi(decision.Delay) - if err != nil { + if err != nil { parsedDelay = 0 } @@ -390,7 +387,7 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor }() } } else if (result.Status == "" || result.Status == "WAITING") && mappedOutput.Status == "FINISHED" { - if debug { + if debug { log.Printf("[INFO][%s] Agent action %s marked as FINISHED, updating result status to SUCCESS.", workflowExecution.ExecutionId, action.ID) } @@ -840,7 +837,11 @@ func SetWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecuti log.Printf("[DEBUG] Final string size of execution is: %d", len(executionData)) } - err = indexEs(ctx, nameKey, workflowExecution.ExecutionId, executionData) + err = writeExecutionDocument(ctx, workflowExecution.ExecutionId, workflowExecution.Status, executionData) + if err == ErrExecutionArchived { + log.Printf("[INFO][%s] Rejected write to archived execution", workflowExecution.ExecutionId) + return ErrExecutionArchived + } if err != nil { if strings.Contains(err.Error(), "immense term") { retried := false @@ -909,7 +910,7 @@ func SetWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecuti } log.Printf("[DEBUG][%s] Retrying OpenSearch save after trimming remaining oversized values", workflowExecution.ExecutionId) - err = indexEs(ctx, nameKey, workflowExecution.ExecutionId, executionData) + err = writeExecutionDocument(ctx, workflowExecution.ExecutionId, workflowExecution.Status, executionData) } } @@ -1135,48 +1136,12 @@ func GetWorkflowExecution(ctx context.Context, id string, bypassCache ...bool) ( var getErr error = nil if project.DbType == "opensearch" { - resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{ - Index: strings.ToLower(GetESIndexPrefix(nameKey)), - DocumentID: id, - }) - - if err != nil { - if strings.Contains(err.Error(), "has more than one index associated with it") { - fallbackExec, fallbackErr := getWorkflowExecutionByAliasSearch(ctx, strings.ToLower(GetESIndexPrefix(nameKey)), id) - if fallbackErr != nil { - log.Printf("[WARNING][%s] Error for %s: %s", workflowExecution.ExecutionId, cacheKey, err) - log.Printf("[WARNING][%s] WorkflowExecution alias fallback failed for %s: %s", workflowExecution.ExecutionId, cacheKey, fallbackErr) - return workflowExecution, fallbackErr - } - - workflowExecution = fallbackExec - } else { - log.Printf("[WARNING][%s] Error for %s: %s", workflowExecution.ExecutionId, cacheKey, err) - return workflowExecution, err - } + fetched, fetchErr := getExecutionDocument(ctx, id) + if fetchErr != nil { + return workflowExecution, fetchErr } - if err == nil { - res := resp.Inspect().Response - defer res.Body.Close() - if res.StatusCode == 404 { - return workflowExecution, errors.New("execution doesn't exist") - } - - respBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return workflowExecution, err - } - - wrapped := ExecWrapper{} - err = json.Unmarshal(respBody, &wrapped) - //err = gojson.Unmarshal(respBody, &wrapped) - if err != nil && len(wrapped.Source.ExecutionId) == 0 { - return workflowExecution, err - } - - workflowExecution = &wrapped.Source - } + workflowExecution = fetched } else { key := datastore.NameKey(nameKey, strings.ToLower(id), nil) if getErr = project.Dbclient.Get(ctx, key, workflowExecution); getErr != nil { diff --git a/go.mod b/go.mod index d28b4a46..9aaf4454 100644 --- a/go.mod +++ b/go.mod @@ -24,18 +24,17 @@ require ( github.com/google/go-github/v28 v28.1.1 github.com/google/go-querystring v1.1.0 github.com/google/uuid v1.6.0 + github.com/klauspost/compress v1.19.2 github.com/microcosm-cc/bluemonday v1.0.27 github.com/openai/openai-go/v3 v3.8.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/sashabaranov/go-openai v1.40.5 github.com/satori/go.uuid v1.2.0 github.com/sendgrid/sendgrid-go v3.16.1+incompatible - github.com/shirou/gopsutil/v3 v3.24.5 github.com/shuffle/opensearch-go/v4 v4.0.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e golang.org/x/crypto v0.48.0 golang.org/x/oauth2 v0.34.0 - golang.org/x/sys v0.41.0 google.golang.org/api v0.236.0 google.golang.org/appengine v1.6.8 gopkg.in/yaml.v2 v2.4.0 @@ -82,7 +81,6 @@ require ( github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect @@ -98,7 +96,6 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect @@ -114,10 +111,8 @@ require ( github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/sendgrid/rest v2.6.9+incompatible // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect @@ -125,11 +120,8 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect @@ -145,6 +137,7 @@ require ( go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.41.0 // indirect golang.org/x/term v0.40.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.11.0 // indirect diff --git a/go.sum b/go.sum index 5ee18671..17481a00 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf h1:TqhNAT4zKbTdLa62d2HDBFdvgSbIGB3eJE8HqhgiL9I= github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 h1:/P9/RL0xgWE+ehnCUUN5h3RpG3dmoMCOONO1CCvq23Y= @@ -155,8 +157,6 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= @@ -201,7 +201,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= @@ -230,6 +229,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -247,6 +248,8 @@ github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -255,12 +258,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= @@ -304,8 +307,6 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -321,12 +322,6 @@ github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1G github.com/sendgrid/sendgrid-go v3.16.1+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= -github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU= github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -366,10 +361,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ= github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -379,8 +370,6 @@ github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -500,14 +489,12 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -515,8 +502,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/health.go b/health.go index 54873a9d..59509a71 100644 --- a/health.go +++ b/health.go @@ -14,7 +14,6 @@ import ( "mime/multipart" "net/http" "os" - "sort" "strconv" "strings" "time" @@ -796,7 +795,7 @@ func RunOpsHealthCheck(resp http.ResponseWriter, request *http.Request) { agentHealthChannel <- agentHealth errorChannel <- err }() - + platformHealth.Agents = <-agentHealthChannel } @@ -2567,7 +2566,7 @@ func GetStaticWorkflowHealth(ctx context.Context, workflow Workflow) (Workflow, for _, field := range action.Parameters { if (field.Name == "app_name" || field.Name == "appname") && (field.Value == "" || field.Value == "noapp") { - if actionName == "Shuffle Agent" { + if actionName == "Shuffle Agent" { continue } @@ -3712,738 +3711,6 @@ func HandleRerunExecutions(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully RERAN %d executions"}`, total))) } -func FixOpensearchIndexPrefix(ctx context.Context) (OpensearchPrefixFixResult, error) { - result := OpensearchPrefixFixResult{} - if project.Environment == "cloud" { - result.Reason = "Opensearch prefix repair not supported in cloud" - return result, errors.New(result.Reason) - } - - if project.DbType != "opensearch" { - result.Reason = "Opensearch is not configured" - return result, errors.New(result.Reason) - } - - opensearchUrl := strings.TrimRight(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "/") - if len(opensearchUrl) == 0 { - opensearchUrl = "https://shuffle-opensearch:9200" - } - - foundClient := project.Es - allIndices, err := getOpensearchIndices(foundClient, opensearchUrl) - if err != nil { - return result, err - } - - aliasInfo, err := getOpensearchAliases(foundClient, opensearchUrl) - if err != nil { - return result, err - } - - prefix := strings.ToLower(strings.TrimSpace(os.Getenv("SHUFFLE_OPENSEARCH_INDEX_PREFIX"))) - baseIndexes := GetOpensearchBaseIndexes() - expectedAliases := []string{} - for _, baseIndex := range baseIndexes { - expectedAliases = append(expectedAliases, strings.ToLower(GetESIndexPrefix(baseIndex))) - } - - rolloverConfig := []byte(fmt.Sprintf(`{ - "conditions": { - "max_age": "90d", - "max_size": "40gb", - "max_docs": 1000000 - } - }`)) - - customRollover := os.Getenv("OPENSEARCH_INDEX_ROLLOVER") - if len(customRollover) > 0 { - checkValidJson := map[string]interface{}{} - if err := json.Unmarshal([]byte(customRollover), &checkValidJson); err != nil { - log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_ROLLOVER: %s", err) - } else { - rolloverConfig = []byte(customRollover) - } - } - - ismEnabled := strings.ToLower(strings.TrimSpace(os.Getenv("OPENSEARCH_USE_ISM_ROLLOVER"))) != "false" - ismPolicyName := strings.TrimSpace(os.Getenv("OPENSEARCH_ISM_POLICY_NAME")) - if ismPolicyName == "" { - ismPolicyName = "shuffle-rollover" - } - - for _, baseIndex := range baseIndexes { - expectedAlias := strings.ToLower(GetESIndexPrefix(baseIndex)) - doubleAlias := "" - if prefix != "" { - doubleAlias = fmt.Sprintf("%s_%s", prefix, expectedAlias) - } - - if ArrayContains(allIndices, expectedAlias) { - targetIndex := fmt.Sprintf("%s-000001", expectedAlias) - taskID, err := handleAliasCollisionMigration(foundClient, opensearchUrl, expectedAlias, targetIndex) - if err != nil { - result.Skipped = append(result.Skipped, fmt.Sprintf("%s (collision repair failed: %s)", expectedAlias, err)) - continue - } - - if taskID != "" { - result.MigrationTasks = append(result.MigrationTasks, fmt.Sprintf("%s -> %s (task=%s)", expectedAlias, targetIndex, taskID)) - result.Skipped = append(result.Skipped, fmt.Sprintf("%s (collision migration in progress)", expectedAlias)) - continue - } - - allIndices, err = getOpensearchIndices(foundClient, opensearchUrl) - if err != nil { - return result, err - } - - aliasInfo, err = getOpensearchAliases(foundClient, opensearchUrl) - if err != nil { - return result, err - } - } - - targetIndices, writeIndex := selectOpensearchAliasTargets(expectedAlias, doubleAlias, aliasInfo, allIndices) - if len(targetIndices) == 0 { - newIndex := fmt.Sprintf("%s-000001", expectedAlias) - if !ArrayContains(allIndices, newIndex) { - if err := createOpensearchIndex(foundClient, opensearchUrl, newIndex); err != nil { - return result, err - } - result.Created = append(result.Created, newIndex) - allIndices = append(allIndices, newIndex) - } - - targetIndices = []string{newIndex} - writeIndex = newIndex - } - - actions := []OpensearchAliasAction{} - for _, indexName := range targetIndices { - current, hasCurrent := aliasInfo[indexName][expectedAlias] - desiredWrite := indexName == writeIndex - - if hasCurrent { - if current.IsWriteIndex != desiredWrite { - actions = append(actions, OpensearchAliasAction{ - Remove: &OpensearchAliasActionTarget{Index: indexName, Alias: expectedAlias}, - }) - actions = append(actions, OpensearchAliasAction{ - Add: &OpensearchAliasActionTarget{Index: indexName, Alias: expectedAlias, IsWriteIndex: &desiredWrite}, - }) - } - } else { - actions = append(actions, OpensearchAliasAction{ - Add: &OpensearchAliasActionTarget{Index: indexName, Alias: expectedAlias, IsWriteIndex: &desiredWrite}, - }) - } - - if doubleAlias != "" { - doubleAliasState, hasDoubleAlias := aliasInfo[indexName][doubleAlias] - if hasDoubleAlias && doubleAliasState.Present { - actions = append(actions, OpensearchAliasAction{ - Remove: &OpensearchAliasActionTarget{Index: indexName, Alias: doubleAlias}, - }) - } - } - } - - if len(actions) > 0 { - if err := updateOpensearchAliases(foundClient, opensearchUrl, actions); err != nil { - return result, err - } - result.AliasUpdates = append(result.AliasUpdates, fmt.Sprintf("%s -> %s", expectedAlias, writeIndex)) - } - - result.WriteIndexUpdates = append(result.WriteIndexUpdates, fmt.Sprintf("%s -> %s", expectedAlias, writeIndex)) - } - - verifiedAliasInfo, err := getOpensearchAliases(foundClient, opensearchUrl) - if err != nil { - return result, err - } - - result.ExpectedAliases = len(expectedAliases) - result.FoundAliases = 0 - for _, aliasName := range expectedAliases { - indices := []string{} - writeIndices := []string{} - for indexName, aliases := range verifiedAliasInfo { - state, ok := aliases[aliasName] - if !ok || !state.Present { - continue - } - - indices = append(indices, indexName) - if state.IsWriteIndex { - writeIndices = append(writeIndices, indexName) - } - } - - if len(indices) == 0 { - result.MissingAliases = append(result.MissingAliases, aliasName) - continue - } - - result.FoundAliases++ - if len(writeIndices) != 1 { - result.InvalidWriteAlias = append(result.InvalidWriteAlias, fmt.Sprintf("%s (write_indices=%d)", aliasName, len(writeIndices))) - continue - } - - sorted := append([]string{}, indices...) - sort.Slice(sorted, func(i, j int) bool { - gi := getOpensearchGeneration(sorted[i]) - gj := getOpensearchGeneration(sorted[j]) - if gi == gj { - return sorted[i] > sorted[j] - } - return gi > gj - }) - - latest := sorted[0] - if writeIndices[0] != latest { - result.InvalidWriteAlias = append(result.InvalidWriteAlias, fmt.Sprintf("%s (write=%s latest=%s)", aliasName, writeIndices[0], latest)) - } - } - - if len(result.MissingAliases) > 0 || len(result.InvalidWriteAlias) > 0 || result.FoundAliases != result.ExpectedAliases { - result.Success = false - result.Reason = "Opensearch alias verification failed after repair" - log.Printf("[WARNING] %s. expected_aliases=%d found_aliases=%d missing=%d invalid_write=%d", result.Reason, result.ExpectedAliases, result.FoundAliases, len(result.MissingAliases), len(result.InvalidWriteAlias)) - } else { - result.Success = true - result.Reason = "Opensearch alias and index state repaired without data reindexing" - } - - if ismEnabled { - ismReady, ismErr := ensureOpensearchISMRolloverPolicy(ctx, opensearchUrl, expectedAliases, rolloverConfig, ismPolicyName) - if ismErr != nil { - log.Printf("[WARNING] Failed ensuring ISM rollover policy '%s' in prefix fix: %s", ismPolicyName, ismErr) - } else if ismReady { - for _, aliasName := range expectedAliases { - for indexName, aliases := range verifiedAliasInfo { - state, ok := aliases[aliasName] - if !ok || !state.Present { - continue - } - - if err := ensureOpensearchIndexRolloverAliasSetting(ctx, opensearchUrl, indexName, aliasName); err != nil { - log.Printf("[WARNING] Failed ensuring rollover alias on %s for alias %s: %s", indexName, aliasName, err) - continue - } - - if err := ensureOpensearchIndexISMPolicy(ctx, opensearchUrl, indexName, ismPolicyName); err != nil { - log.Printf("[WARNING] Failed attaching ISM policy '%s' to %s: %s", ismPolicyName, indexName, err) - } - } - } - } - } - - return result, nil -} - -func handleAliasCollisionMigration(foundClient opensearchapi.Client, opensearchUrl, sourceIndex, targetIndex string) (string, error) { - targetExists, err := checkOpensearchIndexExists(foundClient, opensearchUrl, targetIndex) - if err != nil { - return "", err - } - - if !targetExists { - if err := createOpensearchIndex(foundClient, opensearchUrl, targetIndex); err != nil { - return "", err - } - } - - sourceCount, err := getOpensearchIndexCount(foundClient, opensearchUrl, sourceIndex) - if err != nil { - return "", err - } - - targetCount, err := getOpensearchIndexCount(foundClient, opensearchUrl, targetIndex) - if err != nil { - return "", err - } - - if sourceCount > 0 && targetCount < sourceCount { - taskID, err := startOpensearchReindexTask(foundClient, opensearchUrl, sourceIndex, targetIndex) - if err != nil { - return "", err - } - - return taskID, nil - } - - if sourceCount > targetCount { - return "", fmt.Errorf("target count %d is lower than source count %d", targetCount, sourceCount) - } - - if err := deleteOpensearchIndex(foundClient, opensearchUrl, sourceIndex); err != nil { - return "", err - } - - isWrite := true - actions := []OpensearchAliasAction{ - { - Add: &OpensearchAliasActionTarget{Index: targetIndex, Alias: sourceIndex, IsWriteIndex: &isWrite}, - }, - } - - if err := updateOpensearchAliases(foundClient, opensearchUrl, actions); err != nil { - return "", err - } - - return "", nil -} - -func checkOpensearchIndexExists(foundClient opensearchapi.Client, opensearchUrl, indexName string) (bool, error) { - req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", opensearchUrl, indexName), nil) - if err != nil { - return false, err - } - - resp, err := foundClient.Client.Transport.Perform(req) - if err != nil { - return false, err - } - - body, readErr := io.ReadAll(resp.Body) - resp.Body.Close() - if readErr != nil { - return false, readErr - } - - if resp.StatusCode == 404 { - return false, nil - } - - if resp.StatusCode >= 300 { - return false, fmt.Errorf("failed checking index %s: %s", indexName, string(body)) - } - - return true, nil -} - -func getOpensearchIndexCount(foundClient opensearchapi.Client, opensearchUrl, indexName string) (int64, error) { - req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s/_count", opensearchUrl, indexName), nil) - if err != nil { - return 0, err - } - - resp, err := foundClient.Client.Transport.Perform(req) - if err != nil { - return 0, err - } - - body, readErr := io.ReadAll(resp.Body) - resp.Body.Close() - if readErr != nil { - return 0, readErr - } - - if resp.StatusCode >= 300 { - return 0, fmt.Errorf("failed counting index %s: %s", indexName, string(body)) - } - - parsed := struct { - Count int64 `json:"count"` - }{} - - if err := json.Unmarshal(body, &parsed); err != nil { - return 0, err - } - - return parsed.Count, nil -} - -func startOpensearchReindexTask(foundClient opensearchapi.Client, opensearchUrl, sourceIndex, targetIndex string) (string, error) { - payload := map[string]interface{}{ - "source": map[string]interface{}{ - "index": sourceIndex, - }, - "dest": map[string]interface{}{ - "index": targetIndex, - }, - "conflicts": "proceed", - } - - body, err := json.Marshal(payload) - if err != nil { - return "", err - } - - req, err := http.NewRequest("POST", fmt.Sprintf("%s/_reindex?wait_for_completion=false", opensearchUrl), bytes.NewBuffer(body)) - if err != nil { - return "", err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := foundClient.Client.Transport.Perform(req) - if err != nil { - return "", err - } - - respBody, readErr := io.ReadAll(resp.Body) - resp.Body.Close() - if readErr != nil { - return "", readErr - } - - if resp.StatusCode >= 300 { - lowerBody := strings.ToLower(string(respBody)) - if strings.Contains(lowerBody, "resource_already_exists_exception") { - return "", nil - } - - return "", fmt.Errorf("failed starting reindex %s -> %s: %s", sourceIndex, targetIndex, string(respBody)) - } - - parsed := struct { - Task string `json:"task"` - }{} - - if err := json.Unmarshal(respBody, &parsed); err != nil { - return "", err - } - - if strings.TrimSpace(parsed.Task) == "" { - return "", fmt.Errorf("reindex task missing in response") - } - - return parsed.Task, nil -} - -func deleteOpensearchIndex(foundClient opensearchapi.Client, opensearchUrl, indexName string) error { - req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/%s", opensearchUrl, indexName), nil) - if err != nil { - return err - } - - resp, err := foundClient.Client.Transport.Perform(req) - if err != nil { - return err - } - - body, readErr := io.ReadAll(resp.Body) - resp.Body.Close() - if readErr != nil { - return readErr - } - - if resp.StatusCode == 404 { - return nil - } - - if resp.StatusCode >= 300 { - return fmt.Errorf("failed deleting index %s: %s", indexName, string(body)) - } - - return nil -} - -type opensearchAliasState struct { - Present bool - IsWriteIndex bool -} - -func getOpensearchAliases(foundClient opensearchapi.Client, opensearchUrl string) (map[string]map[string]opensearchAliasState, error) { - aliasReq, err := http.NewRequest("GET", fmt.Sprintf("%s/_aliases", opensearchUrl), nil) - if err != nil { - return nil, err - } - - aliasResp, err := foundClient.Client.Transport.Perform(aliasReq) - if err != nil { - return nil, err - } - - aliasBody, err := io.ReadAll(aliasResp.Body) - if err != nil { - aliasResp.Body.Close() - return nil, err - } - aliasResp.Body.Close() - - if aliasResp.StatusCode >= 300 { - return nil, fmt.Errorf("failed reading opensearch aliases: %s", string(aliasBody)) - } - - rawAliasInfo := OpensearchAliasResponse{} - if err := json.Unmarshal(aliasBody, &rawAliasInfo); err != nil { - return nil, err - } - - aliasInfo := map[string]map[string]opensearchAliasState{} - type aliasDetails struct { - IsWriteIndex bool `json:"is_write_index,omitempty"` - } - - for indexName, aliasEntry := range rawAliasInfo { - aliasInfo[indexName] = map[string]opensearchAliasState{} - for aliasName, aliasRaw := range aliasEntry.Aliases { - details := aliasDetails{} - _ = json.Unmarshal(aliasRaw, &details) - aliasInfo[indexName][aliasName] = opensearchAliasState{Present: true, IsWriteIndex: details.IsWriteIndex} - } - } - - return aliasInfo, nil -} - -func getOpensearchIndices(foundClient opensearchapi.Client, opensearchUrl string) ([]string, error) { - indicesReq, err := http.NewRequest("GET", fmt.Sprintf("%s/_cat/indices?format=json&h=index", opensearchUrl), nil) - if err != nil { - return nil, err - } - - indicesResp, err := foundClient.Client.Transport.Perform(indicesReq) - if err != nil { - return nil, err - } - - indicesBody, err := io.ReadAll(indicesResp.Body) - if err != nil { - indicesResp.Body.Close() - return nil, err - } - indicesResp.Body.Close() - - if indicesResp.StatusCode >= 300 { - return nil, fmt.Errorf("failed reading opensearch indices: %s", string(indicesBody)) - } - - type indexItem struct { - Index string `json:"index"` - } - - parsedIndices := []indexItem{} - if err := json.Unmarshal(indicesBody, &parsedIndices); err != nil { - return nil, err - } - - indices := []string{} - for _, item := range parsedIndices { - if strings.TrimSpace(item.Index) != "" { - indices = append(indices, item.Index) - } - } - - return indices, nil -} - -func selectOpensearchAliasTargets(expectedAlias, doubleAlias string, aliasInfo map[string]map[string]opensearchAliasState, allIndices []string) ([]string, string) { - candidateMap := map[string]bool{} - - for indexName, aliases := range aliasInfo { - if aliases[expectedAlias].Present { - candidateMap[indexName] = true - } - if doubleAlias != "" && aliases[doubleAlias].Present { - candidateMap[indexName] = true - } - } - - for _, indexName := range allIndices { - if indexName == expectedAlias || strings.HasPrefix(indexName, expectedAlias+"-") { - candidateMap[indexName] = true - continue - } - - if doubleAlias != "" && (indexName == doubleAlias || strings.HasPrefix(indexName, doubleAlias+"-")) { - candidateMap[indexName] = true - } - } - - targetIndices := []string{} - for indexName := range candidateMap { - targetIndices = append(targetIndices, indexName) - } - - if len(targetIndices) == 0 { - return targetIndices, "" - } - - sort.Slice(targetIndices, func(i, j int) bool { - gi := getOpensearchGeneration(targetIndices[i]) - gj := getOpensearchGeneration(targetIndices[j]) - if gi == gj { - return targetIndices[i] > targetIndices[j] - } - return gi > gj - }) - - writeIndex := "" - for _, indexName := range targetIndices { - if indexName == expectedAlias || strings.HasPrefix(indexName, expectedAlias+"-") { - writeIndex = indexName - break - } - } - - if writeIndex == "" { - writeIndex = targetIndices[0] - } - - return targetIndices, writeIndex -} - -func getOpensearchGeneration(indexName string) int { - parts := strings.Split(indexName, "-") - if len(parts) < 2 { - return 0 - } - - generation := parts[len(parts)-1] - value, err := strconv.Atoi(generation) - if err != nil { - return 0 - } - - return value -} - -func createOpensearchIndex(foundClient opensearchapi.Client, opensearchUrl, indexName string) error { - indexConfig := OpensearchIndexConfig{} - customConfig := strings.TrimSpace(os.Getenv("OPENSEARCH_INDEX_CONFIG")) - if customConfig != "" { - if err := json.Unmarshal([]byte(customConfig), &indexConfig); err != nil { - return fmt.Errorf("invalid OPENSEARCH_INDEX_CONFIG: %w", err) - } - - if len(indexConfig.Aliases) > 0 { - indexConfig.Aliases = nil - } - } - - if len(indexConfig.Settings) == 0 && len(indexConfig.Mappings) == 0 { - indexConfig = OpensearchIndexConfig{ - Settings: map[string]interface{}{ - "number_of_shards": 3, - "number_of_replicas": 1, - "refresh_interval": "30s", - }, - Mappings: map[string]interface{}{ - "dynamic_templates": []map[string]interface{}{ - { - "strings_as_keywords": map[string]interface{}{ - "match_mapping_type": "string", - "mapping": map[string]interface{}{ - "type": "keyword", - }, - }, - }, - }, - }, - } - } - - indexConfigJson, err := json.Marshal(indexConfig) - if err != nil { - return err - } - - createReq, err := http.NewRequest("PUT", fmt.Sprintf("%s/%s", opensearchUrl, indexName), bytes.NewBuffer(indexConfigJson)) - if err != nil { - return err - } - createReq.Header.Set("Content-Type", "application/json") - - createResp, err := foundClient.Client.Transport.Perform(createReq) - if err != nil { - return err - } - - createRespBody, err := io.ReadAll(createResp.Body) - if err != nil { - createResp.Body.Close() - return err - } - createResp.Body.Close() - - if createResp.StatusCode >= 300 { - return fmt.Errorf("failed creating index %s: %s", indexName, string(createRespBody)) - } - - return nil -} - -func updateOpensearchAliases(foundClient opensearchapi.Client, opensearchUrl string, actions []OpensearchAliasAction) error { - aliasActions := OpensearchAliasActionsRequest{Actions: actions} - aliasBody, err := json.Marshal(aliasActions) - if err != nil { - return err - } - - aliasReq, err := http.NewRequest("POST", fmt.Sprintf("%s/_aliases", opensearchUrl), bytes.NewBuffer(aliasBody)) - if err != nil { - return err - } - aliasReq.Header.Set("Content-Type", "application/json") - - aliasResp, err := foundClient.Client.Transport.Perform(aliasReq) - if err != nil { - return err - } - - aliasRespBody, err := io.ReadAll(aliasResp.Body) - if err != nil { - aliasResp.Body.Close() - return err - } - aliasResp.Body.Close() - - if aliasResp.StatusCode >= 300 { - return fmt.Errorf("failed updating aliases: %s", string(aliasRespBody)) - } - - return nil -} - -func HandleFixOpensearchPrefix(resp http.ResponseWriter, request *http.Request) { - cors := HandleCors(resp, request) - if cors { - return - } - - user, err := HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("[WARNING] Api authentication failed in opensearch prefix fix: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Api authentication failed"}`)) - return - } - - if user.Role != "admin" { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Only admins or support can run this"}`)) - return - } - - ctx := GetContext(request) - result, err := FixOpensearchIndexPrefix(ctx) - if err != nil { - log.Printf("[ERROR] Failed fixing opensearch index prefix: %s", err) - result.Success = false - result.Reason = err.Error() - responseData, _ := json.Marshal(result) - resp.WriteHeader(500) - resp.Write(responseData) - return - } - - responseData, err := json.Marshal(result) - if err != nil { - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed JSON parsing"}`)) - return - } - - resp.Header().Set("Content-Type", "application/json") - resp.WriteHeader(200) - resp.Write(responseData) -} - func RunOpensearchOps(ctx context.Context) (*opensearchapi.ClusterHealthResp, error) { if project.Environment == "cloud" { return nil, errors.New("Not running opensearch health check") @@ -4687,7 +3954,7 @@ func startAgentExecution(baseUrl, apiKey, orgId string) (agentStartResult, error "input": map[string]string{ "text": "Get the current weather of new york using https://wttr.in/New+York?format=%t api and just output the current weather temperature without any commentary, just output the number in celcius and dont include the decimals, use action as custom_action, tool as http and category as singul keep the url as it and not needed for any other hallucinated params or headers, just include the url as is and the method name which is GET.", }, - "tool_name" : "http", + "tool_name": "http", }, } @@ -4785,7 +4052,6 @@ func fetchAgentExecutionResults(baseUrl, apiKey, orgId, executionId, authorizati return execution, nil } - func extractAgentOutputFromResults(execution WorkflowExecution) (AgentOutput, bool) { var agentOutput AgentOutput @@ -4831,7 +4097,7 @@ func extractAgentOutputFromResults(execution WorkflowExecution) (AgentOutput, bo return agentOutput, true } } - + return AgentOutput{}, false } @@ -4841,7 +4107,7 @@ func RunOpsAgent(apiKey string, orgId string, cloudRunUrl string) (AgentHealth, agentHealth := AgentHealth{ Create: true, // Not creating workflow, but keeping for compatibility BackendVersion: os.Getenv("SHUFFLE_BACKEND_VERSION"), - Delete: true, + Delete: true, } baseUrl := resolveAgentBaseUrl(cloudRunUrl) diff --git a/notification_retention.go b/notification_retention.go new file mode 100644 index 00000000..c5f475a0 --- /dev/null +++ b/notification_retention.go @@ -0,0 +1,231 @@ +package shuffle + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/shuffle/opensearch-go/v4/opensearchapi" +) + +const defaultNotificationRetentionDaysValue = 0 + +// getNotificationRetentionDays returns how many days a read/ignored +// notification is kept before the cleanup sweep deletes it. +func getNotificationRetentionDays() int { + raw := strings.TrimSpace(os.Getenv("OPENSEARCH_NOTIFICATION_RETENTION_DAYS")) + if raw == "" { + return defaultNotificationRetentionDaysValue + } + + parsed, err := strconv.Atoi(raw) + if err != nil || parsed <= 0 { + if raw != "0" { + log.Printf("[WARNING] Invalid OPENSEARCH_NOTIFICATION_RETENTION_DAYS %q, notification retention stays disabled", raw) + } + return defaultNotificationRetentionDaysValue + } + + return parsed +} + +// buildNotificationRetentionQuery builds the query for finding notifications +// eligible for deletion: read OR ignored, and last updated before the +// retention cutoff. +func buildNotificationRetentionQuery(now time.Time, retentionDays int) map[string]interface{} { + cutoff := now.AddDate(0, 0, -retentionDays).Unix() + + return map[string]interface{}{ + "size": 1000, + "query": map[string]interface{}{ + "bool": map[string]interface{}{ + "must": []map[string]interface{}{ + { + "bool": map[string]interface{}{ + "should": []map[string]interface{}{ + {"term": map[string]interface{}{"read": true}}, + {"term": map[string]interface{}{"ignored": true}}, + }, + }, + }, + { + "range": map[string]interface{}{ + "updated_at": map[string]interface{}{ + "lt": cutoff, + }, + }, + }, + }, + }, + }, + } +} + +// sweepOldNotifications deletes notifications that are read or ignored and +// older than getNotificationRetentionDays(). +func sweepOldNotifications(ctx context.Context) error { + retentionDays := getNotificationRetentionDays() + if retentionDays <= 0 { + return nil + } + + query := buildNotificationRetentionQuery(time.Now(), retentionDays) + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(query); err != nil { + return err + } + + resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{ + Indices: []string{strings.ToLower(GetESIndexPrefix("notifications"))}, + Body: &buf, + }) + if err != nil { + if strings.Contains(err.Error(), "index_not_found_exception") { + return nil + } + return err + } + + res := resp.Inspect().Response + defer res.Body.Close() + if res.StatusCode == 404 { + return nil + } + + respBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + + type notificationHit struct { + Source Notification `json:"_source"` + } + type notificationSearchWrapper struct { + Hits struct { + Hits []notificationHit `json:"hits"` + } `json:"hits"` + } + + wrapped := notificationSearchWrapper{} + if err := json.Unmarshal(respBody, &wrapped); err != nil { + return err + } + + deleted := 0 + for _, hit := range wrapped.Hits.Hits { + if err := DeleteKey(ctx, "notifications", hit.Source.Id); err != nil { + log.Printf("[WARNING] Failed deleting old notification %s: %s", hit.Source.Id, err) + continue + } + deleted++ + } + + if deleted > 0 { + log.Printf("[INFO] Deleted %d notifications older than %d days retention", deleted, getNotificationRetentionDays()) + } + + return nil +} + +// notificationRetentionMappingWarning checks whether the notifications index +// mapping actually supports the retention sweep's query (read/ignored as +// boolean, updated_at as date). On indexes created before opensearchCoreMappings +// existed, these fields may be dynamically mapped as keyword/long instead, +// which makes buildNotificationRetentionQuery's term/range clauses silently +// match nothing - the sweep looks "active" (no errors, ticks every 24h) but +// never actually deletes anything. This surfaces that as an explicit +// [WARNING] log line instead of a silent no-op, so retention reads as +// "inactive" rather than "active" when it can't work. +func notificationRetentionMappingWarning(ctx context.Context) { + alias := strings.ToLower(GetESIndexPrefix("notifications")) + + resp, err := project.Es.Indices.Mapping.Get(ctx, &opensearchapi.MappingGetReq{ + Indices: []string{alias}, + }) + if err != nil { + return + } + + res := resp.Inspect().Response + if res == nil { + return + } + defer res.Body.Close() + if res.StatusCode != 200 { + return + } + + for indexName, indexMapping := range resp.Indices { + var parsed struct { + Properties map[string]struct { + Type string `json:"type"` + } `json:"properties"` + } + if err := json.Unmarshal(indexMapping.Mappings, &parsed); err != nil { + continue + } + + fieldTypes := make(map[string]string, len(parsed.Properties)) + for field, prop := range parsed.Properties { + fieldTypes[field] = prop.Type + } + + for _, warning := range notificationRetentionFieldMappingWarnings(indexName, fieldTypes) { + log.Printf("[WARNING] %s", warning) + } + } +} + +// notificationRetentionFieldMappingWarnings is the pure comparison logic +// behind notificationRetentionMappingWarning, factored out so it's +// unit-testable without a live OpenSearch cluster. fieldTypes maps a field +// name to its mapped OpenSearch type (empty string / absent means the field +// has no mapping yet, e.g. an empty freshly-created index - not a mismatch). +func notificationRetentionFieldMappingWarnings(indexName string, fieldTypes map[string]string) []string { + var warnings []string + + if fieldType := fieldTypes["read"]; fieldType != "" && fieldType != "boolean" { + warnings = append(warnings, fmt.Sprintf("Notification retention sweep on %s: 'read' field is mapped as %q, not 'boolean' - the retention query will silently match nothing on this index. Retention is effectively inactive here until a reindex fixes the mapping.", indexName, fieldType)) + } + if fieldType := fieldTypes["ignored"]; fieldType != "" && fieldType != "boolean" { + warnings = append(warnings, fmt.Sprintf("Notification retention sweep on %s: 'ignored' field is mapped as %q, not 'boolean' - the retention query will silently match nothing on this index. Retention is effectively inactive here until a reindex fixes the mapping.", indexName, fieldType)) + } + if fieldType := fieldTypes["updated_at"]; fieldType != "" && fieldType != "date" { + warnings = append(warnings, fmt.Sprintf("Notification retention sweep on %s: 'updated_at' field is mapped as %q, not 'date' - the retention query's date range will silently match nothing on this index. Retention is effectively inactive here until a reindex fixes the mapping.", indexName, fieldType)) + } + + return warnings +} + +// StartNotificationRetentionSweeper runs sweepOldNotifications once per day. +func StartNotificationRetentionSweeper(ctx context.Context) { + if project.DbType != "opensearch" { + return + } + + if getNotificationRetentionDays() > 0 { + notificationRetentionMappingWarning(ctx) + } + + ticker := time.NewTicker(24 * time.Hour) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := sweepOldNotifications(ctx); err != nil { + log.Printf("[WARNING] Notification retention sweep failed: %s", err) + } + } + } +} diff --git a/notification_retention_test.go b/notification_retention_test.go new file mode 100644 index 00000000..1ff2be29 --- /dev/null +++ b/notification_retention_test.go @@ -0,0 +1,133 @@ +package shuffle + +import ( + "os" + "strings" + "testing" + "time" +) + +func TestGetNotificationRetentionDaysDefaultDisabled(t *testing.T) { + os.Unsetenv("OPENSEARCH_NOTIFICATION_RETENTION_DAYS") + if got := getNotificationRetentionDays(); got != 0 { + t.Fatalf("expected default 0 (disabled), got %d", got) + } +} + +func TestGetNotificationRetentionDaysOverride(t *testing.T) { + os.Setenv("OPENSEARCH_NOTIFICATION_RETENTION_DAYS", "30") + defer os.Unsetenv("OPENSEARCH_NOTIFICATION_RETENTION_DAYS") + + if got := getNotificationRetentionDays(); got != 30 { + t.Fatalf("expected 30, got %d", got) + } +} + +func TestGetNotificationRetentionDaysInvalidFallsBackToDisabled(t *testing.T) { + os.Setenv("OPENSEARCH_NOTIFICATION_RETENTION_DAYS", "not-a-number") + defer os.Unsetenv("OPENSEARCH_NOTIFICATION_RETENTION_DAYS") + + if got := getNotificationRetentionDays(); got != 0 { + t.Fatalf("expected fallback to disabled (0), got %d", got) + } +} + +func TestSweepOldNotificationsNoOpsWhenDisabled(t *testing.T) { + os.Unsetenv("OPENSEARCH_NOTIFICATION_RETENTION_DAYS") + + if err := sweepOldNotifications(nil); err != nil { + t.Fatalf("expected nil error when retention disabled (no-op), got %s", err) + } +} + +func TestBuildNotificationRetentionQuery(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + query := buildNotificationRetentionQuery(now, 90) + + boolQuery, ok := query["query"].(map[string]interface{})["bool"].(map[string]interface{}) + if !ok { + t.Fatalf("expected bool query, got %v", query) + } + + must, ok := boolQuery["must"].([]map[string]interface{}) + if !ok || len(must) != 2 { + t.Fatalf("expected 2 must clauses (read-or-ignored + updated_at cutoff), got %v", must) + } + + shouldClause, ok := must[0]["bool"].(map[string]interface{}) + if !ok { + t.Fatalf("expected first clause to be a bool/should for read OR ignored, got %v", must[0]) + } + should, ok := shouldClause["should"].([]map[string]interface{}) + if !ok || len(should) != 2 { + t.Fatalf("expected 2 should clauses (read=true, ignored=true), got %v", should) + } + + rangeQuery, ok := must[1]["range"].(map[string]interface{}) + if !ok { + t.Fatalf("expected second clause to be a range on updated_at, got %v", must[1]) + } + updatedAtRange, ok := rangeQuery["updated_at"].(map[string]interface{}) + if !ok { + t.Fatalf("expected updated_at range, got %v", rangeQuery) + } + + wantCutoff := now.AddDate(0, 0, -90).Unix() + if updatedAtRange["lt"] != wantCutoff { + t.Fatalf("expected cutoff %d, got %v", wantCutoff, updatedAtRange["lt"]) + } +} + +func TestNotificationRetentionFieldMappingWarningsAllCompatible(t *testing.T) { + fieldTypes := map[string]string{ + "read": "boolean", + "ignored": "boolean", + "updated_at": "date", + } + + warnings := notificationRetentionFieldMappingWarnings("notifications", fieldTypes) + if len(warnings) != 0 { + t.Fatalf("expected no warnings for compatible mapping, got %v", warnings) + } +} + +func TestNotificationRetentionFieldMappingWarningsLegacyUpdatedAtAsLong(t *testing.T) { + // Mirrors what a legacy pre-opensearchCoreMappings index looks like: JSON + // booleans dynamically infer as "boolean" fine, but a raw epoch-second + // int64 dynamically infers as "long", not "date". + fieldTypes := map[string]string{ + "read": "boolean", + "ignored": "boolean", + "updated_at": "long", + } + + warnings := notificationRetentionFieldMappingWarnings("notifications", fieldTypes) + if len(warnings) != 1 { + t.Fatalf("expected exactly 1 warning for updated_at mismatch, got %v", warnings) + } + if !strings.Contains(warnings[0], "updated_at") || !strings.Contains(warnings[0], "long") { + t.Fatalf("expected warning to mention updated_at/long, got %q", warnings[0]) + } +} + +func TestNotificationRetentionFieldMappingWarningsReadIgnoredAsKeyword(t *testing.T) { + fieldTypes := map[string]string{ + "read": "keyword", + "ignored": "keyword", + "updated_at": "date", + } + + warnings := notificationRetentionFieldMappingWarnings("notifications", fieldTypes) + if len(warnings) != 2 { + t.Fatalf("expected 2 warnings (read + ignored), got %v", warnings) + } +} + +func TestNotificationRetentionFieldMappingWarningsMissingFieldsNotAWarning(t *testing.T) { + // An empty freshly-created index has no properties yet - absence isn't a + // mismatch, it just means no docs have been written. + warnings := notificationRetentionFieldMappingWarnings("notifications", map[string]string{}) + if len(warnings) != 0 { + t.Fatalf("expected no warnings when fields are simply absent, got %v", warnings) + } +} diff --git a/opensearch_indices.go b/opensearch_indices.go new file mode 100644 index 00000000..bb921a22 --- /dev/null +++ b/opensearch_indices.go @@ -0,0 +1,382 @@ +// This file is the single declarative source of truth for OpenSearch index +// definitions: which base indices Shuffle explicitly manages, which of +// those are safe to put behind alias+rollover, and their curated field mappings. +// Nothing in this file talks to OpenSearch over the network - it only describes shape. +// +// For everything that acts on this data (creating indices, migrating mappings, +// rolling over, fixing alias collisions, etc.), see opensearch_lifecycle.go. +package shuffle + +import ( + "encoding/json" + "fmt" + "os" + "strings" +) + +// opensearchIgnoreAboveLength is the single source of truth for the +// "ignore_above" setting applied to dynamically mapped string fields (via the +// strings_as_keywords dynamic template) across all explicitly managed +// indices. It mirrors OpenSearch's own default dynamic keyword mapping +// (text+keyword sub-field with ignore_above:256). +const opensearchIgnoreAboveLength = 256 + +// Default index settings applied to every explicitly managed index (unless +// overridden - see getOpensearchDefaultIndexSettings), and default rollover +// thresholds applied to every rollover-eligible index (unless overridden - +// see getOpensearchDefaultRolloverConditions). +const ( + opensearchDefaultShards = 3 + opensearchDefaultReplicas = 1 + opensearchDefaultRefreshInterval = "30s" + + opensearchDefaultRolloverMaxAge = "90d" + opensearchDefaultRolloverMaxSize = "40gb" + opensearchDefaultRolloverMaxDocs = 1000000 +) + +// getOpensearchDefaultIndexSettings returns the default +// number_of_shards/number_of_replicas/refresh_interval settings block for a +// freshly created index. Override the whole set via OPENSEARCH_INDEX_CONFIG. +func getOpensearchDefaultIndexSettings() map[string]interface{} { + return map[string]interface{}{ + "number_of_shards": opensearchDefaultShards, + "number_of_replicas": opensearchDefaultReplicas, + "refresh_interval": opensearchDefaultRefreshInterval, + } +} + +// getOpensearchDefaultRolloverConditions returns Shuffle's default rollover +// thresholds in the Index Rollover API's "max_*" key format (used by +// InitOpensearchIndices/FixOpensearchIndexPrefix to build a direct +// POST /_rollover body). Override the whole set via +// OPENSEARCH_INDEX_ROLLOVER. +func getOpensearchDefaultRolloverConditions() map[string]interface{} { + return map[string]interface{}{ + "max_age": opensearchDefaultRolloverMaxAge, + "max_size": opensearchDefaultRolloverMaxSize, + "max_docs": opensearchDefaultRolloverMaxDocs, + } +} + +// opensearchStringsAsKeywordsDynamicTemplate returns the dynamic template +// that maps every dynamically-added string field to a plain keyword with +// opensearchIgnoreAboveLength, matching OpenSearch's own default dynamic +// string mapping. Shared by every index-create/mapping body so the +// ignore_above value has one source of truth. +func opensearchStringsAsKeywordsDynamicTemplate() map[string]interface{} { + return map[string]interface{}{ + "strings_as_keywords": map[string]interface{}{ + "match_mapping_type": "string", + "mapping": map[string]interface{}{"type": "keyword", "ignore_above": opensearchIgnoreAboveLength}, + }, + } +} + +// opensearchDynamicMappingSettings returns the "mappings"-level settings +// that must accompany opensearchStringsAsKeywordsDynamicTemplate on every +// index-create body, so dynamically-added fields are consistently typed by +// that template alone. +// +// "date_detection" defaults to true in OpenSearch and runs BEFORE custom +// dynamic_templates are considered: a dynamically-added string field whose +// first-seen value happens to parse as a date (e.g. a user-authored +// workflow action parameter's example value like "2024-01-01") gets +// classified as "date" by this built-in check, which pre-empts +// match_mapping_type:"string" ever matching - completely bypassing the +// strings_as_keywords template for that field. Any later document whose +// value for that same field path is plain text then fails with "mapper +// [...] cannot be changed from type [date] to [keyword]", permanently +// (mappings are immutable once set). Disabling date_detection ensures only +// strings_as_keywords governs dynamic string typing, regardless of what a +// field's value happens to look like. +// +// "numeric_detection" is the same content-sniffing hazard for numbers +// instead of dates (a string like "42" would get promoted to long/double +// instead of keyword). It already defaults to false, so this isn't fixing +// an active bug - it's set explicitly so correctness here doesn't depend on +// an OpenSearch/cluster-level default that could change out from under us. +func opensearchDynamicMappingSettings() map[string]interface{} { + return map[string]interface{}{ + "date_detection": false, + "numeric_detection": false, + "dynamic_templates": []map[string]interface{}{ + opensearchStringsAsKeywordsDynamicTemplate(), + }, + } +} + +// Create ElasticSearch/OpenSearch index prefix +// It is used where a single cluster of ElasticSearch/OpenSearch utilized by several +// Shuffle instance +// E.g. Instance1_Workflowapp +func GetESIndexPrefix(index string) string { + prefix := os.Getenv("SHUFFLE_OPENSEARCH_INDEX_PREFIX") + if len(prefix) > 0 { + return fmt.Sprintf("%s_%s", prefix, index) + } + + return index +} + +// GetOpensearchBaseIndices returns a list of indices managed by Shuffle. +// Shuffle also uses other indices, that are implicitly created on first write. +// Indices in this list are explicitly created by Shuffle on startup. +func GetOpensearchBaseIndices() []string { + return []string{ + "workflowexecution", + "workflowexecution_live", + "datastore_ngram", + "org_cache", + "org_cache_revisions", + "notifications", + "shuffle_logs", + "environments", + "org_statistics", + "workflowapp", + "workflow", + "workflow_revisions", + "datastore_category", + } +} + +// GetOpensearchRolloverIndices returns the subset of base indices that are +// genuinely append-only (or, for workflowexecution, append-only AFTER the +// hot/cold lifecycle split below) and therefore want alias + automatic +// rollover + ISM retention. +func GetOpensearchRolloverIndices() []string { + return []string{ + "shuffle_logs", + // Content-addressed / fresh-id-per-write revision stores: a given _id is + // never rewritten once created, so alias+rollover is safe here. + "workflow_revisions", // _id = md5(name+id+actions+triggers+variables) + "org_cache_revisions", // _id = _[_]_ + // workflowexecution is the ARCHIVE for confirmed-terminal + // executions (see execution_lifecycle.go). Nothing writes to it + // except archiveExecutionDocument, which resolves any existing copy of an + // execution_id to its concrete backing index before writing (rather than + // writing through the alias blindly), so rollover is safe here even though + // the same execution_id can in rare cases be revisited (see the unarchive + // path in writeExecutionDocument). + "workflowexecution", + } +} + +// opensearchCoreMappings holds the curated field mappings for base indices: +// applied to fresh index-create bodies, and used as the target that +// migrateOpensearchSingleIndex (opensearch_lifecycle.go) compares live +// indices against on every startup - a mismatch is migrated onto these +// mappings automatically via reindex + alias cutover, since OpenSearch +// mappings themselves are immutable in place. Keys are the base index names +// from GetOpensearchBaseIndices. +// +// id/ref fields are keyword, date/epoch fields are date (epoch_second), and +// numeric counters/priorities are long so numeric sorting and filtering work. +// The default strings_as_keywords dynamic template only handles strings; these +// explicit types keep the mapping stable regardless of how a value is written. +// +// Large/binary blobs (e.g. images) are mapped with index:false + doc_values:false: +// they stay retrievable via _source but are never added to the inverted index +// or doc_values, avoiding index bloat and too-big-field failures. Add a field +// only when its type is known to be stable and it is actually sorted/filtered +// on (or when it must be excluded from indexing). +// +// created/edited/started_at on workflowexecution are deliberately "long", not +// "date": these three fields are sorted across workflowexecution_live+archive +// (started_at, in GetUnfinishedExecutions/GetAllWorkflowExecutions[V2]/ +// GetWorkflowRunsBySearch) or across archive generations (created/edited, in +// findExecutionInArchive). OpenSearch stores "date" doc values internally as +// epoch milliseconds regardless of the "format" annotation - format only +// affects range-query parsing, not the raw value a sort clause returns. Any +// pre-existing customer index created before this mapping existed has these +// fields dynamically inferred as "long" (the app always wrote raw epoch +// SECONDS as JSON numbers). Mapping them "date" here would make freshly +// created generations sort in milliseconds while old/legacy generations sort +// in seconds - since ms values are ~1000x larger, every doc in a +// date-mapped generation would silently outrank every doc in a long-mapped +// generation regardless of true chronological order. +// Keeping these three fields "long" matches what already-deployed clusters +// have and keeps sort units identical across every generation, old and new. +// completed_at has no cross-generation/cross-index sort today (only a +// same-index numeric range filter in the archival sweep), so it is left as +// "date" for existing single-index range-query compatibility. +var opensearchCoreMappings = map[string]map[string]interface{}{ + "workflowexecution": { + "properties": map[string]interface{}{ + "execution_id": map[string]interface{}{"type": "keyword"}, + "workflow_id": map[string]interface{}{"type": "keyword"}, + "execution_org": map[string]interface{}{"type": "keyword"}, + "status": map[string]interface{}{"type": "keyword"}, + "created": map[string]interface{}{"type": "long"}, + "edited": map[string]interface{}{"type": "long"}, + "started_at": map[string]interface{}{"type": "long"}, + "completed_at": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "priority": map[string]interface{}{"type": "long"}, + }, + }, + "notifications": { + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "keyword"}, + "org_id": map[string]interface{}{"type": "keyword"}, + "user_id": map[string]interface{}{"type": "keyword"}, + "execution_id": map[string]interface{}{"type": "keyword"}, + "workflow_id": map[string]interface{}{"type": "keyword"}, + "org_notification_id": map[string]interface{}{"type": "keyword"}, + "created_at": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "updated_at": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "amount": map[string]interface{}{"type": "long"}, + "image": map[string]interface{}{"type": "keyword", "index": false, "doc_values": false}, + "read": map[string]interface{}{"type": "boolean"}, + "ignored": map[string]interface{}{"type": "boolean"}, + "dismissable": map[string]interface{}{"type": "boolean"}, + "personal": map[string]interface{}{"type": "boolean"}, + }, + }, + "org_statistics": { + "properties": map[string]interface{}{ + "org_id": map[string]interface{}{"type": "keyword"}, + "last_cleared": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "total_app_executions": map[string]interface{}{"type": "long"}, + "total_workflow_executions": map[string]interface{}{"type": "long"}, + "total_agent_executions": map[string]interface{}{"type": "long"}, + "total_agent_tokens": map[string]interface{}{"type": "long"}, + "total_ai_executions": map[string]interface{}{"type": "long"}, + "total_app_executions_failed": map[string]interface{}{"type": "long"}, + }, + }, + "datastore_ngram": { + "properties": map[string]interface{}{ + "key": map[string]interface{}{"type": "keyword"}, + "org_id": map[string]interface{}{"type": "keyword"}, + "amount": map[string]interface{}{"type": "long"}, + }, + }, + "workflow": { + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "keyword"}, + "org_id": map[string]interface{}{"type": "keyword"}, + "created": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "edited": map[string]interface{}{"type": "date", "format": "epoch_second"}, + "last_runtime": map[string]interface{}{"type": "long"}, + }, + }, + "workflowapp": { + "properties": map[string]interface{}{ + "app_id": map[string]interface{}{"type": "keyword"}, + "app_version": map[string]interface{}{"type": "keyword"}, + "generated": map[string]interface{}{"type": "boolean"}, + "small_image": map[string]interface{}{"type": "keyword", "index": false, "doc_values": false}, + "large_image": map[string]interface{}{"type": "keyword", "index": false, "doc_values": false}, + }, + }, + "org_cache": { + "properties": map[string]interface{}{ + "key": map[string]interface{}{"type": "keyword"}, + "org_id": map[string]interface{}{"type": "keyword"}, + }, + }, +} + +// ensureOpensearchIndexRolloverAlias sets the +// "plugins.index_state_management.rollover_alias" setting on an index-create +// body to alias, so ISM knows which alias to roll over once this generation +// meets its rollover conditions. +// No-op (returns indexConfig unchanged) if the body can't be parsed as JSON. +func ensureOpensearchIndexRolloverAlias(indexConfig []byte, alias string) []byte { + unmarshalled := map[string]interface{}{} + if err := json.Unmarshal(indexConfig, &unmarshalled); err != nil { + return indexConfig + } + + settings, ok := unmarshalled["settings"].(map[string]interface{}) + if !ok || settings == nil { + settings = map[string]interface{}{} + } + + settings["plugins.index_state_management.rollover_alias"] = alias + unmarshalled["settings"] = settings + + updated, err := json.Marshal(unmarshalled) + if err != nil { + return indexConfig + } + + return updated +} + +// applyOpensearchCoreMappings injects the pragmatic core field mappings for a +// base index into a fresh index-create body. +func applyOpensearchCoreMappings(indexConfig []byte, index string) []byte { + key := strings.TrimPrefix(strings.ToLower(index), strings.ToLower(GetESIndexPrefix(""))) + properties, ok := opensearchCoreMappings[key] + if !ok { + return indexConfig + } + + unmarshalled := map[string]interface{}{} + if err := json.Unmarshal(indexConfig, &unmarshalled); err != nil { + return indexConfig + } + + mappings, _ := unmarshalled["mappings"].(map[string]interface{}) + if mappings == nil { + mappings = map[string]interface{}{} + } + + if _, exists := mappings["properties"]; !exists { + mappings["properties"] = properties["properties"] + } + unmarshalled["mappings"] = mappings + + updated, err := json.Marshal(unmarshalled) + if err != nil { + return indexConfig + } + + return updated +} + +// opensearchMappingsFor builds the mappings section (dynamic string->keyword +// template plus the curated core field mappings) for a base index. +func opensearchMappingsFor(baseIndex string) map[string]interface{} { + mappings := opensearchDynamicMappingSettings() + + if props, ok := opensearchCoreMappings[baseIndex]["properties"]; ok { + mappings["properties"] = props + } + + return mappings +} + +// opensearchMappingsDiffer reports whether the live index mapping (its +// "properties" subtree) is missing any curated field, or has a field whose +// type/format/indexability no longer matches the desired core mappings. Only +// fields we explicitly map are compared. +func opensearchMappingsDiffer(baseIndex string, actualProps map[string]interface{}) bool { + desired, ok := opensearchCoreMappings[baseIndex]["properties"].(map[string]interface{}) + if !ok { + return false + } + + for name, dv := range desired { + desiredProp, _ := dv.(map[string]interface{}) + actualProp, exists := actualProps[name].(map[string]interface{}) + if !exists { + return true + } + + if fmt.Sprint(desiredProp["type"]) != fmt.Sprint(actualProp["type"]) { + return true + } + + if df, ok := desiredProp["format"]; ok && fmt.Sprint(actualProp["format"]) != fmt.Sprint(df) { + return true + } + + if fmt.Sprint(desiredProp["index"]) == "false" && fmt.Sprint(actualProp["index"]) != "false" { + return true + } + } + + return false +} diff --git a/opensearch_lifecycle.go b/opensearch_lifecycle.go new file mode 100644 index 00000000..3b015c3a --- /dev/null +++ b/opensearch_lifecycle.go @@ -0,0 +1,2273 @@ +// This file contains all OpenSearch index lifecycle management: creation, +// mapping-drift migration, rollover, ISM retention policies, and the +// low-level index/alias/task helpers those flows are built from. +package shuffle + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/shuffle/opensearch-go/v4/opensearchapi" +) + +// resolveAliasWriteIndex looks up whether the given alias already has a write +// index attached in OpenSearch. +func resolveAliasWriteIndex(aliasInfo map[string]map[string]opensearchAliasState, alias string) (writeIndex string, found bool) { + for indexName, aliases := range aliasInfo { + if state, ok := aliases[alias]; ok && state.Present && state.IsWriteIndex { + return indexName, true + } + } + + return "", false +} + +// resolveAppendIndexCreationTarget decides which concrete backing index the +// create-loop in InitOpensearchIndices should target for a given append +// (rollover) base index: either a brand new "-000001" generation (if none +// exists yet) or the highest existing generation (if the index has already +// been created/rolled/collapsed before). +// +// existingIndices is the full list of real index names currently in the +// cluster (from getOpensearchIndices). +func resolveAppendIndexCreationTarget(existingIndices []string, index string) (target string, alreadyExists bool) { + prefix := index + "-" + highestGen := -1 + highestName := "" + + for _, name := range existingIndices { + if !strings.HasPrefix(name, prefix) { + continue + } + + gen := getOpensearchGeneration(name) + if gen <= 0 { + continue + } + if gen > highestGen { + highestGen = gen + highestName = name + } + } + + if highestName == "" { + return fmt.Sprintf("%s-000001", index), false + } + + return highestName, true +} + +// InitOpensearchIndices is the entry point for OpenSearch startup +// bootstrapping: creates every base index (from GetOpensearchBaseIndices) +// that doesn't exist yet, attaches rollover aliases/ISM policies for the +// rollover-eligible subset, registers mapping templates for future rollover +// generations, and migrates any single/keyed index whose live mapping has +// drifted from opensearchCoreMappings. +// +// Safe to call on every backend restart and from multiple replicas +// concurrently - every step is idempotent or existence-checked first. No-op +// if DbType isn't "opensearch" or if SHUFFLE_SKIP_OPENSEARCH_INDEX_INIT is +// set. +func InitOpensearchIndices() { + if project.DbType != "opensearch" { + return + } + + if os.Getenv("SHUFFLE_SKIP_OPENSEARCH_INDEX_INIT") == "true" { + return + } + + // Check if the "workflowexecution" index exists and configuring rollovers if possible + log.Printf("[INFO] Configuring Opensearch indices for scaling") + + ctx := context.Background() + opensearchUrl := strings.TrimRight(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "/") + if len(opensearchUrl) == 0 { + opensearchUrl = "https://shuffle-opensearch:9200" + } + + relevantScaleIndices := []string{} + for _, baseIndex := range GetOpensearchBaseIndices() { + relevantScaleIndices = append(relevantScaleIndices, GetESIndexPrefix(baseIndex)) + } + + // Only append-heavy stores get rollover. Stateful keyed stores stay on a + // single backing index (rollover there splits _id across generations + // and breaks single-document reads, e.g. org_statistics). + appendIndices := []string{} + for _, baseIndex := range GetOpensearchRolloverIndices() { + appendIndices = append(appendIndices, strings.ToLower(GetESIndexPrefix(baseIndex))) + } + + singleIndices := []string{} + for _, index := range relevantScaleIndices { + index = strings.ToLower(index) + if !ArrayContains(appendIndices, index) { + singleIndices = append(singleIndices, index) + } + } + + customConfig := os.Getenv("OPENSEARCH_INDEX_CONFIG") + if len(customConfig) > 0 { + checkValidJson := map[string]interface{}{} + if err := json.Unmarshal([]byte(customConfig), &checkValidJson); err != nil { + log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_CONFIG: %s", err) + customConfig = "" + } else { + log.Printf("[DEBUG] Using custom index config for relevant scale indices: %s", customConfig) + } + } + + customRollover := os.Getenv("OPENSEARCH_INDEX_ROLLOVER") + if len(customRollover) > 0 { + checkValidJson := map[string]interface{}{} + if err := json.Unmarshal([]byte(customRollover), &checkValidJson); err != nil { + log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_ROLLOVER: %s", err) + customRollover = "" + } else { + log.Printf("[DEBUG] Using custom rollover config for relevant scale indices: %s", customRollover) + } + } + + rolloverConfig, err := json.Marshal(map[string]interface{}{ + "conditions": getOpensearchDefaultRolloverConditions(), + }) + if err != nil { + log.Printf("[ERROR] Failed building default rollover config: %s", err) + return + } + + if len(customRollover) > 0 { + rolloverConfig = []byte(customRollover) + } + + ismEnabled := strings.ToLower(strings.TrimSpace(os.Getenv("OPENSEARCH_USE_ISM_ROLLOVER"))) != "false" + ismPolicyName := strings.TrimSpace(os.Getenv("OPENSEARCH_ISM_POLICY_NAME")) + if ismPolicyName == "" { + ismPolicyName = "shuffle-rollover" + } + + // Ensure all ISM rollover policies exist and are up to date. + ismReady := false + if ismEnabled { + for _, baseIndex := range GetOpensearchRolloverIndices() { + alias := strings.ToLower(GetESIndexPrefix(baseIndex)) + retention := getOpensearchRetentionDays(baseIndex) + ready, err := ensureOpensearchISMRolloverPolicy(ctx, opensearchUrl, alias, rolloverConfig, retention, ismPolicyName) + if err != nil { + log.Printf("[WARNING] Failed ensuring ISM rollover policy '%s': %s", ismPolicyName, err) + continue + } + if ready { + ismReady = true + } + } + } + + // Fix existing indices + if fixResult, fixErr := FixOpensearchIndexPrefix(ctx); fixErr != nil { + log.Printf("[WARNING] Prefix repair before init failed: %s", fixErr) + } else if !fixResult.Success { + log.Printf("[WARNING] Prefix repair before init completed with verification warnings: %s", fixResult.Reason) + } else { + log.Printf("[INFO] Prefix repair before init: expected aliases=%d found=%d", fixResult.ExpectedAliases, fixResult.FoundAliases) + } + + // Ensure an IndexTemplate exists for all rollover indices. + if len(customConfig) == 0 { + ensureOpensearchMappingTemplates(ctx, opensearchUrl) + } + + existingOpensearchIndices, existingIndicesErr := getOpensearchIndices(project.Es, opensearchUrl) + if existingIndicesErr != nil { + log.Printf("[WARNING] Failed listing existing OpenSearch indices before create-loop (falling back to blind -000001 creation for all indices): %s", existingIndicesErr) + existingOpensearchIndices = []string{} + } + + existingOpensearchAliases, existingAliasesErr := getOpensearchAliases(project.Es, opensearchUrl) + if existingAliasesErr != nil { + log.Printf("[WARNING] Failed listing existing OpenSearch aliases before create-loop (falling back to name-based existence checks only): %s", existingAliasesErr) + existingOpensearchAliases = map[string]map[string]opensearchAliasState{} + } + + for _, index := range relevantScaleIndices { + indexConfig, err := json.Marshal(map[string]interface{}{ + "aliases": map[string]interface{}{ + index: map[string]bool{"is_write_index": true}, + }, + "settings": getOpensearchDefaultIndexSettings(), + "mappings": opensearchDynamicMappingSettings(), + }) + if err != nil { + log.Printf("[ERROR] Failed building default index config for %s: %s", index, err) + continue + } + + if len(customConfig) > 0 { + indexConfig = []byte(customConfig) + + // Check if alias is in the index or not, otherwise inject it + unmarshalled := map[string]interface{}{} + if err := json.Unmarshal(indexConfig, &unmarshalled); err != nil { + log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_CONFIG (2): %s", err) + } else { + if _, ok := unmarshalled["aliases"]; !ok { + // Inject it + aliasPart := map[string]interface{}{ + index: map[string]bool{ + "is_write_index": true, + }, + } + unmarshalled["aliases"] = aliasPart + newConfig, err := json.Marshal(unmarshalled) + if err != nil { + log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_CONFIG (3): %s", err) + } else { + indexConfig = newConfig + log.Printf("[INFO] Injected alias into OPENSEARCH_INDEX_CONFIG for index %s", index) + } + } + } + } + + index = strings.ToLower(index) + isAppend := ArrayContains(appendIndices, index) + if len(customConfig) == 0 { + indexConfig = applyOpensearchCoreMappings(indexConfig, index) + } + initialIndexName, alreadyExists := resolveAppendIndexCreationTarget(existingOpensearchIndices, index) + if !alreadyExists { + // Name-prefix matching found nothing, but the alias may still + // already be served by a legacy, oddly-named backing index + // (e.g. from an old double-prefix bug). + // + // Check the alias's actual write-index assignment before + // attempting to create a new index - creating one now would + // give the alias two write indices and OpenSearch would reject + // it outright. + if writeIndex, aliasHasWriteIndex := resolveAliasWriteIndex(existingOpensearchAliases, index); aliasHasWriteIndex { + initialIndexName = writeIndex + alreadyExists = true + } + } + if isAppend { + indexConfig = ensureOpensearchIndexRolloverAlias(indexConfig, index) + } + // Directly try to force create it. Opensearch throws a 400 if it fails. + + var resp *opensearchapi.IndicesCreateResp + var createErr error + if alreadyExists { + log.Printf("[INFO] Index %s already exists at generation %s - skipping creation, ensuring ISM/rollover on existing index", index, initialIndexName) + } else { + resp, createErr = project.Es.Indices.Create(ctx, opensearchapi.IndicesCreateReq{ + Index: initialIndexName, + Body: bytes.NewReader(indexConfig), + }) + + res := resp.Inspect().Response + defer res.Body.Close() + if createErr != nil { + if !strings.Contains(fmt.Sprintf("%s", createErr), "serverless mode") && !strings.Contains(fmt.Sprintf("%s", createErr), "resource_already_exists_exception") { + log.Printf("[WARNING] Error creating index %s: %s", index, createErr) + } + + // Make sure if the resource exist it is part of correct alias + if strings.Contains(fmt.Sprintf("%s", createErr), "resource_already_exists_exception") { + body := fmt.Sprintf(`{ + "actions": [ + { + "add": { + "index": "%s", + "alias": "%s", + "is_write_index": true + } + } + ] + }`, initialIndexName, index) + + aliasResp, aerr := project.Es.Aliases(ctx, opensearchapi.AliasesReq{ + Body: strings.NewReader(body), + }) + if aerr != nil { + log.Printf("[WARNING] Failed to ensure alias %s for index %s: %s", index, initialIndexName, aerr) + return + } + + res := aliasResp.Inspect().Response + defer res.Body.Close() + + if res.StatusCode >= 300 { + log.Printf("[WARNING] Alias enforcement failed: %s", res.String()) + return + } + } + } else { + if res.IsError() { + if !strings.Contains(res.String(), "resource_already_exists_exception") { + log.Printf("[DEBUG] Error creating index %s with custom config: %s", index, res.String()) + } + + } else { + log.Printf("[DEBUG] Successfully created index %s with custom config", index) + } + } + } + + // Non-append indices stay on a single backing index - no rollover/ISM. + if !isAppend { + continue + } + + if ismReady { + if err := ensureOpensearchIndexRolloverAliasSetting(ctx, opensearchUrl, initialIndexName, index); err != nil { + log.Printf("[WARNING] Failed ensuring rollover_alias on index %s: %s", initialIndexName, err) + } + + policyID := fmt.Sprintf("%s-%s", ismPolicyName, index) + if err := ensureOpensearchIndexISMPolicy(ctx, opensearchUrl, initialIndexName, policyID); err != nil { + log.Printf("[WARNING] Failed attaching ISM policy '%s' to %s: %s", policyID, initialIndexName, err) + } + + continue + } + + rolloverResp, err := project.Es.Indices.Rollover(ctx, opensearchapi.IndicesRolloverReq{ + Alias: index, + Body: bytes.NewReader(rolloverConfig), + }) + + if err != nil { + if !strings.Contains(fmt.Sprintf("%s", err), "serverless mode") && !strings.Contains(fmt.Sprintf("%s", err), "status: 404") { + log.Printf("[WARNING] Problem during rollover config for %s: %s", index, err) + } + + continue + } + + rolloverRes := rolloverResp.Inspect().Response + defer rolloverRes.Body.Close() + if rolloverRes.IsError() { + log.Printf("[ERROR] Rollover config failed for %s: %s", index, rolloverRes.String()) + } else { + log.Printf("[INFO] Rollover executed successfully for %s", index) + } + + } + + // Migrate existing deployments that rolled stateful indices in the past: + // collapse all generations of each single index into its newest backing + // index and detach ISM so it never rolls again. Idempotent. + for _, singleIndex := range singleIndices { + if err := collapseSingleIndexAliases(ctx, opensearchUrl, singleIndex); err != nil { + log.Printf("[WARNING] Failed collapsing single index %s: %s", singleIndex, err) + } + } + + // Apply mapping migrations to existing single/keyed indices when the live + // mapping has drifted from opensearchCoreMappings. Skipped when a custom + // OPENSEARCH_INDEX_CONFIG is set (the operator owns those mappings). + if len(customConfig) == 0 { + for _, singleIndex := range singleIndices { + if err := migrateOpensearchSingleIndex(ctx, opensearchUrl, singleIndex); err != nil { + log.Printf("[WARNING] Failed migrating mapping for single index %s: %s", singleIndex, err) + } + } + } + + if fixResult, fixErr := FixOpensearchIndexPrefix(ctx); fixErr != nil { + log.Printf("[WARNING] Alias verification after init failed: %s", fixErr) + } else if !fixResult.Success { + log.Printf("[WARNING] Alias verification after init completed with warnings: %s", fixResult.Reason) + } else { + log.Printf("[INFO] Alias verification after init passed: expected aliases=%d found=%d", fixResult.ExpectedAliases, fixResult.FoundAliases) + } + +} + +// getOpensearchIndexProperties returns the "properties" subtree of an index's live mappings. +func getOpensearchIndexProperties(foundClient opensearchapi.Client, opensearchUrl, indexName string) (map[string]interface{}, error) { + resp, err := foundClient.Indices.Mapping.Get(context.Background(), &opensearchapi.MappingGetReq{Indices: []string{indexName}}) + if err != nil { + return nil, fmt.Errorf("failed reading mapping for %s: %w", indexName, err) + } + + for _, idx := range resp.Indices { + mappings := map[string]interface{}{} + if len(idx.Mappings) > 0 { + if err := json.Unmarshal(idx.Mappings, &mappings); err != nil { + return nil, err + } + } + props, _ := mappings["properties"].(map[string]interface{}) + return props, nil + } + + return nil, nil +} + +// createOpensearchIndexFromBody creates an index with an explicit create body. +func createOpensearchIndexFromBody(ctx context.Context, opensearchUrl, indexName string, body []byte) error { + if _, err := project.Es.Indices.Create(ctx, opensearchapi.IndicesCreateReq{ + Index: indexName, + Body: bytes.NewReader(body), + }); err != nil { + return fmt.Errorf("failed creating index %s: %w", indexName, err) + } + + return nil +} + +// migrateOpensearchSingleIndex re-creates a single (keyed) index with the +// current core mappings when its live mapping has drifted. +// +// It bulk-copies the existing backing index into a fresh generation (via +// reindexOpensearchIndex's failure-aware async task polling - not a bare +// synchronous call, so a mapping rejection or task-level error, e.g. a batch +// overflowing OpenSearch's 2GB transport limit, aborts the migration instead +// of silently deleting a partially-copied source), then write-blocks the +// source for a final catch-up copy and verifies an exact document count +// match before atomically swapping the alias to the new generation and +// dropping the old one. +// +// Any failure at any step aborts without deleting the source or touching +// the alias, leaving the next automatic retry (this runs idempotently on +// every startup) to pick up from current state. +func migrateOpensearchSingleIndex(ctx context.Context, opensearchUrl, baseIndex string) error { + foundClient := project.Es + allIndices, err := getOpensearchIndices(foundClient, opensearchUrl) + if err != nil { + return err + } + + generations := []string{} + for _, idx := range allIndices { + if idx == baseIndex || strings.HasPrefix(idx, baseIndex+"-") { + generations = append(generations, idx) + } + } + if len(generations) == 0 { + return nil + } + + sort.Slice(generations, func(i, j int) bool { + return getOpensearchGeneration(generations[i]) > getOpensearchGeneration(generations[j]) + }) + + // collapseSingleIndexAliases runs just before this; if multiple generations + // remain, defer to it rather than racing a partial collapse. + if len(generations) > 1 { + return nil + } + + src := generations[0] + actualProps, err := getOpensearchIndexProperties(foundClient, opensearchUrl, src) + if err != nil { + return err + } + if !opensearchMappingsDiffer(baseIndex, actualProps) { + return nil + } + + nextGen := getOpensearchGeneration(src) + 1 + dest := fmt.Sprintf("%s-%06d", baseIndex, nextGen) + + body := map[string]interface{}{ + "settings": getOpensearchDefaultIndexSettings(), + "mappings": opensearchMappingsFor(baseIndex), + } + bodyJSON, err := json.Marshal(body) + if err != nil { + return err + } + + if err := createOpensearchIndexFromBody(ctx, opensearchUrl, dest, bodyJSON); err != nil { + return err + } + + log.Printf("[INFO] Opensearch single-index mapping migration: starting bulk copy %s -> %s", src, dest) + if err := reindexOpensearchIndex(ctx, opensearchUrl, src, dest); err != nil { + return fmt.Errorf("bulk copy: %w", err) + } + + // Freeze the source so a final catch-up pass can close any gap opened + // by writes that landed concurrently during the bulk copy above, before + // we trust the document counts to match exactly and delete the source. + // + // This mirrors the same write-block/catch-up/verify pattern used for + // legacy alias-collision migrations (runOpensearchCollisionMigration) - + // without it, a write landing in src during the copy could be silently + // lost the moment src is deleted below. + if err := setOpensearchIndexWriteBlock(foundClient, opensearchUrl, src, true); err != nil { + return fmt.Errorf("write-blocking source before final catch-up: %w", err) + } + + if err := reindexOpensearchIndex(ctx, opensearchUrl, src, dest); err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, src) + return fmt.Errorf("final write-blocked catch-up copy: %w", err) + } + + // _count (like _search) only sees refreshed segments, not documents + // written moments ago - force a refresh on both indices before trusting + // the comparison below, otherwise the tail of the catch-up copy above + // can make destCount look behind even though the copy fully succeeded. + if err := refreshOpensearchIndex(foundClient, opensearchUrl, src); err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, src) + return fmt.Errorf("refreshing source before final count check: %w", err) + } + if err := refreshOpensearchIndex(foundClient, opensearchUrl, dest); err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, src) + return fmt.Errorf("refreshing target before final count check: %w", err) + } + + srcCount, err := getOpensearchIndexCount(foundClient, opensearchUrl, src) + if err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, src) + return fmt.Errorf("getting final source count: %w", err) + } + destCount, err := getOpensearchIndexCount(foundClient, opensearchUrl, dest) + if err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, src) + return fmt.Errorf("getting final target count: %w", err) + } + if destCount < srcCount { + // Unblock and let the next automatic retry (this function is + // idempotent and reruns on every startup) redo the copy - something + // left the target still behind, and deleting the source with data + // still missing would be permanent data loss. + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, src) + return fmt.Errorf("target count %d still behind source count %d after write-blocked catch-up - not deleting source", destCount, srcCount) + } + + // atomically move the write alias from the old generation to the new one + write := true + actions := []OpensearchAliasAction{ + {Remove: &OpensearchAliasActionTarget{Index: src, Alias: baseIndex}}, + {Add: &OpensearchAliasActionTarget{Index: dest, Alias: baseIndex, IsWriteIndex: &write}}, + } + if err := updateOpensearchAliases(foundClient, opensearchUrl, actions); err != nil { + // This request is atomic (OpenSearch applies remove+add as a single + // cluster-state update), so a failure here leaves src still holding + // the baseIndex alias, exactly as before the attempt - unblock it so + // the application can keep writing to it normally. Without this, src + // (the currently-serving index) would stay write-blocked until the + // next backend restart re-runs this idempotent migration, since + // nothing else retries it in between. + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, src) + return err + } + + if err := deleteOpensearchIndex(foundClient, opensearchUrl, src); err != nil { + return err + } + + log.Printf("[INFO] Migrated mapping for %s: %s -> %s (%d documents verified)", baseIndex, src, dest, destCount) + return nil +} + +// ensureOpensearchMappingTemplates registers an index mapping template per +// append/rollover base index so every future rollover generation is created +// with the current core mappings (existing generations are left untouched). +func ensureOpensearchMappingTemplates(ctx context.Context, opensearchUrl string) { + for _, baseIndex := range GetOpensearchRolloverIndices() { + alias := strings.ToLower(GetESIndexPrefix(baseIndex)) + + body := map[string]interface{}{ + "index_patterns": []string{alias + "-*"}, + "template": map[string]interface{}{ + "mappings": opensearchMappingsFor(baseIndex), + }, + "priority": 100, + } + bodyJSON, err := json.Marshal(body) + if err != nil { + log.Printf("[WARNING] Failed building mapping template for %s: %s", alias, err) + continue + } + + templateName := fmt.Sprintf("shuffle-%s-mapping", baseIndex) + if _, err := project.Es.IndexTemplate.Create(ctx, opensearchapi.IndexTemplateCreateReq{ + IndexTemplate: templateName, + Body: bytes.NewReader(bodyJSON), + }); err != nil { + log.Printf("[WARNING] Failed to register mapping template for %s: %s", alias, err) + continue + } + } +} + +// collapseSingleIndexAliases migrates a stateful (non-append) index that may +// have rolled over in previous versions to a single backing index: it +// merges every older generation into the newest (newest document wins per +// _id), drops the older generations, and detaches rollover so the index +// stays single. Safe to run repeatedly. +func collapseSingleIndexAliases(ctx context.Context, opensearchUrl, fullIndex string) error { + foundClient := project.Es + allIndices, err := getOpensearchIndices(foundClient, opensearchUrl) + if err != nil { + return err + } + + generations := []string{} + for _, idx := range allIndices { + if idx == fullIndex || strings.HasPrefix(idx, fullIndex+"-") { + generations = append(generations, idx) + } + } + + if len(generations) == 1 { + // A single surviving index - just make sure it can't roll over. + return detachOpensearchRollover(ctx, opensearchUrl, generations[0]) + } + if len(generations) == 0 { + return nil + } + + sort.Slice(generations, func(i, j int) bool { + return getOpensearchGeneration(generations[i]) > getOpensearchGeneration(generations[j]) + }) + writeGen := generations[0] + olderGens := generations[1:] + + // Merge older generations into the newest (the surviving write target). + // reindexOpensearchIndex uses op_type:create, so a source _id that + // already exists in the newest generation is skipped (conflicts=proceed) + // and the newest copy wins; _ids that only live in older generations are + // copied across. + // + // Iteration order does not matter because the destination always wins + // on collision. + for _, older := range olderGens { + if err := reindexOpensearchIndex(ctx, opensearchUrl, older, writeGen); err != nil { + return err + } + } + + actions := []OpensearchAliasAction{} + for _, older := range olderGens { + actions = append(actions, OpensearchAliasAction{ + Remove: &OpensearchAliasActionTarget{Index: older, Alias: fullIndex}, + }) + } + if err := updateOpensearchAliases(foundClient, opensearchUrl, actions); err != nil { + return err + } + + for _, older := range olderGens { + if err := deleteOpensearchIndex(foundClient, opensearchUrl, older); err != nil { + return err + } + } + + return detachOpensearchRollover(ctx, opensearchUrl, writeGen) +} + +// reindexOpensearchIndex copies documents from source into dest via +// runOpensearchReindexToCompletion +// +// ctx is accepted for API compatibility with existing callers but the +// underlying poll loop is not currently context-aware. +func reindexOpensearchIndex(ctx context.Context, opensearchUrl, source, dest string) error { + return runOpensearchReindexToCompletion(project.Es, opensearchUrl, source, dest) +} + +// detachOpensearchRollover removes the ISM rollover policy and clears the +// rollover_alias index setting so a single (non-append) index never rolls over. +func detachOpensearchRollover(ctx context.Context, opensearchUrl, indexName string) error { + // Remove the ISM rollover policy, if any. Missing policy / missing plugin + // (4xx) is fine - clearing the rollover_alias setting below is what truly + // stops rollover. + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/_plugins/_ism/remove/%s", opensearchUrl, indexName), strings.NewReader("{}")) + if err == nil { + req.Header.Set("Content-Type", "application/json") + if removeResp, performErr := project.Es.Client.Transport.Perform(req); performErr == nil { + _ = removeResp.Body.Close() + } + } + + settingsBody := map[string]interface{}{ + "index": map[string]interface{}{ + "plugins.index_state_management.rollover_alias": nil, + }, + } + + bodyData, marshalErr := json.Marshal(settingsBody) + if marshalErr != nil { + return marshalErr + } + + if _, err := project.Es.Indices.Settings.Put(ctx, opensearchapi.SettingsPutReq{ + Indices: []string{indexName}, + Body: bytes.NewReader(bodyData), + }); err != nil { + if strings.Contains(strings.ToLower(err.Error()), "index_not_found_exception") { + return nil + } + return fmt.Errorf("clear rollover_alias on %s failed: %w", indexName, err) + } + + return nil +} + +// getOpensearchISMRolloverConditions parses the "conditions" object from a +// custom OPENSEARCH_INDEX_ROLLOVER JSON payload (accepting both ISM's native +// min_* keys and the more intuitive max_* aliases), falling back to +// Shuffle's defaults (90d / 40gb / 1,000,000 docs) for any condition not +// set, or if rolloverConfig is empty/invalid. +func getOpensearchISMRolloverConditions(rolloverConfig []byte) map[string]interface{} { + defaultConditions := map[string]interface{}{ + "min_index_age": opensearchDefaultRolloverMaxAge, + "min_size": opensearchDefaultRolloverMaxSize, + "min_doc_count": opensearchDefaultRolloverMaxDocs, + } + + parsed := struct { + Conditions map[string]interface{} `json:"conditions"` + }{} + + if err := json.Unmarshal(rolloverConfig, &parsed); err != nil { + return defaultConditions + } + + if len(parsed.Conditions) == 0 { + return defaultConditions + } + + conditions := map[string]interface{}{} + if value, ok := parsed.Conditions["min_index_age"]; ok { + conditions["min_index_age"] = value + } else if value, ok := parsed.Conditions["max_age"]; ok { + conditions["min_index_age"] = value + } + + if value, ok := parsed.Conditions["min_size"]; ok { + conditions["min_size"] = value + } else if value, ok := parsed.Conditions["max_size"]; ok { + conditions["min_size"] = value + } + + if value, ok := parsed.Conditions["min_doc_count"]; ok { + conditions["min_doc_count"] = value + } else if value, ok := parsed.Conditions["max_docs"]; ok { + conditions["min_doc_count"] = value + } + + if len(conditions) == 0 { + return defaultConditions + } + + return conditions +} + +// getOpensearchRetentionDays returns how long rolled-over generations of +// baseIndex should be kept before ISM deletes them (e.g. "90d"), preferring +// a per-index override from OPENSEARCH_INDEX_RETENTION_DAYS (a JSON map) over +// Shuffle's built-in defaults. Returns "" (no retention/keep forever) for any +// base index without a default and without an override. +func getOpensearchRetentionDays(baseIndex string) string { + defaults := map[string]string{ + "shuffle_logs": "90d", + "workflowexecution": "365d", + } + + value := defaults[baseIndex] + if value == "" { + return "" + } + + custom := strings.TrimSpace(os.Getenv("OPENSEARCH_INDEX_RETENTION_DAYS")) + if custom == "" { + return value + } + + parsed := map[string]interface{}{} + if err := json.Unmarshal([]byte(custom), &parsed); err != nil { + log.Printf("[WARNING] Invalid JSON in OPENSEARCH_INDEX_RETENTION_DAYS: %s", err) + return value + } + + raw, ok := parsed[baseIndex] + if !ok { + return value + } + if days, ok := raw.(float64); ok { + return fmt.Sprintf("%dd", int(days)) + } + if str, ok := raw.(string); ok { + return str + } + + return value +} + +// existingOpensearchISMPolicy holds the parts of a GET +// /_plugins/_ism/policies/ response needed to decide whether the policy +// needs updating, and (if so) to perform a conflict-safe PUT. +type existingOpensearchISMPolicy struct { + SeqNo int64 `json:"_seq_no"` + PrimaryTerm int64 `json:"_primary_term"` + RawConditions map[string]interface{} // hot state's rollover conditions + RawRetention string // delete transition's min_index_age, if any +} + +// getExistingOpensearchISMPolicy fetches the current ISM policy document for +// policyID, if any, and extracts just the "hot" state's rollover conditions +// and delete-transition retention age (plus the _seq_no/_primary_term +// needed for a conflict-safe PUT). Returns (nil, false, nil) if the policy +// doesn't exist yet, and a distinct "ism plugin not available" error if the +// ISM plugin itself isn't installed on the cluster. +func getExistingOpensearchISMPolicy(ctx context.Context, opensearchUrl, policyID string) (*existingOpensearchISMPolicy, bool, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/_plugins/_ism/policies/%s", opensearchUrl, policyID), nil) + if err != nil { + return nil, false, err + } + + resp, err := project.Es.Client.Transport.Perform(req) + if err != nil { + return nil, false, err + } + defer resp.Body.Close() + + body, _ := ioutil.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + if resp.StatusCode == 404 || resp.StatusCode == 400 { + if strings.Contains(strings.ToLower(string(body)), "_plugins/_ism") || strings.Contains(strings.ToLower(string(body)), "no handler found") { + // ISM plugin isn't installed at all. + return nil, false, fmt.Errorf("ism plugin not available") + } + } + + if resp.StatusCode == 404 { + // Genuinely doesn't exist yet - needs to be created. + return nil, false, nil + } + + return nil, false, fmt.Errorf("status: %d, body: %s", resp.StatusCode, string(body)) + } + + parsed := struct { + SeqNo int64 `json:"_seq_no"` + PrimaryTerm int64 `json:"_primary_term"` + Policy struct { + States []struct { + Name string `json:"name"` + Actions []struct { + Rollover map[string]interface{} `json:"rollover"` + } `json:"actions"` + Transitions []struct { + Conditions struct { + MinIndexAge string `json:"min_index_age"` + } `json:"conditions"` + } `json:"transitions"` + } `json:"states"` + } `json:"policy"` + }{} + + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, false, err + } + + existing := &existingOpensearchISMPolicy{ + SeqNo: parsed.SeqNo, + PrimaryTerm: parsed.PrimaryTerm, + } + for _, state := range parsed.Policy.States { + if state.Name != "hot" { + continue + } + if len(state.Actions) > 0 { + existing.RawConditions = state.Actions[0].Rollover + } + if len(state.Transitions) > 0 { + existing.RawRetention = state.Transitions[0].Conditions.MinIndexAge + } + } + + return existing, true, nil +} + +// ensureOpensearchISMRolloverPolicy creates (or updates, if its rollover +// conditions or retention no longer match) the ISM policy that rolls over +// and eventually deletes generations of alias. Returns (true, nil) if a +// usable policy is in place, or (false, nil) - not an error - if the ISM +// plugin isn't installed, so callers can fall back to direct shard rollover. +func ensureOpensearchISMRolloverPolicy(ctx context.Context, opensearchUrl, alias string, rolloverConfig []byte, retentionAge, policyName string) (bool, error) { + conditions := getOpensearchISMRolloverConditions(rolloverConfig) + policyID := fmt.Sprintf("%s-%s", policyName, alias) + + states := []map[string]interface{}{ + { + "name": "hot", + "actions": []map[string]interface{}{{"rollover": conditions}}, + "transitions": []interface{}{}, + }, + } + + if retentionAge != "" { + states[0]["transitions"] = []map[string]interface{}{ + { + "state_name": "delete", + "conditions": map[string]interface{}{"min_index_age": retentionAge}, + }, + } + states = append(states, map[string]interface{}{ + "name": "delete", + "actions": []map[string]interface{}{{"delete": map[string]interface{}{}}}, + "transitions": []interface{}{}, + }) + } + + policyBody := map[string]interface{}{ + "policy": map[string]interface{}{ + "description": "Shuffle rollover + retention policy", + "default_state": "hot", + "states": states, + "ism_template": []map[string]interface{}{ + { + "index_patterns": []string{fmt.Sprintf("%s-*", alias)}, + "priority": 100, + }, + }, + }, + } + + policyData, err := json.Marshal(policyBody) + if err != nil { + return false, err + } + + // Check whether the policy already exists, and if so, whether its + // rollover conditions/retention already match what we'd write - this + // lets us both (a) avoid a needless PUT (and its 409) when nothing + // changed, and (b) actually apply changes to OPENSEARCH_INDEX_ROLLOVER / + // OPENSEARCH_INDEX_RETENTION_DAYS on restart when something did change, + // which a blind "create-only" PUT can never do once the policy exists. + existing, found, err := getExistingOpensearchISMPolicy(ctx, opensearchUrl, policyID) + if err != nil { + if err.Error() == "ism plugin not available" { + log.Printf("[INFO] ISM plugin not available. Falling back to direct rollover") + return false, nil + } + return false, err + } + + putUrl := fmt.Sprintf("%s/_plugins/_ism/policies/%s", opensearchUrl, policyID) + if found { + // Compare only the specific rollover condition keys Shuffle manages, + // not a full deep-equal of the stored object: OpenSearch enriches + // the stored rollover conditions with its own extra fields we never + // set (e.g. "copy_alias": false), so a full-map compare would never + // match and would cause a needless PUT (and misleading "changed - + // updating" log) on every single restart. + // + // %v formatting sidesteps int (our defaults) vs float64 (values + // decoded from OpenSearch's JSON response) type mismatches on + // otherwise-equal numbers. + conditionsMatch := true + for _, key := range []string{"min_index_age", "min_size", "min_doc_count"} { + if fmt.Sprintf("%v", existing.RawConditions[key]) != fmt.Sprintf("%v", conditions[key]) { + conditionsMatch = false + break + } + } + retentionMatches := existing.RawRetention == retentionAge + if conditionsMatch && retentionMatches { + log.Printf("[DEBUG] ISM rollover policy '%s' already up to date for alias %s - skipping", policyID, alias) + return true, nil + } + + log.Printf("[INFO] ISM rollover policy '%s' conditions/retention changed for alias %s - updating", policyID, alias) + putUrl = fmt.Sprintf("%s?if_seq_no=%d&if_primary_term=%d", putUrl, existing.SeqNo, existing.PrimaryTerm) + } + + req, err := http.NewRequestWithContext(ctx, "PUT", putUrl, bytes.NewReader(policyData)) + if err != nil { + return false, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := project.Es.Client.Transport.Perform(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + + body, _ := ioutil.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + if resp.StatusCode == 404 || resp.StatusCode == 400 { + if strings.Contains(strings.ToLower(string(body)), "_plugins/_ism") || strings.Contains(strings.ToLower(string(body)), "no handler found") { + log.Printf("[INFO] ISM plugin not available. Falling back to direct rollover") + return false, nil + } + } + + return false, fmt.Errorf("status: %d, body: %s", resp.StatusCode, string(body)) + } + + log.Printf("[INFO] Ensured ISM rollover policy '%s' for alias %s", policyID, alias) + return true, nil +} + +// ensureOpensearchIndexRolloverAliasSetting sets the +// "plugins.index_state_management.rollover_alias" setting on an +// already-existing index (unlike ensureOpensearchIndexRolloverAlias, which +// only patches a create body). +// +// Treats a missing index as success rather than an error, since the index +// may have just been rolled/collapsed away by a concurrent replica. +func ensureOpensearchIndexRolloverAliasSetting(ctx context.Context, opensearchUrl, indexName, alias string) error { + settingsBody := map[string]interface{}{ + "index": map[string]interface{}{ + "plugins.index_state_management.rollover_alias": alias, + }, + } + + body, err := json.Marshal(settingsBody) + if err != nil { + return err + } + + _, err = project.Es.Indices.Settings.Put(ctx, opensearchapi.SettingsPutReq{ + Indices: []string{indexName}, + Body: bytes.NewReader(body), + }) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "index_not_found_exception") { + return nil + } + + return err + } + + return nil +} + +// ensureOpensearchIndexISMPolicy attaches policyName as the managing ISM +// policy for indexName. Treats "already has a policy" and "index not found" +// responses as success, since both mean there's nothing left to do here. +func ensureOpensearchIndexISMPolicy(ctx context.Context, opensearchUrl, indexName, policyName string) error { + policyBody := map[string]interface{}{ + "policy_id": policyName, + } + + body, err := json.Marshal(policyBody) + if err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/_plugins/_ism/add/%s", opensearchUrl, indexName), bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := project.Es.Client.Transport.Perform(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respBody, _ := ioutil.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + lowerResp := strings.ToLower(string(respBody)) + if strings.Contains(lowerResp, "already has a policy") { + return nil + } + + if resp.StatusCode == 404 && strings.Contains(lowerResp, "index_not_found_exception") { + return nil + } + + return fmt.Errorf("status: %d, body: %s", resp.StatusCode, string(respBody)) + } + + return nil +} + +// OpensearchPrefixFixResult is the result of FixOpensearchIndexPrefix: a +// summary of what was verified, migrated, or repaired. +type OpensearchPrefixFixResult struct { + Success bool `json:"success"` + Reason string `json:"reason,omitempty"` + ExpectedAliases int `json:"expected_aliases,omitempty"` + FoundAliases int `json:"found_aliases,omitempty"` + MissingAliases []string `json:"missing_aliases,omitempty"` + InvalidWriteAlias []string `json:"invalid_write_aliases,omitempty"` + MigrationTasks []string `json:"migration_tasks,omitempty"` + Created []string `json:"created,omitempty"` + WriteIndexUpdates []string `json:"write_index_updates,omitempty"` + AliasUpdates []string `json:"alias_updates,omitempty"` + Skipped []string `json:"skipped,omitempty"` +} + +// FixOpensearchIndexPrefix verifies (and repairs) that every base index has +// exactly one correctly-named write alias attached, for every base index +// from GetOpensearchBaseIndices. +// +// It detects two distinct problems and heals both without operator +// interaction: (1) a legacy plain index colliding with its intended alias +// name (kicks off an async collision migration via +// startOpensearchCollisionMigrationAsync), and (2) a missing/invalid write +// alias that can be fixed by a plain alias update. +func FixOpensearchIndexPrefix(ctx context.Context) (OpensearchPrefixFixResult, error) { + result := OpensearchPrefixFixResult{} + if project.Environment == "cloud" { + result.Reason = "Opensearch prefix repair not supported in cloud" + return result, errors.New(result.Reason) + } + + if project.DbType != "opensearch" { + result.Reason = "Opensearch is not configured" + return result, errors.New(result.Reason) + } + + opensearchUrl := strings.TrimRight(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "/") + if len(opensearchUrl) == 0 { + opensearchUrl = "https://shuffle-opensearch:9200" + } + + foundClient := project.Es + allIndices, err := getOpensearchIndices(foundClient, opensearchUrl) + if err != nil { + return result, err + } + + aliasInfo, err := getOpensearchAliases(foundClient, opensearchUrl) + if err != nil { + return result, err + } + + prefix := strings.ToLower(strings.TrimSpace(os.Getenv("SHUFFLE_OPENSEARCH_INDEX_PREFIX"))) + baseIndices := GetOpensearchBaseIndices() + expectedAliases := []string{} + for _, baseIndex := range baseIndices { + expectedAliases = append(expectedAliases, strings.ToLower(GetESIndexPrefix(baseIndex))) + } + + rolloverConfig, err := json.Marshal(map[string]interface{}{ + "conditions": getOpensearchDefaultRolloverConditions(), + }) + if err != nil { + return result, fmt.Errorf("failed building default rollover config: %w", err) + } + + customRollover := os.Getenv("OPENSEARCH_INDEX_ROLLOVER") + if len(customRollover) > 0 { + checkValidJson := map[string]interface{}{} + if err := json.Unmarshal([]byte(customRollover), &checkValidJson); err != nil { + log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_ROLLOVER: %s", err) + } else { + rolloverConfig = []byte(customRollover) + } + } + + ismEnabled := strings.ToLower(strings.TrimSpace(os.Getenv("OPENSEARCH_USE_ISM_ROLLOVER"))) != "false" + ismPolicyName := strings.TrimSpace(os.Getenv("OPENSEARCH_ISM_POLICY_NAME")) + if ismPolicyName == "" { + ismPolicyName = "shuffle-rollover" + } + + for _, baseIndex := range baseIndices { + expectedAlias := strings.ToLower(GetESIndexPrefix(baseIndex)) + + if ArrayContains(allIndices, expectedAlias) { + targetIndex := fmt.Sprintf("%s-000001", expectedAlias) + // Migration runs fully in the background (see + // startOpensearchCollisionMigrationAsync) - a legacy monolithic + // index blocking this alias name can be hundreds of GB, and + // copying it synchronously here would block backend startup + // for hours. + // + // It resumes automatically (idempotently) on every restart + // until the alias is finally freed and swapped, with no + // operator action required. + startOpensearchCollisionMigrationAsync(foundClient, opensearchUrl, expectedAlias, targetIndex, baseIndex) + result.MigrationTasks = append(result.MigrationTasks, fmt.Sprintf("%s -> %s (background migration running)", expectedAlias, targetIndex)) + result.Skipped = append(result.Skipped, fmt.Sprintf("%s (collision migration running in background; alias unavailable until it completes)", expectedAlias)) + continue + } + + targetIndices, writeIndex := selectOpensearchAliasTargets(baseIndex, prefix, aliasInfo, allIndices) + if len(targetIndices) == 0 { + newIndex := fmt.Sprintf("%s-000001", expectedAlias) + if !ArrayContains(allIndices, newIndex) { + if err := createOpensearchIndex(foundClient, opensearchUrl, newIndex, baseIndex); err != nil { + return result, err + } + result.Created = append(result.Created, newIndex) + allIndices = append(allIndices, newIndex) + } + + targetIndices = []string{newIndex} + writeIndex = newIndex + } + + actions := []OpensearchAliasAction{} + for _, indexName := range targetIndices { + current, hasCurrent := aliasInfo[indexName][expectedAlias] + desiredWrite := indexName == writeIndex + + if hasCurrent { + if current.IsWriteIndex != desiredWrite { + actions = append(actions, OpensearchAliasAction{ + Remove: &OpensearchAliasActionTarget{Index: indexName, Alias: expectedAlias}, + }) + actions = append(actions, OpensearchAliasAction{ + Add: &OpensearchAliasActionTarget{Index: indexName, Alias: expectedAlias, IsWriteIndex: &desiredWrite}, + }) + } + } else { + actions = append(actions, OpensearchAliasAction{ + Add: &OpensearchAliasActionTarget{Index: indexName, Alias: expectedAlias, IsWriteIndex: &desiredWrite}, + }) + } + + // Drop any other alias attached to this index that belongs to + // the same baseIndex - i.e. any legacy multi-prefixed variant + // left over from the historical double-prefix bug. + for aliasName, state := range aliasInfo[indexName] { + if aliasName == expectedAlias || !state.Present { + continue + } + if opensearchIndexBelongsTo(aliasName, baseIndex, prefix) { + actions = append(actions, OpensearchAliasAction{ + Remove: &OpensearchAliasActionTarget{Index: indexName, Alias: aliasName}, + }) + } + } + } + + if len(actions) > 0 { + if err := updateOpensearchAliases(foundClient, opensearchUrl, actions); err != nil { + return result, err + } + result.AliasUpdates = append(result.AliasUpdates, fmt.Sprintf("%s -> %s", expectedAlias, writeIndex)) + } + + result.WriteIndexUpdates = append(result.WriteIndexUpdates, fmt.Sprintf("%s -> %s", expectedAlias, writeIndex)) + } + + verifiedAliasInfo, err := getOpensearchAliases(foundClient, opensearchUrl) + if err != nil { + return result, err + } + + result.ExpectedAliases = len(expectedAliases) + result.FoundAliases = 0 + for _, aliasName := range expectedAliases { + indices := []string{} + writeIndices := []string{} + for indexName, aliases := range verifiedAliasInfo { + state, ok := aliases[aliasName] + if !ok || !state.Present { + continue + } + + indices = append(indices, indexName) + if state.IsWriteIndex { + writeIndices = append(writeIndices, indexName) + } + } + + if len(indices) == 0 { + result.MissingAliases = append(result.MissingAliases, aliasName) + continue + } + + result.FoundAliases++ + if len(writeIndices) != 1 { + result.InvalidWriteAlias = append(result.InvalidWriteAlias, fmt.Sprintf("%s (write_indices=%d)", aliasName, len(writeIndices))) + continue + } + + // "Latest" prefers a canonically-named generation (aliasName or + // aliasName+"-NNNNNN") over raw generation number, same preference + // selectOpensearchAliasTargets uses to pick a write index. Without + // this, a legacy differently-prefixed generation kept attached + // read-only after double-prefix cleanup (see + // opensearchIndexBelongsTo) can coincidentally share its generation + // number with the canonical one, and a name-string tiebreak alone + // can then misidentify that legacy copy as "latest" and flag a + // perfectly correct write-index as invalid. + sorted := append([]string{}, indices...) + sort.Slice(sorted, func(i, j int) bool { + iCanonical := sorted[i] == aliasName || strings.HasPrefix(sorted[i], aliasName+"-") + jCanonical := sorted[j] == aliasName || strings.HasPrefix(sorted[j], aliasName+"-") + if iCanonical != jCanonical { + return iCanonical + } + + gi := getOpensearchGeneration(sorted[i]) + gj := getOpensearchGeneration(sorted[j]) + if gi == gj { + return sorted[i] > sorted[j] + } + return gi > gj + }) + + latest := sorted[0] + if writeIndices[0] != latest { + result.InvalidWriteAlias = append(result.InvalidWriteAlias, fmt.Sprintf("%s (write=%s latest=%s)", aliasName, writeIndices[0], latest)) + } + } + + if len(result.MissingAliases) > 0 || len(result.InvalidWriteAlias) > 0 || result.FoundAliases != result.ExpectedAliases { + result.Success = false + result.Reason = "Opensearch alias verification failed after repair" + log.Printf("[WARNING] %s. expected_aliases=%d found_aliases=%d missing=%d invalid_write=%d", result.Reason, result.ExpectedAliases, result.FoundAliases, len(result.MissingAliases), len(result.InvalidWriteAlias)) + } else { + result.Success = true + result.Reason = "Opensearch alias and index state repaired without data reindexing" + } + + // Surface in-progress migrations and skipped items in logs too, not just + // in the API response, so a stuck or silently-failing collision migration + // is visible without needing to poll the repair endpoint. + if len(result.MigrationTasks) > 0 { + log.Printf("[INFO] Opensearch collision migrations in progress: %s", strings.Join(result.MigrationTasks, "; ")) + } + if len(result.Skipped) > 0 { + log.Printf("[INFO] Opensearch prefix repair skipped items: %s", strings.Join(result.Skipped, "; ")) + } + + if ismEnabled { + appendBase := map[string]bool{} + for _, baseIndex := range GetOpensearchRolloverIndices() { + appendBase[strings.ToLower(baseIndex)] = true + } + + for _, aliasName := range expectedAliases { + bi := strings.TrimPrefix(strings.ToLower(aliasName), strings.ToLower(GetESIndexPrefix(""))) + if !appendBase[bi] { + continue + } + retention := getOpensearchRetentionDays(bi) + policyID := fmt.Sprintf("%s-%s", ismPolicyName, aliasName) + ismReady, ismErr := ensureOpensearchISMRolloverPolicy(ctx, opensearchUrl, aliasName, rolloverConfig, retention, ismPolicyName) + if ismErr != nil { + log.Printf("[WARNING] Failed ensuring ISM rollover policy '%s' in prefix fix: %s", policyID, ismErr) + continue + } + if !ismReady { + continue + } + for indexName, aliases := range verifiedAliasInfo { + state, ok := aliases[aliasName] + if !ok || !state.Present { + continue + } + + if err := ensureOpensearchIndexRolloverAliasSetting(ctx, opensearchUrl, indexName, aliasName); err != nil { + log.Printf("[WARNING] Failed ensuring rollover alias on %s for alias %s: %s", indexName, aliasName, err) + continue + } + + if err := ensureOpensearchIndexISMPolicy(ctx, opensearchUrl, indexName, policyID); err != nil { + log.Printf("[WARNING] Failed attaching ISM policy '%s' to %s: %s", policyID, indexName, err) + } + } + } + } + + return result, nil +} + +// inFlightOpensearchCollisionMigrations tracks alias/source-index names +// currently being migrated by a background goroutine in this process, so a +// repeated call within this process (e.g. the startup init path and the +// manual repair API both invoking FixOpensearchIndexPrefix) never launches a +// second concurrent migration for the same index. +// +// Shuffle backends commonly run as multiple replicas against the same +// OpenSearch cluster, so this in-process guard alone can't stop every +// replica from starting its own copy of the same legacy index at once. +// Cross-replica duplicate-avoidance is handled separately by +// findRunningOpensearchReindexTask, which asks OpenSearch's own _tasks API +// (shared cluster state, not a Shuffle-owned lock) whether a matching +// reindex is already under way before starting a new one. +// +// This is deliberately best-effort rather than a hard mutex: every +// operation in this migration (op_type:create+conflicts:proceed copy, +// 404-tolerant delete, idempotent alias-add) is safe to run redundantly, so +// a missed race only costs some wasted duplicate cluster work, never data +// corruption. +var inFlightOpensearchCollisionMigrations sync.Map + +// startOpensearchCollisionMigrationAsync launches (or, if a migration for +// this source index is already running in this process, no-ops) a full +// background migration of a legacy plain index that collides with its +// intended alias name into a correctly-named, correctly-mapped generation +// index, finishing with an atomic alias cutover once - and only once - the +// copy is verified complete with zero document failures. +// +// Runs entirely in a goroutine so a multi-hour reindex of a large legacy +// index (hundreds of GB is realistic here) never blocks backend startup. +// +// Intentionally requires no operator interaction: this must be safe to run +// unattended across every Shuffle deployment, including ones nobody is +// actively watching, and safe to run on every replica of a multi-pod +// deployment - cross-replica duplicate avoidance is handled inside +// runOpensearchReindexToCompletion by checking OpenSearch's own _tasks API +// for an already-running matching reindex before starting a new one. +func startOpensearchCollisionMigrationAsync(foundClient opensearchapi.Client, opensearchUrl, sourceIndex, targetIndex, baseIndex string) { + if _, alreadyRunning := inFlightOpensearchCollisionMigrations.LoadOrStore(sourceIndex, true); alreadyRunning { + return + } + + go func() { + defer inFlightOpensearchCollisionMigrations.Delete(sourceIndex) + + if err := runOpensearchCollisionMigration(foundClient, opensearchUrl, sourceIndex, targetIndex, baseIndex); err != nil { + log.Printf("[ERROR] Opensearch collision migration %s -> %s did not complete: %s. Nothing was deleted; this will be retried automatically (from wherever the idempotent copy left off) the next time Opensearch index init runs.", sourceIndex, targetIndex, err) + } + }() +} + +// runOpensearchCollisionMigration performs the full migration for one +// colliding index: create target (with the corrected mapping) if missing, +// bulk-copy while the source is still live, write-block the source and do a +// final catch-up copy to close any gap from concurrent writes during the +// (potentially very long) bulk copy, verify an exact document count match, +// and only then delete the source and swap the alias in. +// +// Any failure at any step aborts without deleting the source or touching +// the alias, leaving the next automatic retry to pick up from current +// state. +// +// Safe to run concurrently from multiple replicas: every step is +// idempotent/tolerant of having already been done (op_type:create + +// conflicts:proceed copy, 404-tolerant delete, idempotent alias-add), and +// runOpensearchReindexToCompletion additionally avoids starting duplicate +// reindex work when another replica already has a matching task running. +func runOpensearchCollisionMigration(foundClient opensearchapi.Client, opensearchUrl, sourceIndex, targetIndex, baseIndex string) error { + targetExists, err := checkOpensearchIndexExists(foundClient, opensearchUrl, targetIndex) + if err != nil { + return fmt.Errorf("checking target index: %w", err) + } + if !targetExists { + if err := createOpensearchIndex(foundClient, opensearchUrl, targetIndex, baseIndex); err != nil { + return fmt.Errorf("creating target index: %w", err) + } + } + + log.Printf("[INFO] Opensearch collision migration: starting bulk copy %s -> %s (large legacy indices can take a long time here; safe to restart the backend during this phase)", sourceIndex, targetIndex) + if err := runOpensearchReindexToCompletion(foundClient, opensearchUrl, sourceIndex, targetIndex); err != nil { + return fmt.Errorf("bulk copy: %w", err) + } + + // Freeze the source so a final catch-up pass can close any gap opened + // by writes that landed concurrently during the (potentially + // hours-long) bulk copy above, before we trust the document counts to + // match exactly. + if err := setOpensearchIndexWriteBlock(foundClient, opensearchUrl, sourceIndex, true); err != nil { + return fmt.Errorf("write-blocking source before final catch-up: %w", err) + } + + if err := runOpensearchReindexToCompletion(foundClient, opensearchUrl, sourceIndex, targetIndex); err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, sourceIndex) + return fmt.Errorf("final write-blocked catch-up copy: %w", err) + } + + // _count (like _search) only sees refreshed segments, not documents + // written moments ago - force a refresh on both indices before trusting + // the comparison below, otherwise the tail of the catch-up copy above + // can make targetCount look behind even though the copy fully + // succeeded, for indices still receiving writes right up to the + // write-block (exactly the high-volume case this is built for). + if err := refreshOpensearchIndex(foundClient, opensearchUrl, sourceIndex); err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, sourceIndex) + return fmt.Errorf("refreshing source before final count check: %w", err) + } + if err := refreshOpensearchIndex(foundClient, opensearchUrl, targetIndex); err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, sourceIndex) + return fmt.Errorf("refreshing target before final count check: %w", err) + } + + sourceCount, err := getOpensearchIndexCount(foundClient, opensearchUrl, sourceIndex) + if err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, sourceIndex) + return fmt.Errorf("getting final source count: %w", err) + } + targetCount, err := getOpensearchIndexCount(foundClient, opensearchUrl, targetIndex) + if err != nil { + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, sourceIndex) + return fmt.Errorf("getting final target count: %w", err) + } + if targetCount < sourceCount { + // Unblock and let the next automatic retry redo the whole idempotent + // copy - something (most likely a write landing in the narrow window + // between the write-block taking effect and the catch-up reindex + // starting) left the target still behind, and deleting the source + // with data still missing would be permanent data loss. + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, sourceIndex) + return fmt.Errorf("target count %d still behind source count %d after write-blocked catch-up - not deleting source", targetCount, sourceCount) + } + + // Delete the source index and create the alias in a SINGLE atomic + // _aliases request (remove_index + add), rather than two separate + // calls. OpenSearch applies all actions in one request as a single + // cluster-state update - it cannot partially apply this batch, so + // there is no "source deleted but alias not yet created" window for a + // crash/restart/network-blip to land in. + isWrite := true + actions := []OpensearchAliasAction{ + {RemoveIndex: &OpensearchAliasActionTarget{Index: sourceIndex}}, + {Add: &OpensearchAliasActionTarget{Index: targetIndex, Alias: sourceIndex, IsWriteIndex: &isWrite}}, + } + var swapErr error + for attempt := 0; attempt < 5; attempt++ { + if attempt > 0 { + time.Sleep(time.Duration(attempt) * 2 * time.Second) + } + swapErr = updateOpensearchAliases(foundClient, opensearchUrl, actions) + if swapErr == nil { + break + } + log.Printf("[WARNING] Opensearch collision migration %s -> %s: atomic delete-source+add-alias attempt %d/5 failed (source index is untouched - retrying): %s", sourceIndex, targetIndex, attempt+1, swapErr) + } + if swapErr != nil { + // The _aliases request failed as a whole, so - per OpenSearch's + // atomic-batch guarantee - neither remove_index nor add was applied: + // sourceIndex is fully intact with all its data. Unblock writes so + // the application can keep using it normally; the whole migration + // (bulk copy included) is idempotent and will simply retry from + // scratch on the next restart. + clearOpensearchIndexWriteBlockBestEffort(foundClient, opensearchUrl, sourceIndex) + return fmt.Errorf("finalizing atomic delete-source+add-alias after 5 attempts (source index '%s' is untouched and write access has been restored - safe to retry on next restart): %w", sourceIndex, swapErr) + } + + log.Printf("[INFO] Opensearch collision migration complete: alias '%s' now served by '%s' (%d documents verified)", sourceIndex, targetIndex, targetCount) + return nil +} + +// runOpensearchReindexToCompletion starts an async reindex task +// (op_type:create + conflicts:proceed, so it is safe to re-run in full any +// number of times) and polls it to completion, surfacing any per-document +// failures as an error. +// +// Before starting a new task, it checks whether a matching reindex (same +// source and target index) is already running somewhere in the cluster - +// e.g. started a moment earlier by another backend replica - via +// findRunningOpensearchReindexTask, and polls that one instead. +// +// This is a best-effort optimization, not a hard mutex: it relies on +// OpenSearch's _tasks API description field including both index names, +// which is long standing behavior but not a formally documented guarantee. +// That's fine here because every step of this migration is safe to run +// redundantly; missing the existing task in a race just means briefly +// running two copies of the same idempotent reindex rather than any data +// loss. +func runOpensearchReindexToCompletion(foundClient opensearchapi.Client, opensearchUrl, sourceIndex, targetIndex string) error { + taskID, err := findRunningOpensearchReindexTask(foundClient, opensearchUrl, sourceIndex, targetIndex) + if err != nil { + log.Printf("[DEBUG] Opensearch collision migration %s -> %s: failed checking for an already-running reindex task (continuing to start a new one): %s", sourceIndex, targetIndex, err) + } + if taskID != "" { + log.Printf("[INFO] Opensearch collision migration %s -> %s: found an already-running matching reindex task %s (likely started by another replica) - polling it instead of starting a duplicate", sourceIndex, targetIndex, taskID) + } else { + taskID, err = startOpensearchReindexTask(foundClient, opensearchUrl, sourceIndex, targetIndex) + if err != nil { + return err + } + if taskID == "" { + // Nothing to do (e.g. destination already existed under a + // resource_already_exists_exception, which + // startOpensearchReindexTask treats as a benign no-op). + return nil + } + } + + for { + completed, failures, statusErr := getOpensearchTaskStatus(foundClient, opensearchUrl, taskID) + if statusErr != nil { + return fmt.Errorf("checking reindex task %s: %w", taskID, statusErr) + } + if !completed { + time.Sleep(30 * time.Second) + continue + } + if len(failures) > 0 { + sample := failures[0] + return fmt.Errorf("reindex task %s reported %d document failure(s), first: %s", taskID, len(failures), sample) + } + return nil + } +} + +// findRunningOpensearchReindexTask looks for an already-running reindex +// task (GET _tasks?actions=*reindex&detailed=true) whose description +// mentions both sourceIndex and targetIndex, and returns its task ID if +// found. Used purely to avoid starting redundant duplicate reindex work +// when multiple backend replicas race to migrate the same colliding index; +// returns an empty ID (no error) if none is found. +func findRunningOpensearchReindexTask(foundClient opensearchapi.Client, opensearchUrl, sourceIndex, targetIndex string) (string, error) { + resp, err := foundClient.Tasks.List(context.Background(), &opensearchapi.TasksListReq{ + Params: opensearchapi.TasksListParams{ + Actions: []string{"*reindex"}, + Detailed: opensearchapi.ToPointer(true), + }, + }) + if err != nil { + return "", fmt.Errorf("failed listing tasks: %w", err) + } + + for nodeID, node := range resp.Nodes { + for taskID, task := range node.Tasks { + if strings.Contains(task.Description, sourceIndex) && strings.Contains(task.Description, targetIndex) { + // _tasks/ expects the ":" form; the map + // key from the per-node "tasks" object is already that + // composite ID, so prefer it over the numeric ID field. + if strings.Contains(taskID, ":") { + return taskID, nil + } + return fmt.Sprintf("%s:%d", nodeID, task.ID), nil + } + } + } + + return "", nil +} + +type opensearchTaskStatusResponse struct { + Completed bool `json:"completed"` + Response struct { + Failures []struct { + Index string `json:"index"` + ID string `json:"id"` + Cause struct { + Reason string `json:"reason"` + } `json:"cause"` + } `json:"failures"` + } `json:"response"` + // Error is populated when the task aborted with a fatal, task-level + // exception (e.g. a shard-level query failure like OpenSearch's + // "ReleasableBytesStreamOutput cannot hold more than 2GB of data" when + // a batch of large documents overflows a single internal transport + // message) rather than per-document failures. + // + // This is reported alongside completed:true with an empty + // response.failures, so it must be checked independently - treating a + // populated Error as success just because failures is empty would + // silently accept a reindex that only got a fraction of the way + // through. + Error *struct { + Type string `json:"type"` + Reason string `json:"reason"` + } `json:"error"` +} + +// getOpensearchTaskStatus polls a single reindex task by ID (GET +// _tasks/), returning whether it has completed and a human-readable +// summary of any per-document failures reported in its final response. +func getOpensearchTaskStatus(foundClient opensearchapi.Client, opensearchUrl, taskID string) (bool, []string, error) { + req, err := http.NewRequest("GET", fmt.Sprintf("%s/_tasks/%s", opensearchUrl, taskID), nil) + if err != nil { + return false, nil, err + } + + resp, err := foundClient.Client.Transport.Perform(req) + if err != nil { + return false, nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return false, nil, err + } + + if resp.StatusCode == 403 { + return false, nil, fmt.Errorf("failed checking task %s: permission denied (403) - the OpenSearch role used by Shuffle needs the cluster permission 'cluster:monitor/task/get' to poll reindex tasks to completion: %s", taskID, string(body)) + } + if resp.StatusCode >= 300 { + return false, nil, fmt.Errorf("failed checking task %s: %s", taskID, string(body)) + } + + parsed := opensearchTaskStatusResponse{} + if err := json.Unmarshal(body, &parsed); err != nil { + return false, nil, err + } + if parsed.Error != nil { + // Task-level fatal error (aborted the whole reindex, distinct from + // per-document failures below) - never treat this as success even + // though it's reported with completed:true. + return true, nil, fmt.Errorf("reindex task %s aborted with a fatal error (%s): %s", taskID, parsed.Error.Type, parsed.Error.Reason) + } + if !parsed.Completed { + return false, nil, nil + } + + failures := make([]string, 0, len(parsed.Response.Failures)) + for _, f := range parsed.Response.Failures { + failures = append(failures, fmt.Sprintf("%s/%s: %s", f.Index, f.ID, f.Cause.Reason)) + } + + return true, failures, nil +} + +// setOpensearchIndexWriteBlock toggles index.blocks.write on an index. Used +// to briefly freeze the legacy source index for a final catch-up reindex +// pass before trusting a document-count comparison enough to delete it. +func setOpensearchIndexWriteBlock(foundClient opensearchapi.Client, opensearchUrl, indexName string, block bool) error { + body, err := json.Marshal(map[string]interface{}{ + "index.blocks.write": block, + }) + if err != nil { + return err + } + + if _, err := foundClient.Indices.Settings.Put(context.Background(), opensearchapi.SettingsPutReq{ + Indices: []string{indexName}, + Body: bytes.NewReader(body), + }); err != nil { + return fmt.Errorf("failed setting write block=%v on %s: %w", block, indexName, err) + } + + return nil +} + +// clearOpensearchIndexWriteBlockBestEffort undoes a write block after an +// earlier migration step has already failed, so callers can return their +// original (more relevant) error without it being shadowed by this +// best-effort cleanup. A failure here is not merely cosmetic though: it +// means indexName stays write-blocked - a real write outage on whatever +// alias currently points at it - until the next successful retry, which for +// these startup-only migrations can mean until the next backend restart. +// That must not be silently dropped, so it's logged here. +func clearOpensearchIndexWriteBlockBestEffort(foundClient opensearchapi.Client, opensearchUrl, indexName string) { + if err := setOpensearchIndexWriteBlock(foundClient, opensearchUrl, indexName, false); err != nil { + log.Printf("[ERROR] Failed to clear write block on %s after an earlier migration step failed - %s will remain write-blocked until the next successful retry: %s", indexName, indexName, err) + } +} + +// checkOpensearchIndexExists reports whether indexName currently exists, +// via a plain HEAD (200 = exists, 404 = doesn't). +func checkOpensearchIndexExists(foundClient opensearchapi.Client, opensearchUrl, indexName string) (bool, error) { + resp, err := foundClient.Indices.Exists(context.Background(), opensearchapi.IndicesExistsReq{Indices: []string{indexName}}) + if err != nil { + if resp != nil && resp.StatusCode == 404 { + return false, nil + } + return false, fmt.Errorf("failed checking index %s: %w", indexName, err) + } + + return true, nil +} + +// refreshOpensearchIndex forces a refresh (POST /_refresh) so +// documents written moments ago become visible to _count/_search +// immediately, instead of waiting for the index's normal refresh_interval +// (30s by default on Shuffle-created indices). Without this, +// getOpensearchIndexCount can under-count an index that just received +// writes (e.g. the tail of a reindex catch-up copy), causing a false +// "target count still behind source count" failure that would never +// converge for an index still receiving writes right up to the +// write-block. +func refreshOpensearchIndex(foundClient opensearchapi.Client, opensearchUrl, indexName string) error { + _, err := foundClient.Indices.Refresh(context.Background(), &opensearchapi.IndicesRefreshReq{Indices: []string{indexName}}) + if err != nil { + return fmt.Errorf("failed refreshing index %s: %w", indexName, err) + } + return nil +} + +// getOpensearchIndexCount returns the current document count for indexName +// via _count. Callers that need this to be accurate right after a write +// should call refreshOpensearchIndex first. +func getOpensearchIndexCount(foundClient opensearchapi.Client, opensearchUrl, indexName string) (int64, error) { + resp, err := foundClient.Indices.Count(context.Background(), &opensearchapi.IndicesCountReq{Indices: []string{indexName}}) + if err != nil { + return 0, fmt.Errorf("failed counting index %s: %w", indexName, err) + } + + return int64(resp.Count), nil +} + +// startOpensearchReindexTask starts an async (wait_for_completion=false) +// _reindex from sourceIndex to targetIndex with conflicts:proceed (so +// pre-existing target documents are skipped, not treated as errors), and +// returns the OpenSearch task ID for polling via getOpensearchTaskStatus. +// Returns ("", nil) - not an error - if a matching task already exists +// (resource_already_exists_exception), since that means another caller +// already started an equivalent reindex. +func startOpensearchReindexTask(foundClient opensearchapi.Client, opensearchUrl, sourceIndex, targetIndex string) (string, error) { + payload := map[string]interface{}{ + "source": map[string]interface{}{ + "index": sourceIndex, + // Caps how many source documents OpenSearch fetches per + // underlying batch. Left at the default (1000), a batch of + // large documents (workflow execution results routinely run + // into the hundreds of KB each) can overflow a single internal + // transport message and abort the entire reindex with + // "ReleasableBytesStreamOutput cannot hold more than 2GB of + // data" - a fatal task-level error, not a per-document failure + // (see the Error field on opensearchTaskStatusResponse). A + // small fixed batch size keeps each batch comfortably under + // that limit regardless of how large individual documents get. + "size": 100, + }, + "dest": map[string]interface{}{ + "index": targetIndex, + }, + "conflicts": "proceed", + } + + body, err := json.Marshal(payload) + if err != nil { + return "", err + } + + resp, err := foundClient.Reindex(context.Background(), opensearchapi.ReindexReq{ + Body: bytes.NewReader(body), + Params: opensearchapi.ReindexParams{WaitForCompletion: opensearchapi.ToPointer(false)}, + }) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "resource_already_exists_exception") { + return "", nil + } + return "", fmt.Errorf("failed starting reindex %s -> %s: %w", sourceIndex, targetIndex, err) + } + + if strings.TrimSpace(resp.Task) == "" { + return "", fmt.Errorf("reindex task missing in response") + } + + return resp.Task, nil +} + +// deleteOpensearchIndex deletes indexName. Treats "already doesn't exist" +// (404) as success, so it's safe to call redundantly (e.g. after a partial +// retry) without special-casing the not-found case at every call site. +func deleteOpensearchIndex(foundClient opensearchapi.Client, opensearchUrl, indexName string) error { + resp, err := foundClient.Indices.Delete(context.Background(), opensearchapi.IndicesDeleteReq{Indices: []string{indexName}}) + if err != nil { + if resp != nil && resp.Inspect().Response != nil && resp.Inspect().Response.StatusCode == 404 { + return nil + } + return fmt.Errorf("failed deleting index %s: %w", indexName, err) + } + + return nil +} + +type opensearchAliasState struct { + Present bool + IsWriteIndex bool +} + +// shuffleOwnedOpensearchIndexPatterns returns a comma-separated index +// pattern list matching only the indices Shuffle itself manages +// (GetOpensearchBaseIndices, combined with whatever +// SHUFFLE_OPENSEARCH_INDEX_PREFIX is configured - which defaults to empty, +// i.e. no prefix). +// +// Scoping the cluster-wide _cat/indices and _alias lookups below to this +// pattern lets a least-privilege OpenSearch role grant +// indices:monitor/settings/get only on Shuffle's own indices instead of +// requiring visibility into every index in the cluster (which +// shared/enterprise clusters with other tenants' indices typically won't +// grant) - e.g. a role scoped to exactly "shuffle_*". +// +// When a prefix is configured, this returns a single "_*" pattern +// rather than one "_*" per base index: a legacy +// double-prefixed name (e.g. "shuffle_shuffle_notifications-000001", from +// the historical double-prefix bug) does not start with +// "shuffle_notifications", but it still starts with "shuffle_" - so this +// stays a strict subset of the customary "_*" role grant while +// still catching any depth of prefix duplication. +// opensearchIndexBelongsTo strictly re-validates every name this turns up, +// so the wider net here only widens what gets considered, never what gets +// migrated. +// +// Without a prefix, "_*" would be just "_*" (matching nothing +// useful), so each base index keeps its own unprefixed "*" +// pattern instead - double-prefixing cannot occur without a prefix to +// duplicate in the first place. +func shuffleOwnedOpensearchIndexPatterns() string { + prefix := strings.ToLower(strings.TrimSpace(os.Getenv("SHUFFLE_OPENSEARCH_INDEX_PREFIX"))) + if prefix != "" { + return prefix + "_*" + } + + baseIndices := GetOpensearchBaseIndices() + patterns := make([]string, 0, len(baseIndices)) + for _, baseIndex := range baseIndices { + patterns = append(patterns, strings.ToLower(baseIndex)+"*") + } + + return strings.Join(patterns, ",") +} + +// OpensearchAliasResponse is the shape of a GET /_alias response. +type OpensearchAliasResponse map[string]OpensearchAliasEntry + +// OpensearchAliasEntry holds the aliases attached to a single index. +type OpensearchAliasEntry struct { + Aliases map[string]json.RawMessage `json:"aliases"` +} + +// getOpensearchAliases returns, for every Shuffle-owned index +// (shuffleOwnedOpensearchIndexPatterns), which aliases are attached to it +// and whether each is the write index for that alias - the data +// resolveAliasWriteIndex/selectOpensearchAliasTargets are built on. +func getOpensearchAliases(foundClient opensearchapi.Client, opensearchUrl string) (map[string]map[string]opensearchAliasState, error) { + aliasReq, err := http.NewRequest("GET", fmt.Sprintf("%s/%s/_alias", opensearchUrl, shuffleOwnedOpensearchIndexPatterns()), nil) + if err != nil { + return nil, err + } + + aliasResp, err := foundClient.Client.Transport.Perform(aliasReq) + if err != nil { + return nil, err + } + + aliasBody, err := io.ReadAll(aliasResp.Body) + if err != nil { + aliasResp.Body.Close() + return nil, err + } + aliasResp.Body.Close() + + if aliasResp.StatusCode >= 300 { + return nil, fmt.Errorf("failed reading opensearch aliases: %s", string(aliasBody)) + } + + rawAliasInfo := OpensearchAliasResponse{} + if err := json.Unmarshal(aliasBody, &rawAliasInfo); err != nil { + return nil, err + } + + aliasInfo := map[string]map[string]opensearchAliasState{} + type aliasDetails struct { + IsWriteIndex bool `json:"is_write_index,omitempty"` + } + + for indexName, aliasEntry := range rawAliasInfo { + aliasInfo[indexName] = map[string]opensearchAliasState{} + for aliasName, aliasRaw := range aliasEntry.Aliases { + details := aliasDetails{} + if err := json.Unmarshal(aliasRaw, &details); err != nil { + log.Printf("[WARNING] Failed parsing alias details for %s/%s - assuming is_write_index=false: %s", indexName, aliasName, err) + } + aliasInfo[indexName][aliasName] = opensearchAliasState{Present: true, IsWriteIndex: details.IsWriteIndex} + } + } + + return aliasInfo, nil +} + +// getOpensearchIndices resolves the concrete backing indices matching +// Shuffle's own index patterns. +// +// Deliberately implemented via /_settings rather than /_cat/indices: +// _cat/* endpoints are cluster-level actions in OpenSearch's security +// plugin (requiring cluster:monitor/* even when the path includes an index +// pattern) and, once granted, let the credential query /_cluster/state or +// unscoped /_cat/indices directly to see every index's +// name/mappings/settings cluster-wide - a real cross-tenant metadata leak +// on a shared/multi-tenant cluster. +// +// /_settings (and /_alias below), being genuine per-index API endpoints, +// are enforced per matched index by the security plugin: a request for a +// pattern outside the role's granted index_patterns is rejected outright. +func getOpensearchIndices(foundClient opensearchapi.Client, opensearchUrl string) ([]string, error) { + resp, err := foundClient.Indices.Settings.Get(context.Background(), &opensearchapi.SettingsGetReq{ + Indices: []string{shuffleOwnedOpensearchIndexPatterns()}, + Params: opensearchapi.SettingsGetParams{FilterPath: []string{"*.settings.index.provided_name"}}, + }) + if err != nil { + return nil, fmt.Errorf("failed reading opensearch indices: %w", err) + } + + indices := []string{} + for indexName := range resp.Indices { + if strings.TrimSpace(indexName) != "" { + indices = append(indices, indexName) + } + } + + return indices, nil +} + +// opensearchIndexBelongsTo reports whether name (an index name or an alias +// name) is baseIndex, a generation of it ("-000001"), or a legacy +// variant with the SHUFFLE_OPENSEARCH_INDEX_PREFIX applied more than once +// (a historical bug double- or triple-prefixed some index names). +// +// Rather than guessing one specific corrupted variant up front (e.g. +// "prefix_prefix_baseIndex") and checking for that exact string, this +// strips one prefix layer at a time and re-checks, so any number of +// accidental repeats is recognized uniformly. +func opensearchIndexBelongsTo(name, baseIndex, prefix string) bool { + name = strings.ToLower(strings.TrimSpace(name)) + baseIndex = strings.ToLower(strings.TrimSpace(baseIndex)) + prefixed := "" + if prefix != "" { + prefixed = prefix + "_" + } + + for { + if name == baseIndex || strings.HasPrefix(name, baseIndex+"-") { + return true + } + + if prefixed == "" || !strings.HasPrefix(name, prefixed) { + return false + } + + name = strings.TrimPrefix(name, prefixed) + } +} + +// selectOpensearchAliasTargets finds every existing index that belongs to +// baseIndex - by alias attachment or by generation-numbered name prefix, +// including legacy multi-prefixed variants (see opensearchIndexBelongsTo) +// - so callers can migrate all of them onto the correct alias in one pass. +// +// Returns the full candidate list plus which one is (or should become) the +// write index, preferring an already-correctly-named generation over the +// highest generation number. +func selectOpensearchAliasTargets(baseIndex, prefix string, aliasInfo map[string]map[string]opensearchAliasState, allIndices []string) ([]string, string) { + expectedAlias := strings.ToLower(GetESIndexPrefix(baseIndex)) + candidateMap := map[string]bool{} + + for indexName, aliases := range aliasInfo { + for aliasName, state := range aliases { + if state.Present && opensearchIndexBelongsTo(aliasName, baseIndex, prefix) { + candidateMap[indexName] = true + break + } + } + } + + for _, indexName := range allIndices { + if opensearchIndexBelongsTo(indexName, baseIndex, prefix) { + candidateMap[indexName] = true + } + } + + targetIndices := []string{} + for indexName := range candidateMap { + targetIndices = append(targetIndices, indexName) + } + + if len(targetIndices) == 0 { + return targetIndices, "" + } + + sort.Slice(targetIndices, func(i, j int) bool { + gi := getOpensearchGeneration(targetIndices[i]) + gj := getOpensearchGeneration(targetIndices[j]) + if gi == gj { + return targetIndices[i] > targetIndices[j] + } + return gi > gj + }) + + writeIndex := "" + for _, indexName := range targetIndices { + if indexName == expectedAlias || strings.HasPrefix(indexName, expectedAlias+"-") { + writeIndex = indexName + break + } + } + + if writeIndex == "" { + writeIndex = targetIndices[0] + } + + return targetIndices, writeIndex +} + +// getOpensearchGeneration extracts the trailing "-NNNNNN" rollover +// generation number from an index name (e.g. 2 for "shuffle_logs-000002"), +// or 0 if the name has no numeric suffix. +func getOpensearchGeneration(indexName string) int { + parts := strings.Split(indexName, "-") + if len(parts) < 2 { + return 0 + } + + generation := parts[len(parts)-1] + value, err := strconv.Atoi(generation) + if err != nil { + return 0 + } + + return value +} + +// OpensearchIndexConfig is the parsed shape of a custom +// OPENSEARCH_INDEX_CONFIG override, or the body built for a default index +// create call. +// Aliases is untyped (json.RawMessage) because its only use is a +// len() presence check before being stripped - see createOpensearchIndex. +type OpensearchIndexConfig struct { + Aliases map[string]json.RawMessage `json:"aliases,omitempty"` + Settings map[string]interface{} `json:"settings,omitempty"` + Mappings map[string]interface{} `json:"mappings,omitempty"` +} + +// createOpensearchIndex creates indexName with either the operator-supplied +// OPENSEARCH_INDEX_CONFIG (with any aliases stripped - alias attachment is +// handled separately by the caller) or Shuffle's default settings/mappings +// (3 shards, 1 replica, 30s refresh, strings_as_keywords dynamic template). +// +// baseIndex (the unprefixed, un-generationed alias name, e.g. +// "workflowexecution_live") is used to look up opensearchCoreMappings so a +// freshly created index already has its curated field types instead of +// relying on migrateOpensearchSingleIndex to correct them moments later. +// Pass "" if no curated mapping applies (e.g. an index not in +// opensearchCoreMappings). +func createOpensearchIndex(foundClient opensearchapi.Client, opensearchUrl, indexName, baseIndex string) error { + indexConfig := OpensearchIndexConfig{} + customConfig := strings.TrimSpace(os.Getenv("OPENSEARCH_INDEX_CONFIG")) + if customConfig != "" { + if err := json.Unmarshal([]byte(customConfig), &indexConfig); err != nil { + return fmt.Errorf("invalid OPENSEARCH_INDEX_CONFIG: %w", err) + } + + if len(indexConfig.Aliases) > 0 { + indexConfig.Aliases = nil + } + } + + if len(indexConfig.Settings) == 0 && len(indexConfig.Mappings) == 0 { + mappings := opensearchDynamicMappingSettings() + if baseIndex != "" { + if props, ok := opensearchCoreMappings[baseIndex]["properties"]; ok { + mappings["properties"] = props + } + } + + indexConfig = OpensearchIndexConfig{ + Settings: getOpensearchDefaultIndexSettings(), + Mappings: mappings, + } + } + + indexConfigJson, err := json.Marshal(indexConfig) + if err != nil { + return err + } + + if _, err := foundClient.Indices.Create(context.Background(), opensearchapi.IndicesCreateReq{ + Index: indexName, + Body: bytes.NewReader(indexConfigJson), + }); err != nil { + return fmt.Errorf("failed creating index %s: %w", indexName, err) + } + + return nil +} + +// OpensearchAliasActionsRequest is the body of a POST _aliases request. +type OpensearchAliasActionsRequest struct { + Actions []OpensearchAliasAction `json:"actions"` +} + +// OpensearchAliasAction is a single add or remove action within an +// OpensearchAliasActionsRequest. +type OpensearchAliasAction struct { + Add *OpensearchAliasActionTarget `json:"add,omitempty"` + Remove *OpensearchAliasActionTarget `json:"remove,omitempty"` + // RemoveIndex deletes the named index outright (equivalent to DELETE + // /index), but - unlike a separate DELETE call - can be batched into + // the same _aliases request as an Add action so both happen as a + // single atomic cluster-state update. Only Index is meaningful here. + RemoveIndex *OpensearchAliasActionTarget `json:"remove_index,omitempty"` +} + +// OpensearchAliasActionTarget identifies the index/alias (and, for adds, +// whether it should become the write index) an OpensearchAliasAction +// applies to. +type OpensearchAliasActionTarget struct { + Index string `json:"index"` + // Alias must be omitempty: RemoveIndex actions only ever set Index (no + // Alias), and OpenSearch's strict per-action parser rejects a + // remove_index action that includes an "alias" field at all - even an + // empty one. Without omitempty here, every RemoveIndex action would + // serialize as {"index":"x","alias":""} and get rejected with + // x_content_parse_exception, silently breaking every atomic + // delete+alias-add swap that relies on it. + Alias string `json:"alias,omitempty"` + IsWriteIndex *bool `json:"is_write_index,omitempty"` +} + +// updateOpensearchAliases applies a batch of alias add/remove actions +// atomically via POST _aliases, e.g. moving a write alias from one +// generation to another in a single request. +func updateOpensearchAliases(foundClient opensearchapi.Client, opensearchUrl string, actions []OpensearchAliasAction) error { + aliasActions := OpensearchAliasActionsRequest{Actions: actions} + aliasBody, err := json.Marshal(aliasActions) + if err != nil { + return err + } + + if _, err := foundClient.Aliases(context.Background(), opensearchapi.AliasesReq{Body: bytes.NewReader(aliasBody)}); err != nil { + return fmt.Errorf("failed updating aliases: %w", err) + } + + return nil +} + +// HandleFixOpensearchPrefix is the admin-triggered HTTP endpoint for +// FixOpensearchIndexPrefix - lets an operator manually re-run the +// alias/index verification and repair instead of waiting for the next +// backend restart. +func HandleFixOpensearchPrefix(resp http.ResponseWriter, request *http.Request) { + cors := HandleCors(resp, request) + if cors { + return + } + + user, err := HandleApiAuthentication(resp, request) + if err != nil { + log.Printf("[WARNING] Api authentication failed in opensearch prefix fix: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Api authentication failed"}`)) + return + } + + if user.Role != "admin" { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Only admins or support can run this"}`)) + return + } + + ctx := GetContext(request) + result, err := FixOpensearchIndexPrefix(ctx) + if err != nil { + log.Printf("[ERROR] Failed fixing opensearch index prefix: %s", err) + result.Success = false + result.Reason = err.Error() + responseData, _ := json.Marshal(result) + resp.WriteHeader(500) + resp.Write(responseData) + return + } + + responseData, err := json.Marshal(result) + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed JSON parsing"}`)) + return + } + + resp.Header().Set("Content-Type", "application/json") + resp.WriteHeader(200) + resp.Write(responseData) +} diff --git a/structs.go b/structs.go index dd3c77d2..753e81b7 100755 --- a/structs.go +++ b/structs.go @@ -2535,6 +2535,11 @@ type ExecutionSearchWrapper struct { Source WorkflowExecution `json:"_source"` } `json:"hits"` } `json:"hits"` + Aggregations struct { + UniqueExecutions struct { + Value int `json:"value"` + } `json:"unique_executions"` + } `json:"aggregations"` } type OrgSearchWrapper struct { @@ -5324,81 +5329,6 @@ type MCPToolInputSchema struct { Required []string `json:"required"` } -type OpensearchPrefixFixResult struct { - Success bool `json:"success"` - Reason string `json:"reason,omitempty"` - ExpectedAliases int `json:"expected_aliases,omitempty"` - FoundAliases int `json:"found_aliases,omitempty"` - MissingAliases []string `json:"missing_aliases,omitempty"` - InvalidWriteAlias []string `json:"invalid_write_aliases,omitempty"` - MigrationTasks []string `json:"migration_tasks,omitempty"` - Created []string `json:"created,omitempty"` - WriteIndexUpdates []string `json:"write_index_updates,omitempty"` - Reindexed []string `json:"reindexed,omitempty"` - AliasUpdates []string `json:"alias_updates,omitempty"` - Skipped []string `json:"skipped,omitempty"` - Counts []OpensearchPrefixFixCountSnapshot `json:"counts,omitempty"` -} - -type OpensearchPrefixFixCountSnapshot struct { - SourceIndex string `json:"source_index"` - TargetIndex string `json:"target_index"` - SourceDocs int64 `json:"source_docs"` - TargetDocs int64 `json:"target_docs"` -} - -type OpensearchAliasResponse map[string]OpensearchAliasEntry - -type OpensearchAliasEntry struct { - Aliases map[string]json.RawMessage `json:"aliases"` -} - -type OpensearchIndexInfoResponse map[string]OpensearchIndexInfo - -type OpensearchIndexInfo struct { - Settings map[string]map[string]interface{} `json:"settings"` - Mappings map[string]interface{} `json:"mappings"` -} - -type OpensearchReindexRequest struct { - Source OpensearchReindexSourceDest `json:"source"` - Dest OpensearchReindexSourceDest `json:"dest"` -} - -type OpensearchReindexSourceDest struct { - Index string `json:"index"` -} - -type OpensearchAliasActionsRequest struct { - Actions []OpensearchAliasAction `json:"actions"` -} - -type OpensearchAliasAction struct { - Add *OpensearchAliasActionTarget `json:"add,omitempty"` - Remove *OpensearchAliasActionTarget `json:"remove,omitempty"` -} - -type OpensearchAliasActionTarget struct { - Index string `json:"index"` - Alias string `json:"alias"` - IsWriteIndex *bool `json:"is_write_index,omitempty"` -} - -type OpensearchCreateIndexRequest struct { - Settings map[string]interface{} `json:"settings,omitempty"` - Mappings map[string]interface{} `json:"mappings,omitempty"` -} - -type OpensearchIndexConfig struct { - Aliases map[string]OpensearchIndexAliasConfig `json:"aliases,omitempty"` - Settings map[string]interface{} `json:"settings,omitempty"` - Mappings map[string]interface{} `json:"mappings,omitempty"` -} - -type OpensearchIndexAliasConfig struct { - IsWriteIndex bool `json:"is_write_index,omitempty"` -} - // Only partial part of it type AppBuildRequest struct { Editing bool `datastore:"editing"`