Merge pull request 'enh(issue search): sort by score and term based query for fuzzy search' (#5819) from snematoda/enh-issue-search into forgejo

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5819
Reviewed-by: Otto <otto@codeberg.org>
Reviewed-by: Gusted <gusted@noreply.codeberg.org>

modules/indexer/internal/bleve/query.go

@@ -19,6 +19,15 @@ func NumericEqualityQuery(value int64, field string) *query.NumericRangeQuery {
 	return q
 }
 
+// MatchQuery generates a match query for the given phrase, field and analyzer
+func MatchQuery(matchTerm, field, analyzer string, fuzziness int) *query.MatchQuery {
+	q := bleve.NewMatchQuery(matchTerm)
+	q.FieldVal = field
+	q.Analyzer = analyzer
+	q.Fuzziness = fuzziness
+	return q
+}
+
 // MatchPhraseQuery generates a match phrase query for the given phrase, field and analyzer
 func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query.MatchPhraseQuery {
 	q := bleve.NewMatchPhraseQuery(matchPhrase)

modules/indexer/issues/bleve/bleve.go

@@ -35,13 +35,7 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
 	})
 }
 
-const (
+const maxBatchSize = 16
-	maxBatchSize = 16
-	// fuzzyDenominator determines the levenshtein distance per each character of a keyword
-	fuzzyDenominator = 4
-	// see https://github.com/blevesearch/bleve/issues/1563#issuecomment-786822311
-	maxFuzziness = 2
-)
 
 // IndexerData an update to the issue indexer
 type IndexerData internal.IndexerData
@@ -162,16 +156,25 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
 	var queries []query.Query
 
 	if options.Keyword != "" {
-		fuzziness := 0
 		if options.IsFuzzyKeyword {
-			fuzziness = min(maxFuzziness, len(options.Keyword)/fuzzyDenominator)
+			fuzziness := 1
+			if kl := len(options.Keyword); kl > 3 {
+				fuzziness = 2
+			} else if kl < 2 {
+				fuzziness = 0
+			}
+			queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
+				inner_bleve.MatchQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
+				inner_bleve.MatchQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
+				inner_bleve.MatchQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
+			}...))
+		} else {
+			queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
+				inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, 0),
+				inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, 0),
+				inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, 0),
+			}...))
 		}
-
-		queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
-			inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
-			inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
-			inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
-		}...))
 	}
 
 	if len(options.RepoIDs) > 0 || options.AllPublic {

modules/indexer/issues/dboptions.go

@@ -78,7 +78,9 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp
 	searchOpt.Paginator = opts.Paginator
 
 	switch opts.SortType {
-	case "", "latest":
+	case "", "relevance":
+		searchOpt.SortBy = SortByScore
+	case "latest":
 		searchOpt.SortBy = SortByCreatedDesc
 	case "oldest":
 		searchOpt.SortBy = SortByCreatedAsc

modules/indexer/issues/elasticsearch/elasticsearch.go

@@ -236,7 +236,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
 	}
 
 	if options.SortBy == "" {
-		options.SortBy = internal.SortByCreatedAsc
+		options.SortBy = internal.SortByScore
 	}
 	sortBy := []elastic.Sorter{
 		parseSortBy(options.SortBy),

modules/indexer/issues/indexer.go

@@ -269,6 +269,7 @@ func IsAvailable(ctx context.Context) bool {
 type SearchOptions = internal.SearchOptions
 
 const (
+	SortByScore        = internal.SortByScore
 	SortByCreatedDesc  = internal.SortByCreatedDesc
 	SortByUpdatedDesc  = internal.SortByUpdatedDesc
 	SortByCommentsDesc = internal.SortByCommentsDesc

modules/indexer/issues/internal/model.go

@@ -127,6 +127,7 @@ func (o *SearchOptions) Copy(edit ...func(options *SearchOptions)) *SearchOption
 type SortBy string
 
 const (
+	SortByScore        SortBy = "-_score"
 	SortByCreatedDesc  SortBy = "-created_unix"
 	SortByUpdatedDesc  SortBy = "-updated_unix"
 	SortByCommentsDesc SortBy = "-comment_count"

modules/indexer/issues/internal/tests/tests.go

@@ -126,6 +126,7 @@ var cases = []*testIndexerCase{
 		},
 		SearchOptions: &internal.SearchOptions{
 			Keyword: "hello",
+			SortBy:  internal.SortByCreatedDesc,
 		},
 		ExpectedIDs:   []int64{1002, 1001, 1000},
 		ExpectedTotal: 3,
@@ -139,6 +140,7 @@ var cases = []*testIndexerCase{
 		},
 		SearchOptions: &internal.SearchOptions{
 			Keyword:        "hello world",
+			SortBy:         internal.SortByCreatedDesc,
 			IsFuzzyKeyword: true,
 		},
 		ExpectedIDs:   []int64{1002, 1001, 1000},
@@ -157,6 +159,7 @@ var cases = []*testIndexerCase{
 		},
 		SearchOptions: &internal.SearchOptions{
 			Keyword: "hello",
+			SortBy:  internal.SortByCreatedDesc,
 			RepoIDs: []int64{1, 4},
 		},
 		ExpectedIDs:   []int64{1006, 1002, 1001},
@@ -175,6 +178,7 @@ var cases = []*testIndexerCase{
 		},
 		SearchOptions: &internal.SearchOptions{
 			Keyword:   "hello",
+			SortBy:    internal.SortByCreatedDesc,
 			RepoIDs:   []int64{1, 4},
 			AllPublic: true,
 		},
@@ -597,6 +601,22 @@ var cases = []*testIndexerCase{
 			}
 		},
 	},
+	{
+		Name: "SortByScore",
+		SearchOptions: &internal.SearchOptions{
+			Paginator: &db.ListOptionsAll,
+			SortBy:    internal.SortByScore,
+		},
+		Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
+			assert.Equal(t, len(data), len(result.Hits))
+			assert.Equal(t, len(data), int(result.Total))
+			for i, v := range result.Hits {
+				if i < len(result.Hits)-1 {
+					assert.GreaterOrEqual(t, v.Score, result.Hits[i+1].Score)
+				}
+			}
+		},
+	},
 	{
 		Name: "SortByCreatedAsc",
 		SearchOptions: &internal.SearchOptions{

modules/indexer/issues/meilisearch/meilisearch.go

@@ -208,12 +208,18 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
 		query.And(inner_meilisearch.NewFilterLte("updated_unix", options.UpdatedBeforeUnix.Value()))
 	}
 
-	if options.SortBy == "" {
+	var sortBy []string
-		options.SortBy = internal.SortByCreatedAsc
+	switch options.SortBy {
-	}
+	// sort by relevancy (no explicit sorting)
-	sortBy := []string{
+	case internal.SortByScore:
-		parseSortBy(options.SortBy),
+		fallthrough
-		"id:desc",
+	case "":
+		sortBy = []string{}
+	default:
+		sortBy = []string{
+			parseSortBy(options.SortBy),
+			"id:desc",
+		}
 	}
 
 	skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxTotalHits)

modules/templates/util_string.go

@@ -19,6 +19,10 @@ func NewStringUtils() *StringUtils {
 	return &stringUtils
 }
 
+func (su *StringUtils) Make(arr ...string) []string {
+	return arr
+}
+
 func (su *StringUtils) HasPrefix(s any, prefix string) bool {
 	switch v := s.(type) {
 	case string:

options/locale/locale_en-US.ini

@@ -1586,6 +1586,7 @@ issues.filter_type.mentioning_you = Mentioning you
 issues.filter_type.review_requested = Review requested
 issues.filter_type.reviewed_by_you = Reviewed by you
 issues.filter_sort = Sort
+issues.filter_sort.relevance = Relevance
 issues.filter_sort.latest = Newest
 issues.filter_sort.oldest = Oldest
 issues.filter_sort.recentupdate = Recently updated

templates/repo/issue/filter_list.tmpl

@@ -146,13 +146,11 @@
 	</span>
 	{{svg "octicon-triangle-down" 14 "dropdown icon"}}
 	<div class="menu">
-		<a rel="nofollow" class="{{if or (eq .SortType "latest") (not .SortType)}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=latest&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.latest"}}</a>
+		<a rel="nofollow" class="{{if or (eq .SortType "relevance") (not .SortType)}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=relevency&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.relevance"}}</a>
-		<a rel="nofollow" class="{{if eq .SortType "oldest"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=oldest&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.oldest"}}</a>
+		{{$o := .}}
-		<a rel="nofollow" class="{{if eq .SortType "recentupdate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=recentupdate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.recentupdate"}}</a>
+		{{range $opt := StringUtils.Make "latest" "oldest" "recentupdate" "mostcomment" "leastcomment" "nearduedate" "farduedate"}}
-		<a rel="nofollow" class="{{if eq .SortType "leastupdate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=leastupdate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.leastupdate"}}</a>
+			{{$text := ctx.Locale.Tr (printf "repo.issues.filter_sort.%s" $opt)}}
-		<a rel="nofollow" class="{{if eq .SortType "mostcomment"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=mostcomment&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.mostcomment"}}</a>
+			<a rel="nofollow" class="{{if eq $o.SortType $opt}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort={{$opt}}&state={{$.State}}&labels={{$o.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{$text}}</a>
-		<a rel="nofollow" class="{{if eq .SortType "leastcomment"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=leastcomment&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.leastcomment"}}</a>
+		{{end}}
-		<a rel="nofollow" class="{{if eq .SortType "nearduedate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=nearduedate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.nearduedate"}}</a>
-		<a rel="nofollow" class="{{if eq .SortType "farduedate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=farduedate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.farduedate"}}</a>
 	</div>
 </div>