TudbuT/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge pull request 'enh(issue search): sort by score and term based query for fuzzy search' (#5819) from snematoda/enh-issue-search into forgejo

Browse source 
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5819
Reviewed-by: Otto <otto@codeberg.org>
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
This commit is contained in:
Otto 2024-11-10 12:51:55 +00:00
commit ab36ab57e4
11 changed files with 76 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
modules/indexer/internal/bleve/query.go
View file

@ -19,6 +19,15 @@ func NumericEqualityQuery(value int64, field string) *query.NumericRangeQuery {
return q
}
// MatchQuery generates a match query for the given phrase, field and analyzer
func MatchQuery(matchTerm, field, analyzer string, fuzziness int) *query.MatchQuery {
q := bleve.NewMatchQuery(matchTerm)
q.FieldVal = field
q.Analyzer = analyzer
q.Fuzziness = fuzziness
return q
}
// MatchPhraseQuery generates a match phrase query for the given phrase, field and analyzer
func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query.MatchPhraseQuery {
q := bleve.NewMatchPhraseQuery(matchPhrase)

29
modules/indexer/issues/bleve/bleve.go
View file

@ -35,13 +35,7 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
})
}
const (
maxBatchSize = 16
// fuzzyDenominator determines the levenshtein distance per each character of a keyword
fuzzyDenominator = 4
// see https://github.com/blevesearch/bleve/issues/1563#issuecomment-786822311
maxFuzziness = 2
)
const maxBatchSize = 16
// IndexerData an update to the issue indexer
type IndexerData internal.IndexerData
@ -162,16 +156,25 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
var queries []query.Query
if options.Keyword != "" {
fuzziness := 0
if options.IsFuzzyKeyword {
fuzziness = min(maxFuzziness, len(options.Keyword)/fuzzyDenominator)
fuzziness := 1
if kl := len(options.Keyword); kl > 3 {
fuzziness = 2
} else if kl < 2 {
fuzziness = 0
}
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
inner_bleve.MatchQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
}...))
} else {
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, 0),
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, 0),
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, 0),
}...))
}
}
if len(options.RepoIDs) > 0 || options.AllPublic {

4
modules/indexer/issues/dboptions.go
View file

@ -78,7 +78,9 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp
searchOpt.Paginator = opts.Paginator
switch opts.SortType {
case "", "latest":
case "", "relevance":
searchOpt.SortBy = SortByScore
case "latest":
searchOpt.SortBy = SortByCreatedDesc
case "oldest":
searchOpt.SortBy = SortByCreatedAsc

2
modules/indexer/issues/elasticsearch/elasticsearch.go
View file

@ -236,7 +236,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
}
if options.SortBy == "" {
options.SortBy = internal.SortByCreatedAsc
options.SortBy = internal.SortByScore
}
sortBy := []elastic.Sorter{
parseSortBy(options.SortBy),

1
modules/indexer/issues/indexer.go
View file

@ -269,6 +269,7 @@ func IsAvailable(ctx context.Context) bool {
type SearchOptions = internal.SearchOptions
const (
SortByScore = internal.SortByScore
SortByCreatedDesc = internal.SortByCreatedDesc
SortByUpdatedDesc = internal.SortByUpdatedDesc
SortByCommentsDesc = internal.SortByCommentsDesc

1
modules/indexer/issues/internal/model.go
View file

@ -127,6 +127,7 @@ func (o *SearchOptions) Copy(edit ...func(options *SearchOptions)) *SearchOption
type SortBy string
const (
SortByScore SortBy = "-_score"
SortByCreatedDesc SortBy = "-created_unix"
SortByUpdatedDesc SortBy = "-updated_unix"
SortByCommentsDesc SortBy = "-comment_count"

20
modules/indexer/issues/internal/tests/tests.go
View file

@ -126,6 +126,7 @@ var cases = []*testIndexerCase{
},
SearchOptions: &internal.SearchOptions{
Keyword: "hello",
SortBy: internal.SortByCreatedDesc,
},
ExpectedIDs: []int64{1002, 1001, 1000},
ExpectedTotal: 3,
@ -139,6 +140,7 @@ var cases = []*testIndexerCase{
},
SearchOptions: &internal.SearchOptions{
Keyword: "hello world",
SortBy: internal.SortByCreatedDesc,
IsFuzzyKeyword: true,
},
ExpectedIDs: []int64{1002, 1001, 1000},
@ -157,6 +159,7 @@ var cases = []*testIndexerCase{
},
SearchOptions: &internal.SearchOptions{
Keyword: "hello",
SortBy: internal.SortByCreatedDesc,
RepoIDs: []int64{1, 4},
},
ExpectedIDs: []int64{1006, 1002, 1001},
@ -175,6 +178,7 @@ var cases = []*testIndexerCase{
},
SearchOptions: &internal.SearchOptions{
Keyword: "hello",
SortBy: internal.SortByCreatedDesc,
RepoIDs: []int64{1, 4},
AllPublic: true,
},
@ -597,6 +601,22 @@ var cases = []*testIndexerCase{
}
},
},
{
Name: "SortByScore",
SearchOptions: &internal.SearchOptions{
Paginator: &db.ListOptionsAll,
SortBy: internal.SortByScore,
},
Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) {
assert.Equal(t, len(data), len(result.Hits))
assert.Equal(t, len(data), int(result.Total))
for i, v := range result.Hits {
if i < len(result.Hits)-1 {
assert.GreaterOrEqual(t, v.Score, result.Hits[i+1].Score)
}
}
},
},
{
Name: "SortByCreatedAsc",
SearchOptions: &internal.SearchOptions{

14
modules/indexer/issues/meilisearch/meilisearch.go
View file

@ -208,13 +208,19 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
query.And(inner_meilisearch.NewFilterLte("updated_unix", options.UpdatedBeforeUnix.Value()))
}
if options.SortBy == "" {
options.SortBy = internal.SortByCreatedAsc
}
sortBy := []string{
var sortBy []string
switch options.SortBy {
// sort by relevancy (no explicit sorting)
case internal.SortByScore:
fallthrough
case "":
sortBy = []string{}
default:
sortBy = []string{
parseSortBy(options.SortBy),
"id:desc",
}
}
skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxTotalHits)

4
modules/templates/util_string.go
View file

@ -19,6 +19,10 @@ func NewStringUtils() *StringUtils {
return &stringUtils
}
func (su *StringUtils) Make(arr ...string) []string {
return arr
}
func (su *StringUtils) HasPrefix(s any, prefix string) bool {
switch v := s.(type) {
case string:

1
options/locale/locale_en-US.ini
View file

@ -1586,6 +1586,7 @@ issues.filter_type.mentioning_you = Mentioning you
issues.filter_type.review_requested = Review requested
issues.filter_type.reviewed_by_you = Reviewed by you
issues.filter_sort = Sort
issues.filter_sort.relevance = Relevance
issues.filter_sort.latest = Newest
issues.filter_sort.oldest = Oldest
issues.filter_sort.recentupdate = Recently updated

14
templates/repo/issue/filter_list.tmpl
View file

@ -146,13 +146,11 @@
</span>
{{svg "octicon-triangle-down" 14 "dropdown icon"}}
<div class="menu">
<a rel="nofollow" class="{{if or (eq .SortType "latest") (not .SortType)}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=latest&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.latest"}}</a>
<a rel="nofollow" class="{{if eq .SortType "oldest"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=oldest&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.oldest"}}</a>
<a rel="nofollow" class="{{if eq .SortType "recentupdate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=recentupdate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.recentupdate"}}</a>
<a rel="nofollow" class="{{if eq .SortType "leastupdate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=leastupdate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.leastupdate"}}</a>
<a rel="nofollow" class="{{if eq .SortType "mostcomment"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=mostcomment&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.mostcomment"}}</a>
<a rel="nofollow" class="{{if eq .SortType "leastcomment"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=leastcomment&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.leastcomment"}}</a>
<a rel="nofollow" class="{{if eq .SortType "nearduedate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=nearduedate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.nearduedate"}}</a>
<a rel="nofollow" class="{{if eq .SortType "farduedate"}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=farduedate&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.farduedate"}}</a>
<a rel="nofollow" class="{{if or (eq .SortType "relevance") (not .SortType)}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort=relevency&state={{$.State}}&labels={{.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{ctx.Locale.Tr "repo.issues.filter_sort.relevance"}}</a>
{{$o := .}}
{{range $opt := StringUtils.Make "latest" "oldest" "recentupdate" "mostcomment" "leastcomment" "nearduedate" "farduedate"}}
{{$text := ctx.Locale.Tr (printf "repo.issues.filter_sort.%s" $opt)}}
<a rel="nofollow" class="{{if eq $o.SortType $opt}}active {{end}}item" href="?q={{$.Keyword}}&type={{$.ViewType}}&sort={{$opt}}&state={{$.State}}&labels={{$o.SelectLabels}}&milestone={{$.MilestoneID}}&project={{$.ProjectID}}&assignee={{$.AssigneeID}}&poster={{$.PosterID}}&fuzzy={{$.IsFuzzy}}{{if $.ShowArchivedLabels}}&archived=true{{end}}">{{$text}}</a>
{{end}}
</div>
</div>