In the first installment of this series, we looked at returning results from a limited pool of items in a Lucene full text index. In this second installment, we significantly increase the number of generated items (3,000, by default) and add a numbered paging system, as used by the main commercial search engines and search sites.
The code and code narrative below reflects the changes that have been made since the first post. All source code is available online for this results paging post.
Sample App
The sample application generates 3,000 waffle text records with the exact count being configurable and stored in the appsettings.json file. These waffle items can be searched and return in paginated form with a default page size of 5 records (not configurable). Additional character escaping / nulling has been added to remove characters from searches prior to passing them to the search engine. The site is available online at https://dotnet-lucene-search.azurewebsites.net/
Dynamic Configuration - Settings
The dynamic configuration settings, specifically the size of the waffle text corpus to be generated and the random seed initializer used for generation, are stored in the appsettings.json file and read at runtime.
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"AllowedHosts": "*",
"BogusConfig": {
"Rand": "11784",
"WaffleCount": "3000"
}
}
Dynamic Configuration - Enablement
The Microsoft.Extensions libraries are added to the project in the .csproj file to enable dynamic, JSON-based configuration.
net6.0 enable enable
Dynamic Configuration - Activation
The program's dynamic configuration is implemented in the Program.cs file. The configuration file is open, settings are read and then passed dynamically to the GetData() method of the engine, which generates the sample data.
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
using Microsoft.Extensions.Configuration;
using search.Shared;
using BlazorStrap;
var builder = WebApplication.CreateBuilder(args);
// Add services to the container.
builder.Services.AddRazorPages();
builder.Services.AddServerSideBlazor();
builder.Services.AddBlazorStrap();
// Setup configuration
var cfgBuilder = new ConfigurationBuilder().AddJsonFile($"appsettings.json", true, true);
var config = cfgBuilder.Build();
// Search engine setup
SearchEngine.GetData(Int32.Parse(config["BogusConfig:Rand"]), Int32.Parse(config["BogusConfig:WaffleCount"]));
SearchEngine.Index();
var app = builder.Build();
// Configure the HTTP request pipeline.
if (!app.Environment.IsDevelopment())
{
app.UseExceptionHandler("/Error");
// The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
app.UseHsts();
}
app.UseHttpsRedirection();
app.UseStaticFiles();
app.UseRouting();
app.MapBlazorHub();
app.MapFallbackToPage("/_Host");
app.Run();
Pagination - Model Enablement
To enable pagination, additional attributes are added to the SearchModel class to allow for the total count of pages and current page for a specific search.
using System.ComponentModel.DataAnnotations;
namespace search.Shared
{
public class SearchModel{
[Required]
public string SearchText {get; set;}
public int ResultsCount {get; set;}
public int PageCount {get; set;}
public int CurrentPage {get; set;}
public List CurrentPageSearchResults {get; set;}
}
}
Pagination - Implementation
Pagination is implemented on the back end in the SearchEngine.cs class. The Search method signature and method have been changed significantly from the original post to enable paginated searches. Also, an EscapeSearchTerm function has been added to remove specific characters from the search text. This function is applied to search input within the Search method.
using Bogus;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.QueryParsers.Classic;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Store;
using Lucene.Net.Util;
using System.Text.RegularExpressions;
namespace search.Shared
{
public class SearchEngine{
public static List Data {get; set;}
private static RAMDirectory _directory;
public static IndexWriter Writer { get; set; }
public static void GetData(int Rand, int WaffleCount)
{
Randomizer.Seed = new Random(Rand);
var testWaffles = new Faker()
.RuleFor(wt => wt.GUID, f => Guid.NewGuid().ToString())
.RuleFor(
property: wt => wt.WaffleHead,
setter: (f, wt) => f.WaffleTitle())
.RuleFor(
property: wt => wt.WaffleBody,
setter: (f, wt) => f.WaffleText(
paragraphs: 2,
includeHeading: false));
var waffles = testWaffles.Generate(WaffleCount);
Data = new List();
foreach(WaffleText wt in waffles)
{
Data.Add(wt);
}
}
public static void Index()
{
const LuceneVersion lv = LuceneVersion.LUCENE_48;
Analyzer a = new StandardAnalyzer(lv);
_directory = new RAMDirectory();
var config = new IndexWriterConfig(lv, a);
Writer = new IndexWriter(_directory, config);
var guidField = new StringField("GUID", "", Field.Store.YES);
var headField = new TextField("WaffleHead", "", Field.Store.YES);
var bodyField = new TextField("WaffleBody", "", Field.Store.YES);
var d = new Document()
{
guidField,
headField,
bodyField
};
foreach (WaffleText wt in Data)
{
guidField.SetStringValue(wt.GUID);
headField.SetStringValue(wt.WaffleHead);
bodyField.SetStringValue(wt.WaffleBody);
Writer.AddDocument(d);
}
Writer.Commit();
}
public static void Dispose()
{
Writer.Dispose();
_directory.Dispose();
}
public static SearchModel Search(string input, int page)
{
const LuceneVersion lv = LuceneVersion.LUCENE_48;
Analyzer a = new StandardAnalyzer(lv);
var dirReader = DirectoryReader.Open(_directory);
var searcher = new IndexSearcher(dirReader);
string[] waffles = { "GUID", "WaffleHead", "WaffleBody" };
var multiFieldQP = new MultiFieldQueryParser(lv, waffles, a);
string _input = EscapeSearchTerm(input.Trim());
Query query = multiFieldQP.Parse(_input);
ScoreDoc[] docs = searcher.Search(query, null, 1000).ScoreDocs;
var returnModel = new SearchModel();
returnModel.CurrentPageSearchResults = new List();
returnModel.SearchText = _input;
returnModel.ResultsCount = docs.Length;
returnModel.PageCount = (int)Math.Ceiling(docs.Length/5.0);
returnModel.CurrentPage = page;
int first = (page-1)*5;
int last = first + 5;
for (int i = first; i < last && i < docs.Length; i++)
{
Document d = searcher.Doc(docs[i].Doc);
WaffleText _localWaffle = new WaffleText();
_localWaffle.GUID = d.Get("GUID");
_localWaffle.WaffleHead = d.Get("WaffleHead");
_localWaffle.WaffleBody = d.Get("WaffleBody");
returnModel.CurrentPageSearchResults.Add(_localWaffle);
}
dirReader.Dispose();
return returnModel;
}
// Lucene supports escaping the following chars: + - && || ! ( ) { } [ ] ^ " ~ * ? : \
// To make it easier, I remove / replace
private static string EscapeSearchTerm(string input)
{
input = Regex.Replace(input, @"\+", " ");
input = Regex.Replace(input, @"\-", " ");
input = Regex.Replace(input, @"\&", " ");
input = Regex.Replace(input, @"\|", " ");
input = Regex.Replace(input, @"\!", " ");
input = Regex.Replace(input, @"\(", " ");
input = Regex.Replace(input, @"\)", " ");
input = Regex.Replace(input, @"\{", " ");
input = Regex.Replace(input, @"\}", " ");
input = Regex.Replace(input, @"\[", " ");
input = Regex.Replace(input, @"\]", " ");
input = Regex.Replace(input, @"\^", " ");
input = Regex.Replace(input, @"\"", " ");
input = Regex.Replace(input, @"\~", " ");
input = Regex.Replace(input, @"\*", " ");
input = Regex.Replace(input, @"?", " ");
input = Regex.Replace(input, @"\:", " ");
input = Regex.Replace(input, @"\\", " ");
return input;
}
}
}
Pagination - User Interface
Finally, the front-end pagination is added to the Index.razor class. All of this is made much easier through the presence of a very capable BlazorStrap pagination component.
@page "/"Prose Search @if(@SearchText!=String.Empty) { Search } @if(@SearchResultsCount>0) { @if(@PageCount>1) { @foreach (var result in @searchModel.CurrentPageSearchResults) { } @result.WaffleHead
@result.WaffleBody
} } @code { private SearchModel searchModel = new SearchModel(); [Parameter] public int Page {get; set;} = 1; [Parameter] public int PageCount {get; set;} = 0; [Parameter] public string SearchText {get; set;} = string.Empty; [Parameter] public int SearchResultsCount {get; set;} = 0; private void HandleSearch() { searchModel = SearchEngine.Search(searchModel.SearchText, 1); SearchResultsCount = searchModel.ResultsCount; PageCount = searchModel.PageCount; SearchText = searchModel.SearchText; Page = 1; } private void UpdatePage() { searchModel = SearchEngine.Search(searchModel.SearchText, Page); } }