Background
My organization has been attempting to use FAST Search for Sharepoint 2010 as a search solution for our DMS. I am currently utilizing a .Net indexing connector to index a database of ~ 8 million records and their accompanying files. The full crawl, albeit time consuming (about 2 weeks), works wonderfully. It is certainly taxing on the crawl and index servers, but well within their limitations.
Problem
The first incremental crawl is causing a memory spike on both the crawl server (mssearch.exe) and index server (sql). I have implemented some custom logging in my connector, and I can see that it gathers all the ids and seems to only process the updated items. However, the incremental crawl never completes and, on the crawl server, mssearch.exe eventually consumes all available resources. On the index server, sqlservr.exe seems to have the same fate. Several full crawls have completed successfully, and services/servers have been restarted multiple times, but I have yet to have any luck. Full and incremental crawls work perfectly when only crawling a portion of the repository (test sizes: 30,000 & 100,000 items).
BCS Connector
- ReadList, ReadItem, FileStream method
- Batches files from database based on custom value (tried 100K, 500K, etc)
- Uses LastModified time stamp field for changes
- Custom security mapping
- Custom logging to indicate when batches are returned, when individual entities are processed (based on time interval), and any file stream errors
SharePoint Environment
- 1x WFE
- 1x Database Server
- Application Server (Crawl Server) 12GB Memory
- 1x Index Server (FAST) 16GB Memory
public DMSFile ReadItem(string id)
{
SqlConnection connection = new SqlConnection(connectionString);
SqlCommand command = new SqlCommand(cmd, connection);
DMSFile entity = new DMSFile();
connection.Open();
using (SqlDataReader r = command.ExecuteReader())
{
while (r.Read())
{
entity.DocId = id;
entity.DocNumber = Int32.Parse(r["DocNum"].ToString());
entity.VersionNum = Int32.Parse(r["VersionNum"].ToString());
//etc., etc., etc.
entity.SecurityDescriptor = security.getSecurityDescriptor(r["DefaultSecurity"].ToString(), r["DocNum"].ToString(), r["VersionNum"].ToString());
}
}
logger.writeItem(id);
command.Dispose();
connection.Close();
connection.Dispose();
return entity;
}
public IEnumerable<DMSFile> ReadList(ref string BatchingCookie, ref System.String HasMoreActivities)
{
HasMoreActivities = "false";
batchSize = logger.getBatchCount();
SqlConnection connection = new SqlConnection(connectionString);
SqlCommand command = new SqlCommand(cmd, connection);
List<DMSFile> entityList = new List<DMSFile>(batchSize);
if (entityList.Count > 0){entityList.Clear();}
connection.Open();
using (SqlDataReader rdr = command.ExecuteReader())
{
while (rdr.Read()) {
entityList.Add(new DMSFile() {
DocId = rdr["DocNum"].ToString() + "." + rdr["VersionNum"].ToString(),
LastModified = DateTime.Parse(rdr["LastEdited"].ToString()).ToUniversalTime()
});
}
}
if (entityList.Count == batchSize){
HasMoreActivities = "true";
BatchingCookie = entityList[entityList.Count - 1].DocId;
}
logger.writeAll(entityList.Count.ToString(), BatchingCookie);
DMSFile[] entityArray = entityList.ToArray();
command.Dispose();
connection.Close();
connection.Dispose();
return entityArray;
}
public Stream ReadItemStream(string id)
{
SqlConnection connection = new SqlConnection(connectionString);
SqlCommand command = new SqlCommand(cmd, connection);
connection.Open();
byte[] bytes = null;
//SQL only return document location
string fileLocation = command.ExecuteScalar().ToString();
command.Dispose();
connection.Close();
connection.Dispose();
try
{
using (FileStream fs = new FileStream(fileLocation, FileMode.Open, FileAccess.Read))
{
int length = Int32.Parse(fs.Length.ToString());
bytes = new byte[length];
fs.Read(bytes, 0, length);
}
}
catch (Exception fileError){logger.logError(fileError.Message + " " + fileLocation2);}
return new MemoryStream(bytes);
}
