we are currently developing a generic connector framework to access JSON-resources from an HTTP-Server. As the connector should not depend on a data model it heavily relies on DynamicType. As there is a large amount of data in the external system and an timestamp is provided, we chose to use the cookie filter approach to implement and ChangedIdEnumerator for incremental crawls. For a full crawl we use an IdEnumerator-Method on every entity. The .bdcm-Model (there are more entities in the original model) looks the following:
<?xml version="1.0" encoding="utf-16"?>
<Model xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns="http://schemas.microsoft.com/windows/2007/BusinessDataCatalog"
Name="SalesModel">
<LobSystems>
<LobSystem Name="BpnConnectorLOB" Type="Custom">
<Properties>
<Property Name="SystemUtilityTypeName" Type="System.String">BpnSharepoint.Connector.BpnConnector,...</Property>
<Property Name="SystemUtilityInstallDate" Type="System.DateTime">2012-01-01 00:00:00Z</Property>
<Property Name="InputUriProcessor" Type="System.String">BpnSharepoint.Connector....</Property>
<Property Name="OutputUriProcessor" Type="System.String">BpnSharepoint.Connector....</Property>
</Properties>
<LobSystemInstances>
<LobSystemInstance Name="BpnSharepointModel">
<Properties>
<Property Name="BpnRemoteBaseUrl" Type="System.String">http://localhost:8080/</Property>
<Property Name="DebugOutputEnabled" Type="System.Boolean">false</Property>
<Property Name="AuthenticationType" Type="System.String">Credentials</Property>
<Property Name="SsoApplicationId" Type="System.String">BpnConnector</Property>
</Properties>
</LobSystemInstance>
</LobSystemInstances>
<Entities>
<!-- ############### -->
<!-- ### Contact ### -->
<!-- ############### -->
<Entity Name="Contact"
DefaultDisplayName="büro+ Kontakt"
Namespace="BpnSharepoint.Connector.BpnSharepointModel"
Version="1.0.0.4">
<Properties>
<Property Name="Title" Type="System.String">FullName</Property>
<Property Name="DefaultAction" Type="System.String">Default</Property>
</Properties>
<Identifiers>
<Identifier Name="Id" TypeName="System.String" />
</Identifiers>
<Methods>
<!-- Contacts-ID-Enumerator-Methode starten-->
<Method Name="Read Contact Ids" DefaultDisplayName="Alle büro+ Kontakt-Ids lesen">
<Properties>
<Property Name="MethodUrlPath" Type="System.String">/contacts</Property>
</Properties>
<Parameters>
<Parameter Direction="Return" Name="Id Enumerator List">
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType[]" Name="Nodes" IsCollection="true" >
<TypeDescriptors>
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType" Name="Node">
<TypeDescriptors>
<TypeDescriptor TypeName="System.String" IdentifierName="Id" Name="Id" />
</TypeDescriptors>
</TypeDescriptor>
</TypeDescriptors>
</TypeDescriptor>
</Parameter>
</Parameters>
<MethodInstances>
<MethodInstance Type="IdEnumerator"
Name="Id Enumerator"
DefaultDisplayName="Alle büro+ Kontakt-Ids lesen"
ReturnParameterName="Id Enumerator List"
Default="true">
<Properties>
<!--<Property Name="BatchByDefault" Type="System.Boolean">true</Property>-->
<Property Name="RootFinder" Type="System.String">true</Property>
</Properties>
<AccessControlList>
<AccessControlEntry Principal="NT AUTHORITY\Authenticated Users">
<Right BdcRight="Execute" />
</AccessControlEntry>
<AccessControlEntry Principal="NT AUTHORITY\System">
<Right BdcRight="SetPermissions"/>
</AccessControlEntry>
</AccessControlList>
</MethodInstance>
</MethodInstances>
</Method>
<!-- Contacts-ID-Enumerator-Methode beenden-->
<!-- Changed-ID-Enumerator-Methode starten-->
<Method Name="Read Changed Contact Ids" DefaultDisplayName="Geänderte büro+ Kontakt-Ids lesen">
<Properties>
<Property Name="MethodUrlPath" Type="System.String">/contacts</Property>
</Properties>
<FilterDescriptors>
<FilterDescriptor Name="LastCrawl" Type="InputOutput">
<Properties>
<Property Name="SynchronizationCookie" Type="System.String" />
</Properties>
</FilterDescriptor>
<FilterDescriptor Name="Timestamp" Type="Timestamp" />
</FilterDescriptors>
<Parameters>
<Parameter Direction="InOut" Name="Last Crawl Date">
<TypeDescriptor Name="Last Crawl Date" TypeName="System.DateTime" IsCollection="false" AssociatedFilter="LastCrawl">
<Interpretation>
<NormalizeDateTime LobDateTimeMode="Local"/>
</Interpretation>
</TypeDescriptor>
</Parameter>
<Parameter Direction="Return" Name="Id Enumerator List">
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType[]" Name="Nodes" IsCollection="true" >
<TypeDescriptors>
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType" Name="Node">
<TypeDescriptors>
<TypeDescriptor TypeName="System.String" IdentifierName="Id" Name="Id" />
</TypeDescriptors>
</TypeDescriptor>
</TypeDescriptors>
</TypeDescriptor>
</Parameter>
</Parameters>
<MethodInstances>
<MethodInstance Type="ChangedIdEnumerator"
Name="Changed Id Enumerator"
DefaultDisplayName="Geänderte büro+ Kontakt-Ids lesen"
ReturnParameterName="Id Enumerator List"
Default="true">
<AccessControlList>
<AccessControlEntry Principal="NT AUTHORITY\Authenticated Users">
<Right BdcRight="Execute" />
</AccessControlEntry>
<AccessControlEntry Principal="NT AUTHORITY\System">
<Right BdcRight="SetPermissions"/>
</AccessControlEntry>
</AccessControlList>
</MethodInstance>
</MethodInstances>
</Method>
<!-- Changed-ID-Enumerator-Methode beenden-->
<!-- Deleted-ID-Enumerator-Methode starten-->
<Method Name="Read Deleted Contact Ids" DefaultDisplayName="Gelöschte büro+ Kontakt-Ids lesen">
<Properties>
<Property Name="MethodUrlPath" Type="System.String">/contacts</Property>
</Properties>
<FilterDescriptors>
<FilterDescriptor Name="LastCrawl" Type="InputOutput">
<Properties>
<Property Name="SynchronizationCookie" Type="System.String" />
</Properties>
</FilterDescriptor>
<FilterDescriptor Name="Timestamp" Type="Timestamp" />
</FilterDescriptors>
<Parameters>
<Parameter Direction="InOut" Name="Last Crawl Date">
<TypeDescriptor Name="Last Crawl Date" TypeName="System.DateTime" IsCollection="false" AssociatedFilter="LastCrawl">
<Interpretation>
<NormalizeDateTime LobDateTimeMode="Local"/>
</Interpretation>
</TypeDescriptor>
</Parameter>
<Parameter Direction="Return" Name="Id Enumerator List">
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType[]" Name="Nodes" IsCollection="true" >
<TypeDescriptors>
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType" Name="Node">
<TypeDescriptors>
<TypeDescriptor TypeName="System.String" IdentifierName="Id" Name="Id" />
</TypeDescriptors>
</TypeDescriptor>
</TypeDescriptors>
</TypeDescriptor>
</Parameter>
</Parameters>
<MethodInstances>
<MethodInstance Type="DeletedIdEnumerator"
Name="Deleted Id Enumerator"
DefaultDisplayName="Gelöschte büro+ Kontakt-Ids lesen"
ReturnParameterName="Id Enumerator List"
Default="true">
<AccessControlList>
<AccessControlEntry Principal="NT AUTHORITY\Authenticated Users">
<Right BdcRight="Execute" />
</AccessControlEntry>
<AccessControlEntry Principal="NT AUTHORITY\System">
<Right BdcRight="SetPermissions"/>
</AccessControlEntry>
</AccessControlList>
</MethodInstance>
</MethodInstances>
</Method>
<!-- Deleted-ID-Enumerator-Methode beenden-->
<!-- Read Contacts-Methode starten -->
<Method Name="Read Contacts" DefaultDisplayName="Alle büro+ Kontakte lesen">
<Properties>
<Property Name="MethodUrlPath" Type="System.String">/contacts</Property>
</Properties>
<Parameters>
<Parameter Direction="Return" Name="Contact List">
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType[]" Name="Nodes" IsCollection="true" >
<TypeDescriptors>
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType" Name="Node">
<TypeDescriptors>
<TypeDescriptor TypeName="System.String" IdentifierName="Id" Name="Id" />
<TypeDescriptor TypeName="System.String" Name="CustomerId" />
<TypeDescriptor TypeName="System.String" Name="Title" />
<TypeDescriptor TypeName="System.String" Name="LastName" />
<TypeDescriptor TypeName="System.String" Name="FirstName" />
<TypeDescriptor TypeName="System.String" Name="BusinessAddressStreet" />
<TypeDescriptor TypeName="System.String" Name="BusinessAddressPostalCode" />
<TypeDescriptor TypeName="System.String" Name="BusinessAddressCity" />
<TypeDescriptor TypeName="System.String" Name="CompanyName" />
<TypeDescriptor TypeName="System.String" Name="BusinessFaxNumber" />
<TypeDescriptor TypeName="System.String" Name="PrimaryTelephoneNumber" />
<TypeDescriptor TypeName="System.String" Name="BusinessTelephoneNumber" />
<TypeDescriptor TypeName="System.String" Name="MobileTelephoneNumber" />
<TypeDescriptor TypeName="System.String" Name="BusinessHomePage" />
<TypeDescriptor TypeName="System.String" Name="Email1Address" />
</TypeDescriptors>
</TypeDescriptor>
</TypeDescriptors>
</TypeDescriptor>
</Parameter>
</Parameters>
<MethodInstances>
<MethodInstance Type="Finder" Name="Read List" DefaultDisplayName="Alle büro+ Kontakte lesen" ReturnParameterName="Contact List" ReturnTypeDescriptorName="Nodes" ReturnTypeDescriptorLevel="0">
<AccessControlList>
<AccessControlEntry Principal="NT AUTHORITY\Authenticated Users">
<Right BdcRight="Execute" />
</AccessControlEntry>
<AccessControlEntry Principal="NT AUTHORITY\System">
<Right BdcRight="SetPermissions"/>
</AccessControlEntry>
</AccessControlList>
</MethodInstance>
</MethodInstances>
</Method>
<!-- Read Contacts-Methode beenden -->
<!-- Read Contact-Methode starten -->
<Method Name="Read Contact" DefaultDisplayName="Einen büro+ Kontakt lesen">
<Properties>
<Property Name="MethodUrlPath" Type="System.String">/contacts/{Id}</Property>
<Property Name="DescriptionField" Type="System.String">FullName</Property>
</Properties>
<Parameters>
<Parameter Direction="In" Name="Id">
<TypeDescriptor TypeName="System.String" IdentifierName="Id" Name="Id" />
</Parameter>
<Parameter Direction="Return" Name="Contact">
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType" Name="Node">
<TypeDescriptors>
<TypeDescriptor TypeName="System.String" IdentifierName="Id" Name="Id" ReadOnly="true" />
<TypeDescriptor TypeName="System.String" Name="CustomerId" />
<TypeDescriptor TypeName="System.String" Name="Title" />
<TypeDescriptor TypeName="System.String" Name="LastName" />
<TypeDescriptor TypeName="System.String" Name="MiddleName" />
<TypeDescriptor TypeName="System.String" Name="FirstName" />
<TypeDescriptor TypeName="System.String" Name="FullName" />
<TypeDescriptor TypeName="System.String" Name="NickName" />
<TypeDescriptor TypeName="System.String" Name="BusinessAddressStreet" />
<TypeDescriptor TypeName="System.String" Name="BusinessAddressCity" />
<TypeDescriptor TypeName="System.String" Name="BusinessAddressPostalCode" />
<TypeDescriptor TypeName="System.String" Name="BusinessAddressState" />
<TypeDescriptor TypeName="System.String" Name="BusinessAddressCountry" />
<TypeDescriptor TypeName="System.String" Name="BusinessAddressPostOfficeBox" />
<TypeDescriptor TypeName="System.String" Name="HomeAddressStreet" />
<TypeDescriptor TypeName="System.String" Name="HomeAddressCity" />
<TypeDescriptor TypeName="System.String" Name="HomeAddressPostalCode" />
<TypeDescriptor TypeName="System.String" Name="HomeAddressState" />
<TypeDescriptor TypeName="System.String" Name="HomeAddressCountry" />
<TypeDescriptor TypeName="System.String" Name="HomeAddressPostOfficeBox" />
<TypeDescriptor TypeName="System.String" Name="JobTitle" />
<TypeDescriptor TypeName="System.String" Name="Profession" />
<TypeDescriptor TypeName="System.String" Name="Department" />
<TypeDescriptor TypeName="System.String" Name="CompanyName" />
<TypeDescriptor TypeName="System.String" Name="BusinessFaxNumber" />
<TypeDescriptor TypeName="System.String" Name="PrimaryTelephoneNumber" />
<TypeDescriptor TypeName="System.String" Name="BusinessTelephoneNumber" />
<TypeDescriptor TypeName="System.String" Name="MobileTelephoneNumber" />
<TypeDescriptor TypeName="System.String" Name="BusinessHomePage" />
<TypeDescriptor TypeName="System.String" Name="Email1Address" />
<TypeDescriptor TypeName="System.String" Name="Email2Address" />
<TypeDescriptor TypeName="System.String" Name="Body" />
</TypeDescriptors>
</TypeDescriptor>
</Parameter>
</Parameters>
<MethodInstances>
<MethodInstance Type="SpecificFinder" Name="Read Item" DefaultDisplayName="Einen büro+ Kontakt lesen" ReturnParameterName="Contact" ReturnTypeDescriptorName="Node" ReturnTypeDescriptorLevel="0">
<AccessControlList>
<AccessControlEntry Principal="NT AUTHORITY\Authenticated Users">
<Right BdcRight="Execute" />
</AccessControlEntry>
<AccessControlEntry Principal="NT AUTHORITY\System">
<Right BdcRight="SetPermissions"/>
</AccessControlEntry>
</AccessControlList>
</MethodInstance>
</MethodInstances>
</Method>
<!-- Read Contact-Methode beenden -->
<!-- Update Contact-Methode starten -->
<Method Name="Update Contact" DefaultDisplayName="Einen büro+ Kontakt ändern">
<Properties>
<Property Name="MethodUrlPath" Type="System.String">/contacts/{Id}</Property>
</Properties>
<Parameters>
<Parameter Direction="In" Name="Id">
<TypeDescriptor TypeName="System.String" Name="Id" PreUpdaterField="true" />
</Parameter>
<Parameter Direction="In" Name="Contact">
<TypeDescriptor TypeName="Microsoft.BusinessData.Runtime.DynamicType" Name="Node">
<TypeDescriptors>
<TypeDescriptor TypeName="System.String" Name="CustomerId" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="Title" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="LastName" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="MiddleName" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="FirstName" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="FullName" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="NickName" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="BusinessAddressStreet" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="BusinessAddressCity" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="BusinessAddressPostalCode" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="BusinessAddressState" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="BusinessAddressCountry" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="BusinessAddressPostOfficeBox" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="HomeAddressStreet" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="HomeAddressCity" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="HomeAddressPostalCode" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="HomeAddressState" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="HomeAddressCountry" UpdaterField="true" />
<TypeDescriptor TypeName="System.String" Name="HomeAddressPostOfficeBox" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="JobTitle" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="Profession" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="Department" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="CompanyName" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="BusinessFaxNumber" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="PrimaryTelephoneNumber" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="BusinessTelephoneNumber" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="MobileTelephoneNumber" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="BusinessHomePage" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="Email1Address" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="Email2Address" UpdaterField="true"/>
<TypeDescriptor TypeName="System.String" Name="Body" UpdaterField="true"/>
</TypeDescriptors>
</TypeDescriptor>
</Parameter>
</Parameters>
<MethodInstances>
<MethodInstance Type="Updater" Name="Update Item" DefaultDisplayName="Einen büro+ Kontakt ändern" Default="true">
<AccessControlList>
<AccessControlEntry Principal="NT AUTHORITY\Authenticated Users">
<Right BdcRight="Execute" />
</AccessControlEntry>
<AccessControlEntry Principal="NT AUTHORITY\System">
<Right BdcRight="SetPermissions"/>
</AccessControlEntry>
</AccessControlList>
</MethodInstance>
</MethodInstances>
</Method>
<!-- Update Contact-Methode beenden -->
</Methods>
<Actions>
<Action Name="Default"
Position="1"
Url="/bpnconnector/entities//Contact.aspx?Id={0}"
IsOpenedInNewWindow="false">
<ActionParameters>
<ActionParameter Index="0" Name="Id" />
</ActionParameters>
</Action>
</Actions>
</Entity>
</Entities>
A full crawl works like a charm but puts heavy load on the machine. When I write trace messages to ULS or attach to mssdmn.exe I see that in full crawls the ChangedIdEnumerator, DeletedIdEnumerator and IdEnumeratorMethod is called, then some LobUri and INamingContainer-magic happens and onwards the SpecificFinder is called for every Id returned from IdEnumerator.
When I start a incremental crawl with attached debugger, I see that the ChangedIdEnumerator is called with the right cookie argument. My ChangedIdEnumerator-Impl looks the following:
private void ExecuteChangedIdEnumerator(IMethodInstance mi, ILobSystemInstance si, object[] args, IExecutionContext context)
{
var cookie = Convert.ToDateTime(args[0]);
var timestamp = DateTime.Now;
var ret = new List<DynamicType>();
var methodUrl = BuildUrlFromLobSystemAndMethod(si, mi, true, int.MaxValue);
methodUrl = AppendUpdatedMinParamForChangedIdEnumerator(methodUrl, cookie);
var creds = GetRemoteCredentials(si);
var jsonStr = methodUrl.DownloadJsonFromUrlWithCreds(creds);
var jsonArray = JsonArrayObjects.Parse(jsonStr);
foreach (var jsonObj in jsonArray)
{
var dynType = new DynamicType();
dynType.PopulateNeccessaryTypeDescriptorsWithDictionary(mi, jsonObj);
ret.Add(dynType);
}
args[0] = timestamp;
args[1] = ret;
}
That is more or less the same as my IdEnumerator, but adds the cookie to the HTTP-query-url and has two arguments (one for the cookie INOUT, and the changed ids RETURN). The logic to build the DynamicType-collection with the Id-Fields is exactly the same for IdEnumerator and ChangedIdEnumertor. As far as I can say from debugging the method works right (as the IdEnumerator).
The Problem is now, that nothing happens after I returned the changed ids. The DeletedIdEnumerator, isn't called. No LobUri or INamingContainer methods are called. In ULS log I get the following suspicious entry:
08/04/2012 07:28:16.08 mssearch.exe (0x1918) 0x2C4C SharePoint Server Search GatherPI e5e0 High advising status change NSC_FLUSH (b9a315d3-1e4b-4ffb-8366-9e5cd5c5f0bb-crawl-0 Portal_Content crawl -1) [gatherobj.cxx:4514] d:\office\source\search\native\gather\server\gatherobj.cxx
08/04/2012 07:28:16.08 mssearch.exe (0x1918) 0x2C4C SharePoint Server Search GatherSvc fus3 Medium Transaction failed in plugin RHTG Url bpnshelf://contacts Error 0x80004003 [gthrtrx.cxx:1042] d:\office\source\search\native\gather\gthrsvc\gthrtrx.cxx
08/04/2012 07:28:16.08 mssearch.exe (0x1918) 0x16CC SharePoint Server Search GatherPI dww2 Medium processing status change (1) NSC_FLUSH, (b9a315d3-1e4b-4ffb-8366-9e5cd5c5f0bb-crawl-0, Portal_Content) thread id 0x16cc [statthrd.cxx:97] d:\office\source\search\native\gather\server\statthrd.cxx
08/04/2012 07:28:16.08 mssearch.exe (0x1918) 0x16CC SharePoint Server Search GatherPI dra4 High CGthrPrj::AdviseStatusChange : advising status change NSC_FLUSH (Portal_Content b9a315d3-1e4b-4ffb-8366-9e5cd5c5f0bb-crawl-0 ) [cgatherprj.cxx:1698] d:\office\source\search\native\gather\gthrsvc\cgatherprj.cxx
08/04/2012 07:28:16.08 mssearch.exe (0x1918) 0x16CC SharePoint Server Search GatherPI ds7h High CPluginCollection::AdviseStatusChange : advising status change NSC_FLUSH (Portal_Content b9a315d3-1e4b-4ffb-8366-9e5cd5c5f0bb-crawl-0 ) [plugin.cxx:929] d:\office\source\search\native\gather\gthrsvc\plugin.cxx
08/04/2012 07:28:16.08 mssearch.exe (0x1918) 0x16CC SharePoint Server Search GatherPI ds7h High CPluginCollection::AdviseStatusChange : advising status change NSC_FLUSH (Portal_Content b9a315d3-1e4b-4ffb-8366-9e5cd5c5f0bb-crawl-0 ) [plugin.cxx:929] d:\office\source\search\native\gather\gthrsvc\plugin.cxx
In the craw log of central administration I get an Crawl Error with "Object reference not set to an instance to an object". But as far as I can say (changed debugger settings to NullReferenceException) this one is not thrown in my connector code.
After unsuccessful trials to debug the issue, I also updated Sharepoint to SP1 and June 2012 CU and installed SP1 for Search Server Express. But that didn't solve the problem. (It even made the thing worse: before the GetAccessUri-Method in INamingContainer got called for every changed id, that isn't the case after update).
So the questions are:
What I'm doing wrong? Is this a known issue? (Found hints on the net, that other people have the same issue) What is the right way to impl. this. I would really appreciate any input of experts and insiders, as our possibilities to debug this issue came to a dead end here.
Best Regards
Joachim