Batch Update for 700+ items failing

Question

I have written a code that gets all the URLs from the URL column in a document library , converts them into base64 and stores the resultant string in the respective column.

The code works fine on a few list items , However fails for 700+ items without any errors.

Checked the network trace and found the below

1. Images being fetched

2. Only 100 List Items out of 700+ Updated as shown below

3. Found the below entry in network tab , not sure what this means. The executions stops at this point and doesn't proceed further

The end goal here is to convert all the URLs in the URL column to base64 and store it in the base64 column against every entry in the document library.

Can anyone here suggest how this needs to be handled and if it is possible to skip the entries that cause errors at the time of execution and proceed further?

Below is the code

Get all the list items

function GetItemId() {
            //debugger;
            var deferred = $.Deferred();
            $.ajax({
                url: _spPageContextInfo.webAbsoluteUrl + "/_api/Web/Lists/GetByTitle('DBSDistributorList')/Items?$select=ID,BaseURL,DistributorLogo",
                method: "GET",
                type: 'GET',
                async: false,
                headers: {
                    "accept": "application/json;odata=verbose",
                    "content-Type": "application/json;odata=verbose"
                },
                success: function (result) {
                    deferred.resolve(result);
                    $.each(result.d.results, function (index, row) {
                        UpdateListItemUsingItemId(row["ID"], row["DistributorLogo"]["Url"]);

                    });


                },
                error: function (result) {
                    deferred.reject(result);
                }
            }); // Ajax list items

            return deferred.promise();

        };

Update all the list Items

function UpdateListItemUsingItemId(Id, dLogo) {
            //var deferred = $.Deferred();
            convertImgToBase64URL(dLogo, function (base64Img) {
                //console.log(base64Img);
                var _listItem = {
                    "__metadata": { 'type': 'SP.Data.DBSDistributorListItem' },

                    "BaseURL": base64Img
                };


                $.ajax({
                    url: _spPageContextInfo.webAbsoluteUrl + "/_api/Web/Lists/GetByTitle('DBSDistributorList')/Items(" + Id + ")",
                    method: 'POST',
                    contentType: "application/json;odata=verbose",
                    async: false,
                    data: JSON.stringify(_listItem),
                    headers: {
                        "accept": "application/json;odata=verbose",
                        "X-RequestDigest": $("#__REQUESTDIGEST").val(),
                        "content-Type": "application/json;odata=verbose",
                        "X-Http-Method": "MERGE",
                        "If-Match": "*"
                    },
                    success: onSuccess,
                    error: onError
                    /*success: function (data) {
                        deferred.resolve(data);
                    },
                    error: function (err) {
                        deferred.reject(err);
                    }*/   
                });
                //return deferred.promise();        
         });
       
        function onSuccess(data) {
           console.log('List Item Updated');
        }

        function onError(error) {
           console.log(JSON.stringify(error));
        }
      
   };

Conversion to base64

function convertImgToBase64URL(url, callback, outputFormat) {
        
             var canvas = document.createElement("CANVAS");
             var ctx = canvas.getContext("2d");
             ctx.clearRect( 0 , 0 , canvas.width, canvas.height );
             ctx.fillStyle="#FFFFFF";
             ctx.fillRect(0 , 0 , canvas.width, canvas.height);

            var img = new Image();
            img.crossOrigin = 'Anonymous';
            img.onload = function () {
                //var canvas = document.createElement('CANVAS'),
                   // ctx = canvas.getContext('2d'), dataURL;
                canvas.height = img.height;
                canvas.width = img.width;
                ctx.drawImage(img, 0, 0);
                var dataURL = canvas.toDataURL(outputFormat);
                callback(dataURL);
                canvas = null;
            };
            img.src = url;
        }

Would be grateful if anyone could help me with this.

Also please let me know if any other details are required.

Thanks in advance.

Answer 1

If your query to SharePoint reaches that 100 item limit, SharePoint provides you with the URL you need to get the next batch of items in it's response, in a property called __next that is right alongside the results array:

So what you need to do is make your function that requests all the list items recursive, and make it

1. Take the URL it should use as an argument
2. Check the results for a __next property
3. If the __next property is there, call itself using that URL

This also means moving the initial URL you want to start off with outside of that function. At it's simplest level using your code example, you'd want to do something like this:

// remake your function to accept the URL it should use as an argument
function GetItemId(requestUrl) {
    //debugger;
    var deferred = $.Deferred();
    $.ajax({
        url: requestUrl, // use the URL that was passed in
        method: "GET",
        type: 'GET',
        async: false,
        headers: {
            "accept": "application/json;odata=verbose",
            "content-Type": "application/json;odata=verbose"
        },
        success: function (result) {
            deferred.resolve(result);
            $.each(result.d.results, function (index, row) {
                UpdateListItemUsingItemId(row["ID"], row["DistributorLogo"]["Url"]);
            });
            
            // if there's a __next property,
            // call the function to get the next batch
            // by using the provided URL
            if (result.d.__next) {
                GetItemId(result.d.__next);
            }
        },
        error: function (result) {
            deferred.reject(result);
        }
    });

    return deferred.promise();
};

////
////
// wherever in your code you actually make the call to that function,
// build the initial URL and call your function
var initialUrl = _spPageContextInfo.webAbsoluteUrl + "/_api/Web/Lists/GetByTitle('DBSDistributorList')/Items?$select=ID,BaseURL,DistributorLogo";
GetItemId(initialUrl);

Personally though, I'd clean it up a little. Things are getting confusing in there. I'm not sure why you want to make that call non-async. Also, you are resolving your deferred before you even do your update processing. And with the addition I put in there to recursively call GetItemID, it now makes it so that you are doing the update processing in batches while simultaneously requesting the next batch of items. Very confusing.

So I'd clean it up so that you move the update processing out of that, so that you can cleanly get all the items before you start processing. And I'd let it be async, like it wants to be, and you don't need the content-type header on a GET request... here's how I'd put it together:

function GetAllItems(requestUrl) {
    var deferred = $.Deferred();
    $.ajax({
        url: requestUrl,
        method: "GET",
        headers: {
            "accept": "application/json;odata=verbose",
        }
    }).done(function (thisResult) {
        if (thisResult.d.__next) {
            // if the __next property is present,
            // that measn that there's more to get,
            // so don't start resolving things yet,
            // just make the next request
            var nextRequest = GetAllItems(thisResult.d.__next);
            
            // since GetAllItems returns a promise,
            // we have to wait for that promise to resolve
            // before resolving the promise we made 
            // at the beginning of this function
            nextRequest.done(function(nextResult) {
                // nextResult should be a plain array of items,
                // so we need to combine that with the array
                // of items we got in this call in thisResult.d.results
                var combinedResults = thisResult.d.results.concat(nextResult);
                
                // then resolve the combined results back up to the next level
                deferred.resolve(combinedResults);

            }).fail(function(error) {
                deferred.reject(error);
            });
        } else {
            // __next doesn't exist, so
            // there's nothing more to get, so
            // start resolving the nested promises
            deferred.resolve(thisResult.d.results);
        }
    }).fail(function (result) {
        deferred.reject(result);
    });
    return deferred.promise();
};

//// (somewhere else in your code)
////
// from wherever you want to start off the process,
// build the initial URL
var initialUrl = _spPageContextInfo.webAbsoluteUrl + "/_api/Web/Lists/GetByTitle('DBSDistributorList')/Items?$select=ID,BaseURL,DistributorLogo";

// start the process
var getAllItemsRequest = GetAllItems(initialUrl);

// wait for it all to complete
getAllItemsRequest.done(function (allItems) {
    // allItems should now be
    // an array of all 700+ items

    // make sure your request digest value is good
    // using the built-in SP provided function for doing just that
    UpdateFormDigest(_spPageContextInfo.webServerRelativeUrl, _spFormDigestRefreshInterval);
    
    // begin update processing
    $.each(allItems, function (index, row) {
        UpdateListItemUsingItemId(row["ID"], row["DistributorLogo"]["Url"]);
    });
}).fail(function (error) {
    console.warn("Error getting all items");
    console.warn(JSON.stringify(error));
});