Ignore errors returned from API calls nodeJS Ignore errors returned from API calls nodeJS
I have a NodeJS script which is making thousands of REST API GET calls to SharePoint to retrieve all folders and files recursively. A problem I have encountered is once in a while I am receiving ECONNRESET errors and 404 Not Found for some folders with unsupported characters. This causes my code to stop and I lose my progress in retrieving all the files and folders. Is there something I can do to prevent these errors from stopping my code and just have it ignore the error and continue onto the next folder? I thought catching the error and console logging it would suffice but this still ends the execution of the code. Thanks
//Imports and Declarations
const prompt = require("prompt-sync")({ sigint: true });
const { getCreds, pullFromSharePoint } = require('./helper');
const sprequest = require('sp-request');
var path = require('path');
let creds = getCreds();
let spr = sprequest.create(creds)
let SPUrl = 'url to sharepoint site';
let files = [];
let folders = [];
let scannedFolders = [];
let reiteration = false;
let processingReq = false;
  
//Prompt user for which folder to copy
let targetFolder = prompt("Folder to copy: ");
//Call function to get list of files and folders in specified folder
// getFolderContents(targetFolder);
processInfo([], targetFolder)
//Gets list of files/folders inside folder
function getFolderContents(targetFolder) {
    return new Promise((resolve, reject) => {
        logger.info('Scanning this folder: ' + targetFolder)
        //Skip if folder name contains chars we dont support
        if (targetFolder.includes("'")) {
            targetFolder = targetFolder.replace(/'/g, "''")
        }
        //If reiteration = true, format the target folder
        if (reiteration === true) {
            targetFolder = targetFolder.replace('/sites/sharepointsite', '');
        }
        
        console.log('Scanning this folder: ' + targetFolder)
        targetFolder = encodeURIComponent(targetFolder)
        
        var url = `${SPUrl}/_api/web/GetFolderByServerRelativeUrl('Shared%20Documents/${targetFolder}')/Files?$filter=TimeLastModified gt datetime'2021-07-01T00:00:00'`;
        var url2 = `${SPUrl}/_api/web/GetFolderByServerRelativeUrl('Shared%20Documents/${targetFolder}')?$expand=Folders`;
//THESE SPR CALLS ARE THE REST API CALLS TO SHAREPOINT RESULTING IN THE MOST 404 ERRORS, HOW TO IGNORE ERRORS FROM THESE?
        // Call to get list of files
        spr.get(url).then(response => {
            console.log('Calling SP API to get files')
            //Adds files to array
            let fileRes = response.body.d.results;
            for (let i = 0; i < fileRes.length; i++) {
                files.push(fileRes[i].ServerRelativeUrl)
            }
            console.log(files.length)
        }).catch(function (err) {
            console.log('Fetch Error :-S', err);
        });
        
        //Call to get list of folders
        spr.get(url2).then(response => {
            //Adds folders to array
            let folderRes = response.body.d.Folders.results;
            for (let j = 0; j < folderRes.length; j++) {
                folders.push(folderRes[j].ServerRelativeUrl)
            }
            //Push current folder read through to another array so we dont scan it multiple times
            scannedFolders.push('/sites/sharepointsite/Shared Documents' + targetFolder);
           
            resolve(folders);
        }).catch(function (err) {
            console.log('Fetch Error :-S', err);
        });
    })
}
//If folders exist in folders array scan them; once all folders have been scanned, send files to be copied
async function processInfo(folders, firstFolder) {
    let firstRun = await getFolderContents(firstFolder);
    let firstGone = false;
    if (firstRun) {
        if (firstRun.length > 0) {
            for (let k = 0; k < firstRun.length; k++) {
                await sleep(500);
                //If folder has already been scanned, remove from array and skip to next iteration
                if (scannedFolders.includes(firstRun[k])) {
                    if (k === 0) {
                        firstRun.splice(k, 1)
                    }
                    firstRun.splice(k, k)
                    continue;
                }
                //Else if folder has not been scanned, send it to be scanned
                else {
                    if (firstRun[k].includes('%') || firstRun[k].includes('%')) {
                        console.log('skipping')
                        continue;
                    }
                    reiteration = true;
                    let foldersScanned = await getFolderContents(firstRun[k]).catch(function (err) {
                        console.log('Fetch Error :-S', err);
                    });;
                    //Send each file to pullFile() function to be downloaded
                    // if (foldersScanned && k == firstRun.length - 1) {
                    if (firstRun[k] == 10) {
                        do {
                            await pullFile(files[0]);
                            files.shift();
                        } while (files.length > 0)
                    }
                }
            }
        }
    }
    console.log(files.length)
}
//Manipulate file string to get SP folder and File Name, then call the helper function to download the file
async function pullFile(file) {
    let filename = file.replace(/^.*[\\\/]/, '')
    let spFolder = file.replace('/sites/sharepointsite/', '')
    spFolder = spFolder.replace(filename, '');
    let downloadPath = path.join('./testfolder', spFolder)
    const sppullContext = {
        siteUrl: SPUrl, //SharePoint URL
        creds: creds    //Credentials
    };
    const sppullOptions = {
        spRootFolder: spFolder,      // The folder path for the file in SharePoint
        dlRootFolder: downloadPath,  // Where the file is saved locally
        strictObjects: [filename],  // Only download the filename specified as a query parameter
        muteConsole: true
    };
    pullFromSharePoint(sppullContext, sppullOptions)      //Pull file with filename from SharePoint
        .then(() => {
            return true;
        }).catch(err => {
            return err;
        });
}
//Timer to wait specified amount of time
function sleep(ms) {
    return new Promise((resolve) => {
        setTimeout(resolve, ms);
    });
}
Code from helper.js
//Imports and Declarations
var fs = require('fs');
const { SPPull } = require('sppull');
const winston = require('winston');
//Winston logger
const logger = winston.createLogger({
    level: 'info',
    format: winston.format.json(),
    defaultMeta: { service: 'user-service' },
    transports: [
        new winston.transports.File({ filename: 'error.log', level: 'error' }),
        new winston.transports.File({ filename: 'info.log' }),
    ],
});
/**
 * Returns the SharePoint credentials object to use.
 */
function getCreds() {
    return {
        username: 'UN',
        password: 'PW'
    };
}
module.exports.getCreds = getCreds;
/**
 * Pulls a file from SharePoint and then checks for errors.
 *
 * @param sppullContext context object for sppull
 * @param sppullOptions options object for sppull
 */
async function pullFromSharePoint(sppullContext, sppullOptions) {
    SPPull.download(sppullContext, sppullOptions)
        .then((res) => {
            logger.info(res)
            return res
        })
        .catch((err) => {
            logger.error(err)
            return err
        });
}
from Stackoverflow
Comments
Post a Comment