Running WPT API for multiple URLs

Hi ,did any try out calling WPT.runTest with multiple request Urls?
Im facing an issue with callback not returning with this approach.
However, the same code works with static URL.

brands.forEach((brandEntries)=>{
    //Iterate over brands
    for(let brandEntry of brandEntries.entries()){
        let reqURL=brandEntry[1]
        let reqName=brandEntry[0]
        //setTimeout(() => {
            console.log('Scheduling a web page test for-'+brandEntry[0])
            
            WPT.runTest(reqURL,reqParams,(err,result)=>
            {
                if(err){
                    console.log('Unable run the test for reason-'+err)
                        throw err;
                }
                    console.log('call back got triggered for -'+reqName)
                    let date = convertEpoch(result.data.completed)
                    let transactionName=reqName;
                    let testId = result.data.id;
                   ---
               }
})
}

could you please specify the options you are passing(reqParams) and what is the response you are getting

Hi Abdul
Thanks for the reply.
Here are the options I’m using for my private instance
connectivity: ‘Cable’, location: ‘test’, firstViewOnly: false, runs: 1, pollResults: 10, video: true

You have mentioned callback not returning, could you please help with these questions

  1. Is your callback function not getting called?
  2. Is there any response you are getting from wpt.runTest function, if yes, what?
    Please verify the location you are sending as ‘test’ is not a valid location.

Also you could try wrapping the wpt.runTest in a Promise like as below : -
const runTest = (wpt, url, options) => {
let tempOptions = JSON.parse(JSON.stringify(options));
return new Promise((resolve, reject) => {
console.info(Submitting test for ${url}...);
wpt.runTest(url, tempOptions, async(err, result) => {
try {
if (result) {
return resolve({‘result’:result,‘err’:err});
} else {
return reject(err);
}
} catch (e) {
console.info(e);
}
})
});
}

And use it like below : -
brands.forEach(async (brandEntries)=>{
//Iterate over brands
for(let brandEntry of brandEntries.entries()){
let reqURL=brandEntry[1]
let reqName=brandEntry[0]
console.log(‘Scheduling a web page test for-’+brandEntry[0])
let wptResult = await runTest(wpt,reqURL,reqParams);
if(wptResult.err)
{
console.log(‘Unable run the test for reason-’+wptResult.err)
throw wptResult.err;
}
console.log(‘call back got triggered for -’+reqName)
let date = convertEpoch(wptResult.result.data.completed)
let transactionName=reqName;
let testId = result.data.id;

}
})

Do let me know if this works, also could be great if you could help me with the above questions.
Thanks.

HI Abdul,
Thanks a lot for your quick reply .
Here are my answers -

  1. Is your callback function not getting called?
    yes ,the callback doesn’t seem to return from the second iteration i,e for the second url.
    However, as I validate from the server logs the run test function seems to execute properly.

  2. Is there any response you are getting from wpt.runTest function, if yes, what?
    Please verify the location you are sending as ‘test’ is not a valid location.
    1st Iteration works fine - starting from second iteration - runTest doesn’t return any callback -
    program just continues to be in running state.
    sorry about the location - that was wrong
    Here are the complete list of parameters -
    let requestObj_config={
    location:‘Test’,
    connectivity:‘Cable’,
    Browser:‘Chrome’,
    runs:1,
    video:true,
    timeline:true,
    pollResults:10,
    waitResults:‘auto’,
    timeout:300
    }
    In terms of the code ,Im using almost similar code as you sent me , Here it is -
    const WPT =require(‘./GetWPTObj’)

const WebPageTest = require(‘WebPageTest’)

const reqParams = require(‘./RequestConfig’) //This would return parameters

const fs = require(‘fs’);

const util=require(‘util’);

const { on } = require(‘process’);

const dj_urls=[‘https://www.davidjones.com’,‘https://www.google.com’] //Array of urls

async function runTestSuite(){

for( const dj_url of dj_urls){

    try{

    const results=await runTest(dj_url) //Wait for the processing of one url to finish

    console.log('Finished for-'+dj_url)

    await processResults(results)

    console.log('********RESULTS START***********')

    console.log(results)

    console.log('********RESULTS END***********')

    }

    catch(err){

        console.log('Error in processing test result for'+dj_url+'-'+err)

    }

}

console.log('Processing Completed!!')

}

//Takes a URl and returns the response Object

async function runTest(reqUrl){

return new Promise(function(resolve,reject){

    console.log('Launching webpage test for'+reqUrl)

    WPT.runTest(reqUrl,reqParams,async function(err,res)    //Does Run test supports async callbacks?

    {

        try {

            return resolve(res)

        } 

        catch (error) {

            reject(err);

        }

      

    })

})

}

runTestSuite();

Hi Abdul,
I managed to get it working after using the below line that you mentioned -
let tempOptions = JSON.parse(JSON.stringify(options));
All this while I was sending the options as a JS Object [Object literal]
But things that I dont understand here are-
why does it work for the first Iteration?
Also is there any documentation that suggests the use of JSON object for options or is it something that is newly added.
I was referring to some of the existing documentation examples but never noticed this.

Again , Thanks a lot for your help !

regards
Umashankar.

Ah. I know this bug. :frowning:

I bumped it this myself recently. The node wrapper modifies the options object when it’s passed in. Specifidally, it converts the pollResults and timeout options from seconds to milliseconds. That’s fine the first iteration, but then the next time through a loop, it applies the same math.

So in your situation, your initial pollResults is 10s, which the wrapper converts to 10000ms. The next time through, because the options were directly modified, the wrapper sees 10000 passed in and treats it as if that was a seconds value, converting it to ms (to get 10000000 ms), and so on.

There’s an open issue. 2nd test run with same options object · Issue #55 · WebPageTest/webpagetest-api · GitHub

Let me see if I can jump on that quick and fix it.

2 Likes

Lol…I was going mad rechecking my promise code .
Thanks a lot for letting me know about it .