Custom Connector Load Pages - Sample code?

SimonKing

I am building a custom connector and have managed to get it to get the data a parse (excluding columns which contain gaps). the data I have got so far though is page 1 of 51. Is there any sample code which can cycle around pulling in all the pages of data i.e. mine so far is:

else if (metadata.report == "Vulnerabilities"){
var res = httprequest.get("https://api.appcheck-ng.com/api/v1/"+metadata.account.apikey+"/vulnerabilities");

//display result
DOMO.log(res);
//Parse data
if(httprequest.getStatusCode() == 200) {
var jsonRes = JSON.parse(res).data;

datagrid.addColumn('Category', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Impact', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Target', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Probability', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Title', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Priority', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Host', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Port', datagrid.DATA_TYPE_STRING);

for(var i=0 ; i<jsonRes.length ; i++) {

  datagrid.addCell(jsonRes[i].category);

  datagrid.addCell(jsonRes[i].impact);

  datagrid.addCell(jsonRes[i].target);

  datagrid.addCell(jsonRes[i].probability);

  datagrid.addCell(jsonRes[i].title);

  datagrid.addCell(jsonRes[i].priority);

  datagrid.addCell(jsonRes[i].host);

  datagrid.addCell(jsonRes[i].port);      

  datagrid.endRow();

}

}
}

but, this is just the start as the initial response is:

{"count": 2731, "success": true, "results": 250, "pages": 51, "message": "Vulnerabilities found: 2731 Showing: 250 Page: 1 of 51", "data": [{"category": "xxxx", "impact": "low", "target": "xxxx", "probability": "low", "title": "xxxx", "priority": "low", "host": "xxxx.com", "vuln_id": "xxxxxxxxxxxxxxxxxxxxxxxx", "parameter": "cookie.ex_v", "port": 443}, ……..

I can add "?page=2"); so know I can select pages:

{"count": 2731, "success": true, "results": 250, "pages": 51, "message": "Vulnerabilities found: 2731 Showing: 250 Page: 2 of 51", "data": [{"

Thanks

ArborRose

For paginated data, you might try something like this:

else if (metadata.report == "Vulnerabilities") {


// Define base URL and API key

var baseUrl = "https://api.appcheck-ng.com/api/v1/" + metadata.account.apikey + "/vulnerabilities";

var allData = [];

// Function to fetch data for a specific page

function fetchPage(page) {

    var url = baseUrl + "?page=" + page;

    var res = httprequest.get(url);

    DOMO.log("Fetching page: " + page);

    DOMO.log(res);


    if (httprequest.getStatusCode() == 200) {

        var jsonRes = JSON.parse(res);

        allData = allData.concat(jsonRes.data);

        return jsonRes.meta.pagination.pages; // Return total number of pages

    } else {

        DOMO.log("Failed to fetch data: " + httprequest.getStatusCode());

        return 0; // Return 0 to stop the loop in case of an error

    }

}
 



// Fetch the first page to determine the total number of pages

var totalPages = fetchPage(1);




// Loop through remaining pages if there are more than one

for (var page = 2; page <= totalPages; page++) {

    fetchPage(page);

}




// Define the schema for the data grid

datagrid.addColumn('Category', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Impact', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Target', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Probability', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Title', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Priority', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Host', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Port', datagrid.DATA_TYPE_STRING);




// Populate the data grid with all the fetched data

for (var i = 0; i < allData.length; i++) {

    datagrid.addCell(allData[i].category);

    datagrid.addCell(allData[i].impact);

    datagrid.addCell(allData[i].target);

    datagrid.addCell(allData[i].probability);

    datagrid.addCell(allData[i].title);

    datagrid.addCell(allData[i].priority);

    datagrid.addCell(allData[i].host);

    datagrid.addCell(allData[i].port);

    datagrid.endRow();

}
}

ArborRose

For paginated data, you might try something like this:

else if (metadata.report == "Vulnerabilities") {


// Define base URL and API key

var baseUrl = "https://api.appcheck-ng.com/api/v1/" + metadata.account.apikey + "/vulnerabilities";

var allData = [];

// Function to fetch data for a specific page

function fetchPage(page) {

    var url = baseUrl + "?page=" + page;

    var res = httprequest.get(url);

    DOMO.log("Fetching page: " + page);

    DOMO.log(res);


    if (httprequest.getStatusCode() == 200) {

        var jsonRes = JSON.parse(res);

        allData = allData.concat(jsonRes.data);

        return jsonRes.meta.pagination.pages; // Return total number of pages

    } else {

        DOMO.log("Failed to fetch data: " + httprequest.getStatusCode());

        return 0; // Return 0 to stop the loop in case of an error

    }

}
 



// Fetch the first page to determine the total number of pages

var totalPages = fetchPage(1);




// Loop through remaining pages if there are more than one

for (var page = 2; page <= totalPages; page++) {

    fetchPage(page);

}




// Define the schema for the data grid

datagrid.addColumn('Category', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Impact', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Target', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Probability', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Title', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Priority', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Host', datagrid.DATA_TYPE_STRING);

datagrid.addColumn('Port', datagrid.DATA_TYPE_STRING);




// Populate the data grid with all the fetched data

for (var i = 0; i < allData.length; i++) {

    datagrid.addCell(allData[i].category);

    datagrid.addCell(allData[i].impact);

    datagrid.addCell(allData[i].target);

    datagrid.addCell(allData[i].probability);

    datagrid.addCell(allData[i].title);

    datagrid.addCell(allData[i].priority);

    datagrid.addCell(allData[i].host);

    datagrid.addCell(allData[i].port);

    datagrid.endRow();

}
}

SimonKing

Hi, thanks. This was great. The only changes I made was the returned total pages on line 16 I changed to

    return JSON.parse(res).pages; // Return total number of pages

and I also commented out the log report on line 12 so my Console just shows

Fetching page: 1

Fetching page: 2

Fetching page: 3

Fetching page: 4

Fetching page: 5

……..

This can be used to create the dataset successfully :). Many thanks