Custom Connector Load Pages - Sample code?

Options
SimonKing
SimonKing Member
edited June 10 in Connectors

I am building a custom connector and have managed to get it to get the data a parse (excluding columns which contain gaps). the data I have got so far though is page 1 of 51. Is there any sample code which can cycle around pulling in all the pages of data i.e. mine so far is:

else if (metadata.report == "Vulnerabilities"){
var res = httprequest.get("https://api.appcheck-ng.com/api/v1/"+metadata.account.apikey+"/vulnerabilities");

//display result
DOMO.log(res);
//Parse data
if(httprequest.getStatusCode() == 200) {
var jsonRes = JSON.parse(res).data;

datagrid.addColumn('Category', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Impact', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Target', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Probability', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Title', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Priority', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Host', datagrid.DATA_TYPE_STRING);
datagrid.addColumn('Port', datagrid.DATA_TYPE_STRING);

for(var i=0 ; i<jsonRes.length ; i++) {

  datagrid.addCell(jsonRes[i].category);
datagrid.addCell(jsonRes[i].impact);
datagrid.addCell(jsonRes[i].target);
datagrid.addCell(jsonRes[i].probability);
datagrid.addCell(jsonRes[i].title);
datagrid.addCell(jsonRes[i].priority);
datagrid.addCell(jsonRes[i].host);
datagrid.addCell(jsonRes[i].port);
datagrid.endRow();
}

}
}

but, this is just the start as the initial response is:

{"count": 2731, "success": true, "results": 250, "pages": 51, "message": "Vulnerabilities found: 2731 Showing: 250 Page: 1 of 51", "data": [{"category": "xxxx", "impact": "low", "target": "xxxx", "probability": "low", "title": "xxxx", "priority": "low", "host": "xxxx.com", "vuln_id": "xxxxxxxxxxxxxxxxxxxxxxxx", "parameter": "cookie.ex_v", "port": 443}, ……..

I can add "?page=2"); so know I can select pages:

{"count": 2731, "success": true, "results": 250, "pages": 51, "message": "Vulnerabilities found: 2731 Showing: 250 Page: 2 of 51", "data": [{"

Thanks

Best Answer

  • ArborRose
    ArborRose Coach
    Answer ✓
    Options

    For paginated data, you might try something like this:

    else if (metadata.report == "Vulnerabilities") {
    
    // Define base URL and API key
    var baseUrl = "https://api.appcheck-ng.com/api/v1/" + metadata.account.apikey + "/vulnerabilities";
    var allData = []; // Function to fetch data for a specific page
    function fetchPage(page) {
    var url = baseUrl + "?page=" + page;
    var res = httprequest.get(url);
    DOMO.log("Fetching page: " + page);
    DOMO.log(res);

    if (httprequest.getStatusCode() == 200) {
    var jsonRes = JSON.parse(res);
    allData = allData.concat(jsonRes.data);
    return jsonRes.meta.pagination.pages; // Return total number of pages
    } else {
    DOMO.log("Failed to fetch data: " + httprequest.getStatusCode());
    return 0; // Return 0 to stop the loop in case of an error
    }
    }

    // Fetch the first page to determine the total number of pages
    var totalPages = fetchPage(1);

    // Loop through remaining pages if there are more than one
    for (var page = 2; page <= totalPages; page++) {
    fetchPage(page);
    }

    // Define the schema for the data grid
    datagrid.addColumn('Category', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Impact', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Target', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Probability', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Title', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Priority', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Host', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Port', datagrid.DATA_TYPE_STRING);

    // Populate the data grid with all the fetched data
    for (var i = 0; i < allData.length; i++) {
    datagrid.addCell(allData[i].category);
    datagrid.addCell(allData[i].impact);
    datagrid.addCell(allData[i].target);
    datagrid.addCell(allData[i].probability);
    datagrid.addCell(allData[i].title);
    datagrid.addCell(allData[i].priority);
    datagrid.addCell(allData[i].host);
    datagrid.addCell(allData[i].port);
    datagrid.endRow();
    } }

    ** Was this post helpful? Click Agree or Like below. **
    ** Did this solve your problem? Accept it as a solution! **

Answers

  • ArborRose
    ArborRose Coach
    Answer ✓
    Options

    For paginated data, you might try something like this:

    else if (metadata.report == "Vulnerabilities") {
    
    // Define base URL and API key
    var baseUrl = "https://api.appcheck-ng.com/api/v1/" + metadata.account.apikey + "/vulnerabilities";
    var allData = []; // Function to fetch data for a specific page
    function fetchPage(page) {
    var url = baseUrl + "?page=" + page;
    var res = httprequest.get(url);
    DOMO.log("Fetching page: " + page);
    DOMO.log(res);

    if (httprequest.getStatusCode() == 200) {
    var jsonRes = JSON.parse(res);
    allData = allData.concat(jsonRes.data);
    return jsonRes.meta.pagination.pages; // Return total number of pages
    } else {
    DOMO.log("Failed to fetch data: " + httprequest.getStatusCode());
    return 0; // Return 0 to stop the loop in case of an error
    }
    }

    // Fetch the first page to determine the total number of pages
    var totalPages = fetchPage(1);

    // Loop through remaining pages if there are more than one
    for (var page = 2; page <= totalPages; page++) {
    fetchPage(page);
    }

    // Define the schema for the data grid
    datagrid.addColumn('Category', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Impact', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Target', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Probability', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Title', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Priority', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Host', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Port', datagrid.DATA_TYPE_STRING);

    // Populate the data grid with all the fetched data
    for (var i = 0; i < allData.length; i++) {
    datagrid.addCell(allData[i].category);
    datagrid.addCell(allData[i].impact);
    datagrid.addCell(allData[i].target);
    datagrid.addCell(allData[i].probability);
    datagrid.addCell(allData[i].title);
    datagrid.addCell(allData[i].priority);
    datagrid.addCell(allData[i].host);
    datagrid.addCell(allData[i].port);
    datagrid.endRow();
    } }

    ** Was this post helpful? Click Agree or Like below. **
    ** Did this solve your problem? Accept it as a solution! **

  • SimonKing
    Options

    Hi, thanks. This was great. The only changes I made was the returned total pages on line 16 I changed to

        return JSON.parse(res).pages; // Return total number of pages
    

    and I also commented out the log report on line 12 so my Console just shows

    Fetching page: 1

    Fetching page: 2

    Fetching page: 3

    Fetching page: 4

    Fetching page: 5

    ……..

    This can be used to create the dataset successfully :). Many thanks