Skip to content

    • Connectors

    Connectors

    Custom Connector Load Pages - Sample code?

    Member
    edited June 2024 in Connectors

    I am building a custom connector and have managed to get it to get the data a parse (excluding columns which contain gaps). the data I have got so far though is page 1 of 51. Is there any sample code which can cycle around pulling in all the pages of data i.e. mine so far is:

    else if (metadata.report == "Vulnerabilities"){
    var res = httprequest.get("https://api.appcheck-ng.com/api/v1/"+metadata.account.apikey+"/vulnerabilities");

    //display result
    DOMO.log(res);
    //Parse data
    if(httprequest.getStatusCode() == 200) {
    var jsonRes = JSON.parse(res).data;

    datagrid.addColumn('Category', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Impact', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Target', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Probability', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Title', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Priority', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Host', datagrid.DATA_TYPE_STRING);
    datagrid.addColumn('Port', datagrid.DATA_TYPE_STRING);

    for(var i=0 ; i<jsonRes.length ; i++) {

    1.   datagrid.addCell(jsonRes[i].category);
    2.   datagrid.addCell(jsonRes[i].impact);
    3.   datagrid.addCell(jsonRes[i].target);
    4.   datagrid.addCell(jsonRes[i].probability);
    5.   datagrid.addCell(jsonRes[i].title);
    6.   datagrid.addCell(jsonRes[i].priority);
    7.   datagrid.addCell(jsonRes[i].host);
    8.   datagrid.addCell(jsonRes[i].port);      
    9.   datagrid.endRow();
    10. }

    }
    }

    but, this is just the start as the initial response is:

    {"count": 2731, "success": true, "results": 250, "pages": 51, "message": "Vulnerabilities found: 2731 Showing: 250 Page: 1 of 51", "data": [{"category": "xxxx", "impact": "low", "target": "xxxx", "probability": "low", "title": "xxxx", "priority": "low", "host": "xxxx.com", "vuln_id": "xxxxxxxxxxxxxxxxxxxxxxxx", "parameter": "cookie.ex_v", "port": 443}, ……..

    I can add "?page=2"); so know I can select pages:

    {"count": 2731, "success": true, "results": 250, "pages": 51, "message": "Vulnerabilities found: 2731 Showing: 250 Page: 2 of 51", "data": [{"

    Thanks

    Tagged:

    Best Answer

    • Coach
      Answer ✓

      For paginated data, you might try something like this:

      1. else if (metadata.report == "Vulnerabilities") {

      3. // Define base URL and API key
      4. var baseUrl = "https://api.appcheck-ng.com/api/v1/" + metadata.account.apikey + "/vulnerabilities";
      5. var allData = [];
      6.  
      7. // Function to fetch data for a specific page
      8. function fetchPage(page) {
      9.     var url = baseUrl + "?page=" + page;
      10.     var res = httprequest.get(url);
      11.     DOMO.log("Fetching page: " + page);
      12.     DOMO.log(res);

      13.     if (httprequest.getStatusCode() == 200) {
      14.         var jsonRes = JSON.parse(res);
      15.         allData = allData.concat(jsonRes.data);
      16.         return jsonRes.meta.pagination.pages; // Return total number of pages
      17.     } else {
      18.         DOMO.log("Failed to fetch data: " + httprequest.getStatusCode());
      19.         return 0; // Return 0 to stop the loop in case of an error
      20.     }
      21. }
      22.  


      24. // Fetch the first page to determine the total number of pages
      25. var totalPages = fetchPage(1);
      26.  


      28. // Loop through remaining pages if there are more than one
      29. for (var page = 2; page <= totalPages; page++) {
      30.     fetchPage(page);
      31. }
      32.  


      34. // Define the schema for the data grid
      35. datagrid.addColumn('Category', datagrid.DATA_TYPE_STRING);
      36. datagrid.addColumn('Impact', datagrid.DATA_TYPE_STRING);
      37. datagrid.addColumn('Target', datagrid.DATA_TYPE_STRING);
      38. datagrid.addColumn('Probability', datagrid.DATA_TYPE_STRING);
      39. datagrid.addColumn('Title', datagrid.DATA_TYPE_STRING);
      40. datagrid.addColumn('Priority', datagrid.DATA_TYPE_STRING);
      41. datagrid.addColumn('Host', datagrid.DATA_TYPE_STRING);
      42. datagrid.addColumn('Port', datagrid.DATA_TYPE_STRING);
      43.  


      45. // Populate the data grid with all the fetched data
      46. for (var i = 0; i < allData.length; i++) {
      47.     datagrid.addCell(allData[i].category);
      48.     datagrid.addCell(allData[i].impact);
      49.     datagrid.addCell(allData[i].target);
      50.     datagrid.addCell(allData[i].probability);
      51.     datagrid.addCell(allData[i].title);
      52.     datagrid.addCell(allData[i].priority);
      53.     datagrid.addCell(allData[i].host);
      54.     datagrid.addCell(allData[i].port);
      55.     datagrid.endRow();
      56. }
      57. }

      ** Was this post helpful? Click Agree or Like below. **
      ** Did this solve your problem? Accept it as a solution! **

    Answers

    • Coach
      Answer ✓

      For paginated data, you might try something like this:

      1. else if (metadata.report == "Vulnerabilities") {

      3. // Define base URL and API key
      4. var baseUrl = "https://api.appcheck-ng.com/api/v1/" + metadata.account.apikey + "/vulnerabilities";
      5. var allData = [];
      6.  
      7. // Function to fetch data for a specific page
      8. function fetchPage(page) {
      9.     var url = baseUrl + "?page=" + page;
      10.     var res = httprequest.get(url);
      11.     DOMO.log("Fetching page: " + page);
      12.     DOMO.log(res);

      13.     if (httprequest.getStatusCode() == 200) {
      14.         var jsonRes = JSON.parse(res);
      15.         allData = allData.concat(jsonRes.data);
      16.         return jsonRes.meta.pagination.pages; // Return total number of pages
      17.     } else {
      18.         DOMO.log("Failed to fetch data: " + httprequest.getStatusCode());
      19.         return 0; // Return 0 to stop the loop in case of an error
      20.     }
      21. }
      22.  


      24. // Fetch the first page to determine the total number of pages
      25. var totalPages = fetchPage(1);
      26.  


      28. // Loop through remaining pages if there are more than one
      29. for (var page = 2; page <= totalPages; page++) {
      30.     fetchPage(page);
      31. }
      32.  


      34. // Define the schema for the data grid
      35. datagrid.addColumn('Category', datagrid.DATA_TYPE_STRING);
      36. datagrid.addColumn('Impact', datagrid.DATA_TYPE_STRING);
      37. datagrid.addColumn('Target', datagrid.DATA_TYPE_STRING);
      38. datagrid.addColumn('Probability', datagrid.DATA_TYPE_STRING);
      39. datagrid.addColumn('Title', datagrid.DATA_TYPE_STRING);
      40. datagrid.addColumn('Priority', datagrid.DATA_TYPE_STRING);
      41. datagrid.addColumn('Host', datagrid.DATA_TYPE_STRING);
      42. datagrid.addColumn('Port', datagrid.DATA_TYPE_STRING);
      43.  


      45. // Populate the data grid with all the fetched data
      46. for (var i = 0; i < allData.length; i++) {
      47.     datagrid.addCell(allData[i].category);
      48.     datagrid.addCell(allData[i].impact);
      49.     datagrid.addCell(allData[i].target);
      50.     datagrid.addCell(allData[i].probability);
      51.     datagrid.addCell(allData[i].title);
      52.     datagrid.addCell(allData[i].priority);
      53.     datagrid.addCell(allData[i].host);
      54.     datagrid.addCell(allData[i].port);
      55.     datagrid.endRow();
      56. }
      57. }

      ** Was this post helpful? Click Agree or Like below. **
      ** Did this solve your problem? Accept it as a solution! **

    • Member

      Hi, thanks. This was great. The only changes I made was the returned total pages on line 16 I changed to

      1.     return JSON.parse(res).pages; // Return total number of pages

      and I also commented out the log report on line 12 so my Console just shows

      Fetching page: 1

      Fetching page: 2

      Fetching page: 3

      Fetching page: 4

      Fetching page: 5

      ……..

      This can be used to create the dataset successfully :). Many thanks

    Welcome!

    It looks like you're new here. Members get access to exclusive content, events, rewards, and more. Sign in or register to get started.
    Sign In

    Welcome!

    It looks like you're new here. Members get access to exclusive content, events, rewards, and more. Sign in or register to get started.
    Sign In

    Quick Links

    Categories