6

I'm working on an automated security audit system which analyses custom Ubuntu installations to check for packages affected by security vulnerabilities. Since these machines are not connected to the internet I cannot simple run apt-get, and we need to keep the upgrades as small as possible so only updating packages when they are actually needed.

Part of this process is to monitor the Ubuntu Security Notices (USN) for new reports and extract information. However, I can't find a way to easily access all of the historical USN notices.

There is an RSS feed linked on http://www.ubuntu.com/usn/ but it only returns the most recent 50 reports, which won't work as I'm dealing with older Ubuntu versions and need to check historical reports.

The two options I can see, neither of which are nice, are to scrape the email archives or scrape the website. But I'd rather avoid doing these if possible...

Any ideas?

2 Answers 2

4

No, there is no API, but it should be fairly easy to create one by scraping the main CVE or partner CVE list into a db, instead of the USN website or email archive: enter image description here

1

Here is some node code (tested with node 6.10.0) that will scrape the contents of https://usn.ubuntu.com/usn/ into a spreadsheet.

Here is what the spreadsheet will look like:

CSV output as a spreadsheet

Here is the way to run the code:

var config = {};
config.searchIssuesAfter = new Date('2016-08-28');
config.reportName = 'Ubuntu Report';
config.securityNoticeURL = 'https://usn.ubuntu.com/usn/precise/';
config.usnBaseURL = 'https://usn.ubuntu.com';
config.numberOfPages = 6;
ubuntuReport = require('./getUbuntuData.js'); 
ubuntuReport.go(config);

Here is the supporting file getUbuntuData.js

var fs = require('fs');
var axios = require('axios');
var cheerio = require('cheerio');

module.exports = {
  go: function(config1) {
    //config
    var config = Object.assign({}, config1);
    var local = {};
    local.list = [];


    // Private functions  
    local.getPromiseUSNListFn = function(numPages){
      var promiseList = [];
      promiseList.push(axios.get(config.securityNoticeURL));
      for (var i = 2; i <= numPages; i++){
          promiseList.push(axios.get(config.securityNoticeURL+'?page='+i));
      }
      return promiseList;
    };

    local.sortByDateFn = function(a, b){
      return  b.timestamp - a.timestamp;
    };

    local.createSecurityNoticeListFn = function(response){
      var $ = cheerio.load(response.data);

      $('.notice').each( (i, elm) => {
        var securityNoticeNode = $(elm).find('h3 > a');
        var usn = securityNoticeNode.text().split(':')[0];
        var title = securityNoticeNode.text().split(': ')[1];
        var href = securityNoticeNode.attr("href");
        var dateNode = $(elm).find('h3');
        var dateStr = dateNode.contents().filter(function() {
          return this.nodeType == 3;
        }).text().trim();
        dateStr = dateStr.replace(/^\s*-\s*/,""); 
        var removeNewlinesRegEx = /(\r\n|\n|\r)/gm;
        dateStr = dateStr.replace(removeNewlinesRegEx,""); 
        dateStr = dateStr.replace(/(\d)(th|rd|nd|st)/,"$1");
        var date = new Date(dateStr);

        var infoObj = {
          usn: usn,
          title: title,
          href: config.usnBaseURL+href,
          timestamp: date.getTime(),
          date: dateStr
        };
        local.list.push(infoObj);
      });
    };

    local.getSecurityNoticeData = function(el){

        return axios.get(el.href).then( (response) => {
          el.summary = "";
          el.packages = [];
          el.versions = [];
          var $ = cheerio.load(response.data);
          console.log("Received : "+el.title);
          var removeNewlinesRegEx = /(\r\n|\n|\r)/gm;
          el.summary = $("h3:contains('Summary')").next().text().trim().replace(removeNewlinesRegEx," ");
          $("dl > dd").each((j, packageNode) => {
            el.packages.push($(packageNode).children().first().text().trim().replace(removeNewlinesRegEx," "));
            el.versions.push($(packageNode).children().first().next().text().trim().replace(removeNewlinesRegEx," "));
          });
        });
    };

    local.getPromiseListSecurityNoticesFn = function(){
      var promiseList = [];
      for (var i = 0; i < local.list.length; i++){
          promiseList.push(local.getSecurityNoticeData(local.list[i]));
      }
      return promiseList;
    };

    local.buildCSV = function(){
      var csv = "";
      csv += config.reportName+",,,,,,,,\n";
      csv += "Date,USN,Title,Package,Installed ?,Version,Urgency,Summary,Link\n";
      local.list.forEach((node) => {
        //node - final element
        var date = node.date;
        var usn = node.usn;
        var title = node.title;
        var packages = '"'+node.packages.join(", ")+'"';
        var installed = "?";
        var versions = '"'+node.versions.join(", ")+'"';
        var urgency = "?";
        var summary = '"'+node.summary+'"';
        var link = node.href;
        csv += date+",";
        csv += usn+",";
        csv += title+",";
        csv += packages+",";
        csv += installed+",";
        csv += versions+",";
        csv += urgency+",";
        csv += summary+",";
        csv += link;
        csv += "\n";
      });
      fs.writeFileSync(__dirname+"/../ubuntu_worksheet.csv", csv);
    };

    //Start 
    axios.all(local.getPromiseUSNListFn(config.numberOfPages))
      .then(axios.spread(function () {
        var i;
        for (i = 0; i < arguments.length; i++){
          console.log("Received page : "+i);
          local.createSecurityNoticeListFn(arguments[i]);
        }
        local.list.sort(local.sortByDateFn);
        for (i = 0; i < local.list.length; i++){
          if (local.list[i].timestamp < config.searchIssuesAfter.getTime()){
            //truncate list 
            local.list.length = i;
            break;
          }
        }
      }))
      .then(()=>{
        return axios.all(local.getPromiseListSecurityNoticesFn());
      }).then(()=>{
        local.buildCSV();
        console.log('data saved to ubuntu_worksheet.csv');
      });
   }
};

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .