From e09f49f944fa1d4163bfd52fd824f4216f93558f Mon Sep 17 00:00:00 2001 From: Xisco Fauli Date: Fri, 13 May 2022 15:30:23 +0200 Subject: bin: add script to retrieve info about crashreports... ... from https://crashreport.libreoffice.org The script saves the data into a .csv file Change-Id: I771d144402a3039851c99b025c52bd6e799f71ec Reviewed-on: https://gerrit.libreoffice.org/c/core/+/134283 Tested-by: Jenkins Reviewed-by: Xisco Fauli --- bin/crashreportScraper.py | 171 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100755 bin/crashreportScraper.py (limited to 'bin') diff --git a/bin/crashreportScraper.py b/bin/crashreportScraper.py new file mode 100755 index 000000000000..780db5a9dc2c --- /dev/null +++ b/bin/crashreportScraper.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Use this script to retrieve information from https://crashreport.libreoffice.org +# about a specific version of LibreOffice +# Usage sample: ./crashreportScraper.py 7.2.0.4 + +import requests +from bs4 import BeautifulSoup +import sys +import os +import math +from datetime import datetime + +def convert_str_to_date(value): + value = value.replace('.', '') + value = value.replace('March', 'Mar') + value = value.replace('April', 'Apr') + value = value.replace('June', 'Jun') + value = value.replace('July', 'Jul') + value = value.replace('Sept', 'Sep') + value = value.replace('noon', '12:00 pm') + + if ':' not in value: + if 'am' in value: + value = value.replace(' am', ':00 am') + elif 'pm' in value: + value = value.replace(' pm', ':00 pm') + + return datetime.strptime(value, '%b %d, %Y, %H:%M %p') + +def parse_version_url(url): + crashReports = {} + html_text = requests.get(url).text + soup = BeautifulSoup(html_text, 'html.parser') + + table = soup.find("table", {"id": "data-table"}).tbody + for tr in table.find_all("tr"): + td_list = tr.find_all("td") + crashName = td_list[0].a.text.strip() + crashNumber = int(td_list[1].text.strip()) + firstCrashDate = convert_str_to_date(td_list[5].text.strip()) + lastCrashDate = convert_str_to_date(td_list[6].text.strip()) + crashReports[crashName] = [crashNumber, firstCrashDate, lastCrashDate] + + return crashReports + +def parse_reports_and_get_most_recent_report_from_last_page(url): + html_text = requests.get(url).text + soup = BeautifulSoup(html_text, 'html.parser') + + count = 0 + os_tab = soup.find("table", {"id": "os_tab"}).tbody + tr_list = os_tab.find_all("tr") + for tr in tr_list: + td_list = tr.find_all("td") + count += int(td_list[1].text.strip()) + + # There are 50 reports on each page. + # Go to the last page based on the total count to get a recent report + last_page = math.ceil( count / 50 ) + + if last_page > 1: + url = url + "?page=" + str(last_page) + html_text = requests.get(url).text + soup = BeautifulSoup(html_text, 'html.parser') + + reports = soup.find("div", {"id": "reports"}).tbody + ID, currentID = "", "" + version, currentVersion = "", "" + OS, currentOS = "", "" + + tr_list = reports.find_all("tr") + for tr in tr_list: + td_list = tr.find_all("td") + + currentID = td_list[0].a.text.strip() + currentVersion = td_list[2].text.strip().split(': ')[1] + currentOS = td_list[3].text.strip() + + # get most recent version + # symbols on linux are not very informative generally + if currentOS == "windows" and currentVersion > version: + version = currentVersion + ID = currentID + OS = currentOS + + if not version: + version = currentVersion + + if not ID: + ID = currentID + + if not OS: + OS = currentOS + + return count, ID, version, OS + +def parse_details_and_get_info(url): + html_text = requests.get(url).text + soup = BeautifulSoup(html_text, 'html.parser') + + details = soup.find("div", {"id": "details"}).tbody + tr_list = details.find_all("tr") + reason = tr_list[8].td.text.strip() + + stack = "" + count = 0 + frames = soup.find("div", {"id": "frames"}).tbody + for tr in frames.find_all("tr"): + td_list = tr.find_all("td") + source = td_list[3].text.strip() + if source and count <= 10: + source = source.replace("\\", "/").replace("C:/cygwin64/home/buildslave/source/libo-core/", "") + stack += source + "\n" + count += 1 + + if stack: + #multiline + stack = "\"" + stack + "\"" + return reason, stack + + +if __name__ == '__main__': + + version = sys.argv[1] + + crashes = parse_version_url( + "https://crashreport.libreoffice.org/stats/version/" + version + "?limit=1000&days=30") + + print(str(len(crashes)) + " crash reports in version " + version) + + crashesInFile = [] + fileName = "crashes_" + version.replace(".", "_") + ".csv" + print("Using " + fileName) + + bInsertHeader = False + if os.path.exists(fileName): + with open(fileName, "r") as f: + lines = f.readlines() + for line in lines: + crashesInFile.append(line.split("\t")[0]) + else: + bInsertHeader = True + + with open(fileName, "a") as f: + if bInsertHeader: + line = '\t'.join(["Name", "Count", "First report", "Last Report", + "ID", "Version", "Reason", "OS", "Stack", '\n']) + f.write(line) + f.flush() + + for k, v in crashes.items(): + # ignore unresolved crash signatures + if len(k) < 254 and k not in crashesInFile and '`' not in k and not k.lower().endswith('.dll') and \ + not k.lower().endswith('.so') and ".so." not in k.lower(): + print("Parsing " + k) + crashCount, crashID, crashVersion, crashOS = parse_reports_and_get_most_recent_report_from_last_page( + "https://crashreport.libreoffice.org/stats/signature/" + k) + crashReason, crashStack = parse_details_and_get_info( + "https://crashreport.libreoffice.org/stats/crash_details/" + crashID) + line = '\t'.join([k, str(crashCount), v[1].strftime('%y/%m/%d'), v[2].strftime('%y/%m/%d'), + crashID, crashVersion, crashReason, crashOS, crashStack, '\n']) + f.write(line) + f.flush() + -- cgit