appveyor log - use JSON-like console data, massaged for parsability
This commit is contained in:
Mark Pictor
parent
274de2a91d
commit
c23ba65c41
1 changed files with 158 additions and 49 deletions
|
|
@ -1,5 +1,7 @@
|
|||
//summarize MSVC errors from an appveyor log
|
||||
// compile with 'go build summarize-appveyor-log.go'
|
||||
// takes 0 or 1 args; with 0, gets log from latest
|
||||
// build. with 1, uses that file as raw json-like log
|
||||
package main
|
||||
|
||||
import (
|
||||
|
|
@ -17,22 +19,29 @@ import (
|
|||
const (
|
||||
headerKey = "Authorization"
|
||||
headerVal = "Bearer %s"
|
||||
projUrl = "https://ci.appveyor.com/api/projects/mpictor/stepcode"
|
||||
projUrl = "https://ci.appveyor.com/api/projects/mpictor/stepcode"
|
||||
//"https://ci.appveyor.com/api/buildjobs/2rjxdv1rnb8jcg8y/log"
|
||||
logUrl = "https://ci.appveyor.com/api/buildjobs/%s/log"
|
||||
logUrl = "https://ci.appveyor.com/api/buildjobs/%s/log"
|
||||
consoleUrl = "https://ci.appveyor.com/api/buildjobs/%s/console"
|
||||
)
|
||||
|
||||
//uses stdin and stdout
|
||||
func main() {
|
||||
rawlog, build, err := getLog()
|
||||
var rawlog io.ReadCloser
|
||||
var build string
|
||||
var err error
|
||||
if len(os.Args) == 2 {
|
||||
rawlog, build, err = processArgv()
|
||||
} else {
|
||||
rawlog, build, err = getLog()
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
|
||||
return
|
||||
}
|
||||
defer rawlog.Close()
|
||||
log := unwrap(rawlog)
|
||||
log := decodeConsole(rawlog)
|
||||
warns, errs := countMessages(log)
|
||||
fi, err := os.Create(fmt.Sprintf("appveyor-%d.smy", build))
|
||||
fi, err := os.Create(fmt.Sprintf("appveyor-%s.smy", build))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
|
||||
return
|
||||
|
|
@ -46,20 +55,19 @@ func main() {
|
|||
|
||||
/* categorizes warnings and errors based upon the MSVC message number (i.e. C4244)
|
||||
* the regex will match lines like
|
||||
[ 00:03:42] c:\projects\stepcode\src\base\sc_benchmark.h(45): warning C4251: 'benchmark::descr' : class 'std::basic_string<char,std::char_traits<char>,std::allocator<char>>' needs to have dll-interface to be used by clients of class 'benchmark' [C:\projects\STEPcode\build\src\base\base.vcxproj]
|
||||
c:\projects\stepcode\src\base\sc_benchmark.h(45): warning C4251: 'benchmark::descr' : class 'std::basic_string<char,std::char_traits<char>,std::allocator<char>>' needs to have dll-interface to be used by clients of class 'benchmark' [C:\projects\STEPcode\build\src\base\base.vcxproj]
|
||||
[00:03:48] C:\projects\STEPcode\src\base\sc_benchmark.cc(61): warning C4244: '=' : conversion from 'SIZE_T' to 'long', possible loss of data [C:\projects\STEPcode\build\src\base\base.vcxproj]*
|
||||
*/
|
||||
func countMessages(log []string) (warns, errs map[string][]string) {
|
||||
warns = make(map[string][]string)
|
||||
errs = make(map[string][]string)
|
||||
tstamp := `\[\d\d:\d\d:\d\d\] `
|
||||
fname := " *(.*)" // $1
|
||||
fline := `(?:\((\d+)\)| ): ` // $2 - either line number in parenthesis or a space, followed by a colon
|
||||
msgNr := `([A-Z]+\d+): ` // $3 - C4251, LNK2005, etc
|
||||
msgTxt := `([^\[]*) ` // $4
|
||||
tail := `\[[^\[\]]*\]`
|
||||
warnRe := regexp.MustCompile(tstamp + fname + fline + `warning ` + msgNr + msgTxt + tail)
|
||||
errRe := regexp.MustCompile(tstamp + fname + fline + `(?:fatal )?error ` + msgNr + msgTxt + tail)
|
||||
warnRe := regexp.MustCompile(fname + fline + `warning ` + msgNr + msgTxt + tail)
|
||||
errRe := regexp.MustCompile(fname + fline + `(?:fatal )?error ` + msgNr + msgTxt + tail)
|
||||
for _, line := range log {
|
||||
if warnRe.MatchString(line) {
|
||||
key := warnRe.ReplaceAllString(line, "$3")
|
||||
|
|
@ -125,45 +133,102 @@ func printMessages(typ string, m map[string][]string, w io.Writer) {
|
|||
}
|
||||
}
|
||||
|
||||
//
|
||||
func unwrap(r io.Reader) (log []string) {
|
||||
startNewLine := true
|
||||
unwrapScanner := bufio.NewScanner(r)
|
||||
var lineOut string
|
||||
for unwrapScanner.Scan() {
|
||||
lastNewline := startNewLine
|
||||
lineIn := unwrapScanner.Text()
|
||||
startNewLine = (len(lineIn) < 240) || strings.HasSuffix(lineIn, "vcxproj]")
|
||||
if !lastNewline {
|
||||
lineOut += lineIn[11:]
|
||||
} else {
|
||||
lineOut = lineIn
|
||||
}
|
||||
if startNewLine {
|
||||
log = append(log, lineOut)
|
||||
lineOut = ""
|
||||
}
|
||||
}
|
||||
if len(lineOut) > 0 {
|
||||
log = append(log, lineOut)
|
||||
}
|
||||
if err := unwrapScanner.Err(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "Error reading appveyor log:", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
//structs from http://json2struct.mervine.net/
|
||||
|
||||
//http://json2struct.mervine.net/
|
||||
//{"values":[{"i":0,"t":"Specify a project or solution file. The directory does not contain a project or solution file.\r\n","dt":"00:00:04","bg":12,"fg":15}]}
|
||||
type AppVeyorConsoleLines struct {
|
||||
Values []struct {
|
||||
I int `json:"i"`
|
||||
Text string `json:"t"`
|
||||
DateTime string `json:"dt"`
|
||||
BgColor int `json:"bg"`
|
||||
FgColor int `json:"fg"`
|
||||
}
|
||||
}
|
||||
type AppVeyorBuild struct {
|
||||
Build struct {
|
||||
BuildNumber int `json:"buildNumber"`
|
||||
Jobs []struct {
|
||||
/*BuildNumber int `json:"buildNumber"`*/
|
||||
Version string `json:"version"`
|
||||
Jobs []struct {
|
||||
JobID string `json:"jobId"`
|
||||
} `json:"jobs"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
func getLog() (log io.ReadCloser, build int, err error) {
|
||||
func splitAppend(log *[]string, blob string) {
|
||||
//blob = strings.Replace(blob,"\r\n", "\n",-1)
|
||||
blob = strings.Replace(blob, "\\", "/", -1)
|
||||
r := strings.NewReader(blob)
|
||||
unwrapScanner := bufio.NewScanner(r)
|
||||
for unwrapScanner.Scan() {
|
||||
txt := unwrapScanner.Text()
|
||||
//fmt.Printf("%s\n", txt)
|
||||
*log = append(*log, txt)
|
||||
}
|
||||
}
|
||||
|
||||
//calculate length of string without escape chars
|
||||
// func escapeLen(s string)(l int) {
|
||||
// //s = strings.Replace(s,"\\\\", "/",-1)
|
||||
// s = strings.Replace(s,"\\\"", "",-1)
|
||||
// s = strings.Replace(s,"\r\n", "RN",-1)
|
||||
// return len(s)
|
||||
// }
|
||||
|
||||
|
||||
//decode the almost-JSON console data from appveyor
|
||||
func decodeConsole(r io.Reader) (log []string) {
|
||||
wrapper := Wrap(r)
|
||||
dec := json.NewDecoder(wrapper)
|
||||
var consoleLines AppVeyorConsoleLines
|
||||
var err error
|
||||
var txtBlob string
|
||||
err = dec.Decode(&consoleLines)
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
if err == nil {
|
||||
for _, l := range consoleLines.Values {
|
||||
txtBlob += l.Text
|
||||
//el := escapeLen(l.Text)
|
||||
//something inserts newlines at 229 chars (+\n\r == 231) (found in CMake output)
|
||||
lenTwoThreeOne := len(l.Text) == 231
|
||||
if lenTwoThreeOne {
|
||||
txtBlob = strings.TrimSuffix(txtBlob, "\r\n")
|
||||
}
|
||||
//something else starts new log lines at 1024 chars without inserting newlines (found in CTest error output)
|
||||
if len(l.Text) != 1024 && !lenTwoThreeOne {
|
||||
//fmt.Printf("sa for l %d, el %d\n", len(l.Text),el)
|
||||
splitAppend(&log, txtBlob)
|
||||
txtBlob = ""
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("decode err %s\n", err)
|
||||
}
|
||||
if len(txtBlob) > 0 {
|
||||
splitAppend(&log, txtBlob)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func processArgv() (log io.ReadCloser, build string, err error) {
|
||||
fname := os.Args[1]
|
||||
if len(fname) < 14 {
|
||||
err = fmt.Errorf("Name arg '%s' too short. Run as '%s appveyor-NNN.log'", fname, os.Args[0])
|
||||
return
|
||||
}
|
||||
buildRe := regexp.MustCompile(`appveyor-(.+).log`)
|
||||
build = buildRe.ReplaceAllString(fname, "$1")
|
||||
if len(build) == 0 {
|
||||
err = fmt.Errorf("No build id in %s", fname)
|
||||
return
|
||||
}
|
||||
log, err = os.Open(fname)
|
||||
return
|
||||
}
|
||||
|
||||
func getLog() (log io.ReadCloser, build string, err error) {
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("GET", projUrl, nil)
|
||||
if err != nil {
|
||||
|
|
@ -171,22 +236,23 @@ func getLog() (log io.ReadCloser, build int, err error) {
|
|||
}
|
||||
apikey := os.Getenv("APPVEYOR_API_KEY")
|
||||
//api key isn't necessary for read-only queries on public projects
|
||||
//if len(apikey) < 1 {
|
||||
if len(apikey) > 0 {
|
||||
req.Header.Add(headerKey, fmt.Sprintf(headerVal, apikey))
|
||||
} //else {
|
||||
// fmt.Printf("Env var APPVEYOR_API_KEY is not set.")
|
||||
//}
|
||||
req.Header.Add(headerKey, fmt.Sprintf(headerVal,apikey))
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
build, job := decode(resp.Body)
|
||||
fmt.Printf("build #%d, jobId %s\n", build, job)
|
||||
resp, err = http.Get(fmt.Sprintf(logUrl, job))
|
||||
build, job := decodeProjInfo(resp.Body)
|
||||
fmt.Printf("build #%s, jobId %s\n", build, job)
|
||||
resp, err = http.Get(fmt.Sprintf(consoleUrl, job))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
logName := fmt.Sprintf("appveyor-%d.log", build)
|
||||
logName := fmt.Sprintf("appveyor-%s.log", build)
|
||||
fi, err := os.Create(logName)
|
||||
if err != nil {
|
||||
return
|
||||
|
|
@ -202,7 +268,7 @@ func getLog() (log io.ReadCloser, build int, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
func decode(r io.Reader) (num int, job string) {
|
||||
func decodeProjInfo(r io.Reader) (vers string, job string) {
|
||||
dec := json.NewDecoder(r)
|
||||
var av AppVeyorBuild
|
||||
err := dec.Decode(&av)
|
||||
|
|
@ -213,9 +279,52 @@ func decode(r io.Reader) (num int, job string) {
|
|||
if len(av.Build.Jobs) != 1 {
|
||||
return
|
||||
}
|
||||
num = av.Build.BuildNumber
|
||||
vers = av.Build.Version
|
||||
job = av.Build.Jobs[0].JobID
|
||||
return
|
||||
}
|
||||
|
||||
//wrap a reader, modifying content to make the json decoder happy
|
||||
//only tested with data from appveyor console
|
||||
type jsonWrapper struct {
|
||||
source io.Reader
|
||||
begin bool
|
||||
end bool
|
||||
}
|
||||
|
||||
func Wrap(r io.Reader) *jsonWrapper {
|
||||
return &jsonWrapper{
|
||||
source: r,
|
||||
begin: true,
|
||||
}
|
||||
}
|
||||
|
||||
// func nonNeg(n int) (int) {
|
||||
// if n < 0 {
|
||||
// return 0
|
||||
// }
|
||||
// return n
|
||||
// }
|
||||
|
||||
func (w *jsonWrapper) Read(p []byte) (n int, err error) {
|
||||
if w.end {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if w.begin {
|
||||
w.begin = false
|
||||
n = copy(p, []byte(`{"values":[`))
|
||||
}
|
||||
m, err := w.source.Read(p[n:])
|
||||
n += m
|
||||
if err == io.EOF {
|
||||
w.end = true
|
||||
if n < len(p) {
|
||||
n = copy(p, []byte(`{"dummy":"data"}]}`))
|
||||
} else {
|
||||
err = fmt.Errorf("No room to terminate JSON struct with '}'\n")
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// kate: indent-width 8; space-indent off; replace-tabs off; replace-tabs-save off; replace-trailing-space-save on; remove-trailing-space on; tab-intent on; tab-width 8; show-tabs off;
|
||||
|
|
|
|||
Loading…
Reference in a new issue