1 /* 2 Copyright 2019 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package reporters 18 19 import ( 20 "bytes" 21 "encoding/json" 22 "fmt" 23 "io" 24 "net/http" 25 "strings" 26 "time" 27 28 "github.com/onsi/ginkgo/v2" 29 "github.com/onsi/ginkgo/v2/types" 30 "k8s.io/klog/v2" 31 ) 32 33 // ProgressReporter is a ginkgo reporter which tracks the total number of tests to be run/passed/failed/skipped. 34 // As new tests are completed it updates the values and prints them to stdout and optionally, sends the updates 35 // to the configured URL. 36 37 // One known limitation of the ProgressReporter it this reporter will not consolidate the reporter from each sub-process 38 // if the tests are run in parallel. 39 // As what's observed the reporter sent before test suite is started will be assembled correctly but each process will report 40 // on its own after each test or suite are completed. 41 // Here is a sample report that 5 testcases are executed totally, and run in parallel with 3 procs, 42 // 3 of them are failed and other 2 passed. 43 // {"msg":"","total":5,"completed":0,"skipped":0,"failed":0} 44 // {"msg":"Test Suite starting","total":5,"completed":0,"skipped":0,"failed":0} 45 // {"msg":"FAILED [sig-node] NoExecuteTaintManager Single Pod doesn't evict pod with tolerations from tainted nodes","total":0,"completed":0,"skipped":1332,"failed":1,"failures":["[sig-node] NoExecuteTaintManager..."]} 46 // {"msg":"FAILED [sig-node] NoExecuteTaintManager Single Pod evicts pods from tainted nodes","total":5,"completed":0,"skipped":2524,"failed":1,"failures":["[sig-node] NoExecuteTaintManager Single Pod evicts pods from tainted nodes"]} 47 // {"msg":"PASSED [sig-node] NoExecuteTaintManager Single Pod removing taint cancels eviction [Disruptive] [Conformance]","total":0,"completed":1,"skipped":1181,"failed":0} 48 // {"msg":"Test Suite completed","total":0,"completed":1,"skipped":2592,"failed":0} 49 // {"msg":"PASSED [sig-node] NoExecuteTaintManager Single Pod eventually evict pod with finite tolerations from tainted nodes","total":0,"completed":1,"skipped":1399,"failed":1,"failures":["[sig-node] NoExecuteTaintManager..."]} 50 // {"msg":"Test Suite completed","total":0,"completed":1,"skipped":1399,"failed":1,"failures":["[sig-node] NoExecuteTaintManager Single Pod doesn't evict pod with tolerations from tainted nodes"]} 51 // {"msg":"FAILED [sig-node] NoExecuteTaintManager Single Pod pods evicted from tainted nodes...","total":5,"completed":0,"skipped":3076,"failed":2,"failures":["[sig-node] NoExecuteTaintManager...","[sig-node] NoExecuteTaintManager..."]} 52 // {"msg":"Test Suite completed","total":5,"completed":0,"skipped":3076,"failed":2,"failures":["[sig-node] NoExecuteTaintManager Single Pod evicts pods from tainted nodes","[sig-node] NoExecuteTaintManager..."]} 53 type ProgressReporter struct { 54 LastMsg string `json:"msg"` 55 56 TestsTotal int `json:"total"` 57 TestsCompleted int `json:"completed"` 58 TestsSkipped int `json:"skipped"` 59 TestsFailed int `json:"failed"` 60 61 Failures []string `json:"failures,omitempty"` 62 63 progressURL string 64 client *http.Client 65 } 66 67 // NewProgressReporter returns a progress reporter which posts updates to the given URL. 68 func NewProgressReporter(progressReportURL string) *ProgressReporter { 69 rep := &ProgressReporter{ 70 Failures: []string{}, 71 progressURL: progressReportURL, 72 } 73 if len(progressReportURL) > 0 { 74 rep.client = &http.Client{ 75 Timeout: time.Second * 10, 76 } 77 } 78 return rep 79 } 80 81 // SendUpdates serializes the current progress and posts it to the configured endpoint if set. 82 // It does not print to stdout because that interferes with progress reporting by Ginko 83 // and (when Ginkgo does output redirection) doesn't actually appear on the screen anyway. 84 func (reporter *ProgressReporter) SendUpdates() { 85 // If a progressURL and client is set/available then POST to it. Noop otherwise. 86 if reporter.client == nil || len(reporter.progressURL) == 0 { 87 return 88 } 89 b := reporter.serialize() 90 go reporter.postProgressToURL(b) 91 } 92 93 func (reporter *ProgressReporter) postProgressToURL(b []byte) { 94 resp, err := reporter.client.Post(reporter.progressURL, "application/json", bytes.NewReader(b)) 95 if err != nil { 96 klog.Errorf("Failed to post progress update to %v: %v", reporter.progressURL, err) 97 return 98 } 99 if resp.StatusCode >= 400 { 100 klog.Errorf("Unexpected response when posting progress update to %v: %v", reporter.progressURL, resp.StatusCode) 101 if resp.Body != nil { 102 defer resp.Body.Close() 103 respBody, err := io.ReadAll(resp.Body) 104 if err != nil { 105 klog.Errorf("Failed to read response body from posting progress: %v", err) 106 return 107 } 108 klog.Errorf("Response body from posting progress update: %v", respBody) 109 } 110 111 return 112 } 113 } 114 115 func (reporter *ProgressReporter) serialize() []byte { 116 b, err := json.Marshal(reporter) 117 if err != nil { 118 return []byte(fmt.Sprintf(`{"msg":"%v", "error":"%v"}`, reporter.LastMsg, err)) 119 } 120 return b 121 } 122 123 func (reporter *ProgressReporter) SetStartMsg() { 124 reporter.LastMsg = "Test Suite starting" 125 reporter.SendUpdates() 126 } 127 128 func (reporter *ProgressReporter) SetTestsTotal(totalSpec int) { 129 reporter.TestsTotal = totalSpec 130 reporter.SendUpdates() 131 } 132 133 // ProcessSpecReport summarizes the report state and sends the state to the configured endpoint if set. 134 func (reporter *ProgressReporter) ProcessSpecReport(report ginkgo.SpecReport) { 135 testName := strings.Join(report.ContainerHierarchyTexts, " ") 136 if len(report.LeafNodeText) > 0 { 137 testName = testName + " " + report.LeafNodeText 138 } 139 switch report.State { 140 case types.SpecStateFailed: 141 if len(testName) > 0 { 142 reporter.Failures = append(reporter.Failures, testName) 143 } else { 144 reporter.Failures = append(reporter.Failures, "Unknown test name") 145 } 146 reporter.TestsFailed++ 147 reporter.LastMsg = fmt.Sprintf("FAILED %v", testName) 148 case types.SpecStatePassed: 149 reporter.TestsCompleted++ 150 reporter.LastMsg = fmt.Sprintf("PASSED %v", testName) 151 case types.SpecStateSkipped: 152 reporter.TestsSkipped++ 153 return 154 default: 155 return 156 } 157 158 reporter.SendUpdates() 159 } 160 161 func (reporter *ProgressReporter) SetEndMsg() { 162 reporter.LastMsg = "Test Suite completed" 163 reporter.SendUpdates() 164 } 165